diff --git a/.gitignore b/.gitignore index 241cf9630..132070bf3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ doc/generated examples/.ipynb_checkpoints venv .uv-lock +uv.lock # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/Makefile b/Makefile index b097bd1f9..a25e2972c 100644 --- a/Makefile +++ b/Makefile @@ -20,11 +20,9 @@ inplace: test-code: in $(PYTEST) -s -v tests -test-doc: - $(PYTEST) -s -v doc/*.rst test-coverage: rm -rf coverage .coverage $(PYTEST) -s -v --cov=. tests -test: test-code test-sphinxext test-doc +test: test-code diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 068f69872..5584e6438 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -6,8 +6,6 @@ Please make sure that: * the title of the pull request is descriptive * this pull requests is against the `develop` branch -* for any new function or class added, please add it to doc/api.rst - * the list of classes and functions should be alphabetical * for any new functionality, consider adding a relevant example * add unit tests for new functionalities * collect files uploaded to test server using _mark_entity_for_removal() diff --git a/doc/.nojekyll b/doc/.nojekyll deleted file mode 100644 index 8b1378917..000000000 --- a/doc/.nojekyll +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/Makefile b/doc/Makefile deleted file mode 100644 index 767a9927b..000000000 --- a/doc/Makefile +++ /dev/null @@ -1,181 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -all: html - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - rm -rf generated/ - rm -rf examples/ - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/OpenML.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/OpenML.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/OpenML" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/OpenML" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/doc/_static/codehighlightstyle.css b/doc/_static/codehighlightstyle.css deleted file mode 100644 index ab16693ee..000000000 --- a/doc/_static/codehighlightstyle.css +++ /dev/null @@ -1,7 +0,0 @@ -.highlight .n { color: #000000 } /* code */ -.highlight .c1 { color: #1d8908 } /* comments */ -.highlight .mi { color: #0d9fe3; font-weight: bold } /* integers */ -.highlight .s1 { color: #d73c00 } /* string */ -.highlight .o { color: #292929 } /* operators */ - /* Background color for code highlights. Color for bash highlights */ -pre { background-color: #fbfbfb; color: #000000 } diff --git a/doc/_templates/class.rst b/doc/_templates/class.rst deleted file mode 100644 index 72405badb..000000000 --- a/doc/_templates/class.rst +++ /dev/null @@ -1,8 +0,0 @@ -:orphan: - -:mod:`{{module}}`.{{objname}} -{{ underline }}============== - -.. currentmodule:: {{ module }} - -.. autoclass:: {{ objname }} diff --git a/doc/_templates/class_without_init.rst b/doc/_templates/class_without_init.rst deleted file mode 100644 index 79ff2cf80..000000000 --- a/doc/_templates/class_without_init.rst +++ /dev/null @@ -1,12 +0,0 @@ -:mod:`{{module}}`.{{objname}} -{{ underline }}============== - -.. currentmodule:: {{ module }} - -.. autoclass:: {{ objname }} - -.. include:: {{module}}.{{objname}}.examples - -.. raw:: html - -
diff --git a/doc/_templates/function.rst b/doc/_templates/function.rst deleted file mode 100644 index d8c9bd480..000000000 --- a/doc/_templates/function.rst +++ /dev/null @@ -1,10 +0,0 @@ -:mod:`{{module}}`.{{objname}} -{{ underline }}==================== - -.. currentmodule:: {{ module }} - -.. autofunction:: {{ objname }} - -.. raw:: html - -
diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html deleted file mode 100644 index 11777457e..000000000 --- a/doc/_templates/layout.html +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "!layout.html" %} - -{# Custom CSS overrides #} -{# set bootswatch_css_custom = ['_static/my-styles.css'] #} - -{# Add github banner (from: https://github.com/blog/273-github-ribbons). #} -{% block header %} - {{ super() }} - - -{% endblock %} - diff --git a/doc/api.rst b/doc/api.rst deleted file mode 100644 index 288bf66fb..000000000 --- a/doc/api.rst +++ /dev/null @@ -1,295 +0,0 @@ -:orphan: - -.. _api: - -API -*** - -Modules -======= - -:mod:`openml.datasets` ----------------------- -.. automodule:: openml.datasets - :no-members: - :no-inherited-members: - -Dataset Classes -~~~~~~~~~~~~~~~ - -.. currentmodule:: openml.datasets - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - OpenMLDataFeature - OpenMLDataset - -Dataset Functions -~~~~~~~~~~~~~~~~~ - -.. currentmodule:: openml.datasets - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - attributes_arff_from_df - check_datasets_active - create_dataset - delete_dataset - get_dataset - get_datasets - list_datasets - list_qualities - status_update - edit_dataset - fork_dataset - -:mod:`openml.evaluations` -------------------------- -.. automodule:: openml.evaluations - :no-members: - :no-inherited-members: - -Evaluations Classes -~~~~~~~~~~~~~~~~~~~ - -.. currentmodule:: openml.evaluations - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - OpenMLEvaluation - -Evaluations Functions -~~~~~~~~~~~~~~~~~~~~~ - -.. currentmodule:: openml.evaluations - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - list_evaluations - list_evaluation_measures - list_evaluations_setups - -:mod:`openml.flows`: Flow Functions ------------------------------------ -.. automodule:: openml.flows - :no-members: - :no-inherited-members: - -Flow Classes -~~~~~~~~~~~~ - -.. currentmodule:: openml.flows - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - OpenMLFlow - -Flow Functions -~~~~~~~~~~~~~~ - -.. currentmodule:: openml.flows - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - assert_flows_equal - delete_flow - flow_exists - get_flow - list_flows - -:mod:`openml.runs`: Run Functions ----------------------------------- -.. automodule:: openml.runs - :no-members: - :no-inherited-members: - -Run Classes -~~~~~~~~~~~ - -.. currentmodule:: openml.runs - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - OpenMLRun - -Run Functions -~~~~~~~~~~~~~ - -.. currentmodule:: openml.runs - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - delete_run - get_run - get_runs - get_run_trace - initialize_model_from_run - initialize_model_from_trace - list_runs - run_model_on_task - run_flow_on_task - run_exists - -:mod:`openml.setups`: Setup Functions -------------------------------------- -.. automodule:: openml.setups - :no-members: - :no-inherited-members: - -Setup Classes -~~~~~~~~~~~~~ - -.. currentmodule:: openml.setups - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - OpenMLParameter - OpenMLSetup - -Setup Functions -~~~~~~~~~~~~~~~ - -.. currentmodule:: openml.setups - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - get_setup - initialize_model - list_setups - setup_exists - -:mod:`openml.study`: Study Functions ------------------------------------- -.. automodule:: openml.study - :no-members: - :no-inherited-members: - -Study Classes -~~~~~~~~~~~~~ - -.. currentmodule:: openml.study - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - OpenMLBenchmarkSuite - OpenMLStudy - -Study Functions -~~~~~~~~~~~~~~~ - -.. currentmodule:: openml.study - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - attach_to_study - attach_to_suite - create_benchmark_suite - create_study - delete_study - delete_suite - detach_from_study - detach_from_suite - get_study - get_suite - list_studies - list_suites - update_study_status - update_suite_status - -:mod:`openml.tasks`: Task Functions ------------------------------------ -.. automodule:: openml.tasks - :no-members: - :no-inherited-members: - -Task Classes -~~~~~~~~~~~~ - -.. currentmodule:: openml.tasks - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - OpenMLClassificationTask - OpenMLClusteringTask - OpenMLLearningCurveTask - OpenMLRegressionTask - OpenMLSplit - OpenMLSupervisedTask - OpenMLTask - TaskType - -Task Functions -~~~~~~~~~~~~~~ - -.. currentmodule:: openml.tasks - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - create_task - delete_task - get_task - get_tasks - list_tasks - -.. _api_extensions: - -Extensions -========== - -.. automodule:: openml.extensions - :no-members: - :no-inherited-members: - -Extension Classes ------------------ - -.. currentmodule:: openml.extensions - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - Extension - sklearn.SklearnExtension - -Extension Functions -------------------- - -.. currentmodule:: openml.extensions - -.. autosummary:: - :toctree: generated/ - :template: function.rst - - get_extension_by_flow - get_extension_by_model - register_extension - diff --git a/doc/conf.py b/doc/conf.py deleted file mode 100644 index 61ba4a46c..000000000 --- a/doc/conf.py +++ /dev/null @@ -1,353 +0,0 @@ -# -*- coding: utf-8 -*- -# -# OpenML documentation build configuration file, created by -# sphinx-quickstart on Wed Nov 26 10:46:10 2014. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import os -import sys -import sphinx_bootstrap_theme -import time -import openml - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')# ) - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.autosummary", - "sphinx.ext.doctest", - "sphinx.ext.coverage", - "sphinx.ext.mathjax", - "sphinx.ext.ifconfig", - "sphinx.ext.autosectionlabel", - "sphinx_gallery.gen_gallery", - "numpydoc", -] - -autosummary_generate = True -numpydoc_show_class_members = False - -autodoc_default_options = {"members": True, "inherited-members": True} - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = "OpenML" -copyright = f"2014-{time.localtime().tm_year}, the OpenML-Python team" - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = openml.__version__ -# The full version, including alpha/beta/rc tags. -release = openml.__version__ - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ["_build", "_templates", "_static"] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -# keep_warnings = False - -# Complain about all broken internal links - broken external links can be -# found with `make linkcheck` -# -# currently disabled because without intersphinx we cannot link to numpy.ndarray -# nitpicky = True -linkcheck_ignore = [r"https://test.openml.org/t/.*"] # FIXME: to avoid test server bugs avoiding docs building -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = "bootstrap" - -html_theme_options = { - # Navigation bar title. (Default: ``project`` value) - "navbar_title": "OpenML", - # Tab name for entire site. (Default: "Site") - # 'navbar_site_name': "Site", - # A list of tuples containting pages to link to. The value should - # be in the form [(name, page), ..] - "navbar_links": [ - ("Start", "index"), - ("User Guide", "usage"), - ("API", "api"), - ("Examples", "examples/index"), - ("Extensions", "extensions"), - ("Contributing", "contributing"), - ("Changelog", "progress"), - ], - # Render the next and previous page links in navbar. (Default: true) - "navbar_sidebarrel": False, - # Render the current pages TOC in the navbar. (Default: true) - "navbar_pagenav": False, - # Tab name for the current pages TOC. (Default: "Page") - "navbar_pagenav_name": "On this page", - # Global TOC depth for "site" navbar tab. (Default: 1) - # Switching to -1 shows all levels. - "globaltoc_depth": 1, - # Include hidden TOCs in Site navbar? - # - # Note: If this is "false", you cannot have mixed ``:hidden:`` and - # non-hidden ``toctree`` directives in the same page, or else the build - # will break. - # - # Values: "true" (default) or "false" - "globaltoc_includehidden": "false", - # HTML navbar class (Default: "navbar") to attach to
element. - # For black navbar, do "navbar navbar-inverse" - "navbar_class": "navbar", - # Fix navigation bar to top of page? - # Values: "true" (default) or "false" - "navbar_fixed_top": "true", - # Location of link to source. - # Options are "nav" (default), "footer" or anything else to exclude. - "source_link_position": "None", - # Bootswatch (http://bootswatch.com/) theme. - # - # Options are nothing with "" (default) or the name of a valid theme - # such as "amelia" or "cosmo". - "bootswatch_theme": "flatly", - # Choose Bootstrap version. - # Values: "3" (default) or "2" (in quotes) - "bootstrap_version": "3", -} - -# Add any paths that contain custom themes here, relative to this directory. -html_theme_path = sphinx_bootstrap_theme.get_html_theme_path() - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -# html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -html_sidebars = {"**": ["localtoc.html"]} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_domain_indices = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = "OpenMLdoc" - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # 'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ("index", "OpenML.tex", "OpenML Documentation", "Matthias Feurer", "manual"), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False - -# If true, show page references after internal links. -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# latex_show_urls = False - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [("index", "openml", "OpenML Documentation", ["Matthias Feurer"], 1)] - -# If true, show URL addresses after external links. -# man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ( - "index", - "OpenML", - "OpenML Documentation", - "Matthias Feurer", - "OpenML", - "One line description of project.", - "Miscellaneous", - ), -] - -# Documents to append as an appendix to all manuals. -# texinfo_appendices = [] - -# If false, no module index is generated. -# texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -# texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -# texinfo_no_detailmenu = False - -# prefix each section label with the name of the document it is in, -# in order to avoid ambiguity when there are multiple same section -# labels in different documents. -autosectionlabel_prefix_document = True -# Sphinx-gallery configuration. -sphinx_gallery_conf = { - # disable mini galleries clustered by the used functions - "backreferences_dir": None, - # path to the examples - "examples_dirs": "../examples", - # path where to save gallery generated examples - "gallery_dirs": "examples", - # compile execute examples in the examples dir - "filename_pattern": ".*example.py$|.*tutorial.py$", - # TODO: fix back/forward references for the examples. -} - - -def setup(app): - app.add_css_file("codehighlightstyle.css") - app.warningiserror = True diff --git a/doc/contributing.rst b/doc/contributing.rst deleted file mode 100644 index affe597de..000000000 --- a/doc/contributing.rst +++ /dev/null @@ -1,25 +0,0 @@ -:orphan: - -.. _contributing: - -============ -Contributing -============ - -Contribution to the OpenML package is highly appreciated in all forms. -In particular, a few ways to contribute to openml-python are: - - * A direct contribution to the package, by means of improving the - code, documentation or examples. To get started, see `this file `_ - with details on how to set up your environment to develop for openml-python. - - * A contribution to an openml-python extension. An extension package allows OpenML to interface - with a machine learning package (such as scikit-learn or keras). These extensions - are hosted in separate repositories and may have their own guidelines. - For more information, see the :ref:`extensions`. - - * `Cite OpenML `_ if you use it in a scientific publication. - - * Visit one of our `hackathons `_. - - * Contribute to another OpenML project, such as `the main OpenML project `_. diff --git a/doc/extensions.rst b/doc/extensions.rst deleted file mode 100644 index 0e3d7989e..000000000 --- a/doc/extensions.rst +++ /dev/null @@ -1,165 +0,0 @@ -:orphan: - -.. _extensions: - -========== -Extensions -========== - -OpenML-Python provides an extension interface to connect other machine learning libraries than -scikit-learn to OpenML. Please check the :ref:`api_extensions` and use the -scikit-learn extension in :class:`openml.extensions.sklearn.SklearnExtension` as a starting point. - -List of extensions -================== - -Here is a list of currently maintained OpenML extensions: - -* :class:`openml.extensions.sklearn.SklearnExtension` -* `openml-keras `_ -* `openml-pytorch `_ -* `openml-tensorflow (for tensorflow 2+) `_ - - -Connecting new machine learning libraries -========================================= - -Content of the Library -~~~~~~~~~~~~~~~~~~~~~~ - -To leverage support from the community and to tap in the potential of OpenML, -interfacing with popular machine learning libraries is essential. -The OpenML-Python package is capable of downloading meta-data and results (data, -flows, runs), regardless of the library that was used to upload it. -However, in order to simplify the process of uploading flows and runs from a -specific library, an additional interface can be built. -The OpenML-Python team does not have the capacity to develop and maintain such -interfaces on its own. For this reason, we -have built an extension interface to allows others to contribute back. Building a suitable -extension for therefore requires an understanding of the current OpenML-Python support. - -The :ref:`sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py` tutorial -shows how scikit-learn currently works with OpenML-Python as an extension. The *sklearn* -extension packaged with the `openml-python `_ -repository can be used as a template/benchmark to build the new extension. - - -API -+++ -* The extension scripts must import the `openml` package and be able to interface with - any function from the OpenML-Python :ref:`api`. -* The extension has to be defined as a Python class and must inherit from - :class:`openml.extensions.Extension`. -* This class needs to have all the functions from `class Extension` overloaded as required. -* The redefined functions should have adequate and appropriate docstrings. The - `Sklearn Extension API :class:`openml.extensions.sklearn.SklearnExtension.html` - is a good example to follow. - - -Interfacing with OpenML-Python -++++++++++++++++++++++++++++++ -Once the new extension class has been defined, the openml-python module to -:meth:`openml.extensions.register_extension` must be called to allow OpenML-Python to -interface the new extension. - -The following methods should get implemented. Although the documentation in -the `Extension` interface should always be leading, here we list some additional -information and best practices. -The `Sklearn Extension API :class:`openml.extensions.sklearn.SklearnExtension.html` -is a good example to follow. Note that most methods are relatively simple and can be implemented in several lines of code. - -* General setup (required) - - * :meth:`can_handle_flow`: Takes as argument an OpenML flow, and checks - whether this can be handled by the current extension. The OpenML database - consists of many flows, from various workbenches (e.g., scikit-learn, Weka, - mlr). This method is called before a model is being deserialized. - Typically, the flow-dependency field is used to check whether the specific - library is present, and no unknown libraries are present there. - * :meth:`can_handle_model`: Similar as :meth:`can_handle_flow`, except that - in this case a Python object is given. As such, in many cases, this method - can be implemented by checking whether this adheres to a certain base class. -* Serialization and De-serialization (required) - - * :meth:`flow_to_model`: deserializes the OpenML Flow into a model (if the - library can indeed handle the flow). This method has an important interplay - with :meth:`model_to_flow`. - Running these two methods in succession should result in exactly the same - model (or flow). This property can be used for unit testing (e.g., build a - model with hyperparameters, make predictions on a task, serialize it to a flow, - deserialize it back, make it predict on the same task, and check whether the - predictions are exactly the same.) - The example in the scikit-learn interface might seem daunting, but note that - here some complicated design choices were made, that allow for all sorts of - interesting research questions. It is probably good practice to start easy. - * :meth:`model_to_flow`: The inverse of :meth:`flow_to_model`. Serializes a - model into an OpenML Flow. The flow should preserve the class, the library - version, and the tunable hyperparameters. - * :meth:`get_version_information`: Return a tuple with the version information - of the important libraries. - * :meth:`create_setup_string`: No longer used, and will be deprecated soon. -* Performing runs (required) - - * :meth:`is_estimator`: Gets as input a class, and checks whether it has the - status of estimator in the library (typically, whether it has a train method - and a predict method). - * :meth:`seed_model`: Sets a random seed to the model. - * :meth:`_run_model_on_fold`: One of the main requirements for a library to - generate run objects for the OpenML server. Obtains a train split (with - labels) and a test split (without labels) and the goal is to train a model - on the train split and return the predictions on the test split. - On top of the actual predictions, also the class probabilities should be - determined. - For classifiers that do not return class probabilities, this can just be the - hot-encoded predicted label. - The predictions will be evaluated on the OpenML server. - Also, additional information can be returned, for example, user-defined - measures (such as runtime information, as this can not be inferred on the - server). - Additionally, information about a hyperparameter optimization trace can be - provided. - * :meth:`obtain_parameter_values`: Obtains the hyperparameters of a given - model and the current values. Please note that in the case of a hyperparameter - optimization procedure (e.g., random search), you only should return the - hyperparameters of this procedure (e.g., the hyperparameter grid, budget, - etc) and that the chosen model will be inferred from the optimization trace. - * :meth:`check_if_model_fitted`: Check whether the train method of the model - has been called (and as such, whether the predict method can be used). -* Hyperparameter optimization (optional) - - * :meth:`instantiate_model_from_hpo_class`: If a given run has recorded the - hyperparameter optimization trace, then this method can be used to - reinstantiate the model with hyperparameters of a given hyperparameter - optimization iteration. Has some similarities with :meth:`flow_to_model` (as - this method also sets the hyperparameters of a model). - Note that although this method is required, it is not necessary to implement - any logic if hyperparameter optimization is not implemented. Simply raise - a `NotImplementedError` then. - -Hosting the library -~~~~~~~~~~~~~~~~~~~ - -Each extension created should be a stand-alone repository, compatible with the -`OpenML-Python repository `_. -The extension repository should work off-the-shelf with *OpenML-Python* installed. - -Create a `public Github repo `_ -with the following directory structure: - -:: - -| [repo name] -| |-- [extension name] -| | |-- __init__.py -| | |-- extension.py -| | |-- config.py (optionally) - -Recommended -~~~~~~~~~~~ -* Test cases to keep the extension up to date with the `openml-python` upstream changes. -* Documentation of the extension API, especially if any new functionality added to OpenML-Python's - extension design. -* Examples to show how the new extension interfaces and works with OpenML-Python. -* Create a PR to add the new extension to the OpenML-Python API documentation. - -Happy contributing! diff --git a/doc/index.rst b/doc/index.rst deleted file mode 100644 index 4ab56f5c3..000000000 --- a/doc/index.rst +++ /dev/null @@ -1,113 +0,0 @@ -.. OpenML documentation master file, created by - sphinx-quickstart on Wed Nov 26 10:46:10 2014. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -====== -OpenML -====== - -**Collaborative Machine Learning in Python** - -Welcome to the documentation of the OpenML Python API, a connector to the -collaborative machine learning platform `OpenML.org `_. -The OpenML Python package allows to use datasets and tasks from OpenML together -with scikit-learn and share the results online. - -------- -Example -------- - -.. code:: python - - import openml - from sklearn import impute, tree, pipeline - - # Define a scikit-learn classifier or pipeline - clf = pipeline.Pipeline( - steps=[ - ('imputer', impute.SimpleImputer()), - ('estimator', tree.DecisionTreeClassifier()) - ] - ) - # Download the OpenML task for the pendigits dataset with 10-fold - # cross-validation. - task = openml.tasks.get_task(32) - # Run the scikit-learn model on the task. - run = openml.runs.run_model_on_task(clf, task) - # Publish the experiment on OpenML (optional, requires an API key. - # You can get your own API key by signing up to OpenML.org) - run.publish() - print(f'View the run online: {run.openml_url}') - -You can find more examples in our :ref:`examples-index`. - ----------------------------- -How to get OpenML for python ----------------------------- -You can install the OpenML package via `pip`: - -.. code:: bash - - pip install openml - -For more advanced installation information, please see the -:ref:`installation` section. - -------- -Content -------- - -* :ref:`usage` -* :ref:`api` -* :ref:`examples-index` -* :ref:`extensions` -* :ref:`contributing` -* :ref:`progress` - -------------------- -Further information -------------------- - -* `OpenML documentation `_ -* `OpenML client APIs `_ -* `OpenML developer guide `_ -* `Contact information `_ -* `Citation request `_ -* `OpenML blog `_ -* `OpenML twitter account `_ - ------------- -Contributing ------------- - -Contribution to the OpenML package is highly appreciated. The OpenML package -currently has a 1/4 position for the development and all help possible is -needed to extend and maintain the package, create new examples and improve -the usability. Please see the :ref:`contributing` page for more information. - --------------------- -Citing OpenML-Python --------------------- - -If you use OpenML-Python in a scientific publication, we would appreciate a -reference to the following paper: - -| Matthias Feurer, Jan N. van Rijn, Arlind Kadra, Pieter Gijsbers, Neeratyoy Mallik, Sahithya Ravi, Andreas Müller, Joaquin Vanschoren, Frank Hutter -| **OpenML-Python: an extensible Python API for OpenML** -| Journal of Machine Learning Research, 22(100):1−5, 2021 -| `https://www.jmlr.org/papers/v22/19-920.html `_ - - Bibtex entry:: - - @article{JMLR:v22:19-920, - author = {Matthias Feurer and Jan N. van Rijn and Arlind Kadra and Pieter Gijsbers and Neeratyoy Mallik and Sahithya Ravi and Andreas Müller and Joaquin Vanschoren and Frank Hutter}, - title = {OpenML-Python: an extensible Python API for OpenML}, - journal = {Journal of Machine Learning Research}, - year = {2021}, - volume = {22}, - number = {100}, - pages = {1--5}, - url = {http://jmlr.org/papers/v22/19-920.html} - } - diff --git a/doc/progress.rst b/doc/progress.rst deleted file mode 100644 index 3bf7c05aa..000000000 --- a/doc/progress.rst +++ /dev/null @@ -1,378 +0,0 @@ -:orphan: - -.. _progress: - -============================================= -Changelog (discontinued after version 0.15.0) -============================================= - -See GitHub releases for the latest changes. -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -0.15.0 -~~~~~~ - - * ADD #1335: Improve MinIO support. - * Add progress bar for downloading MinIO files. Enable it with setting `show_progress` to true on either `openml.config` or the configuration file. - * When using `download_all_files`, files are only downloaded if they do not yet exist in the cache. - * FIX #1338: Read the configuration file without overwriting it. - * MAINT #1340: Add Numpy 2.0 support. Update tests to work with scikit-learn <= 1.5. - * ADD #1342: Add HTTP header to requests to indicate they are from openml-python. - * ADD #1345: `task.get_dataset` now takes the same parameters as `openml.datasets.get_dataset` to allow fine-grained control over file downloads. - * MAINT #1346: The ARFF file of a dataset is now only downloaded if parquet is not available. - * MAINT #1349: Removed usage of the `disutils` module, which allows for Py3.12 compatibility. - * MAINT #1351: Image archives are now automatically deleted after they have been downloaded and extracted. - * MAINT #1352, 1354: When fetching tasks and datasets, file download parameters now default to not downloading the file. - Files will be downloaded only when a user tries to access properties which require them (e.g., `dataset.qualities` or `dataset.get_data`). - - -0.14.2 -~~~~~~ - - * MAINT #1280: Use the server-provided ``parquet_url`` instead of ``minio_url`` to determine the location of the parquet file. - * ADD #716: add documentation for remaining attributes of classes and functions. - * ADD #1261: more annotations for type hints. - * MAINT #1294: update tests to new tag specification. - * FIX #1314: Update fetching a bucket from MinIO. - * FIX #1315: Make class label retrieval more lenient. - * ADD #1316: add feature descriptions ontologies support. - * MAINT #1310/#1307: switch to ruff and resolve all mypy errors. - -0.14.1 -~~~~~~ - - * FIX: Fallback on downloading ARFF when failing to download parquet from MinIO due to a ServerError. - -0.14.0 -~~~~~~ - -**IMPORTANT:** This release paves the way towards a breaking update of OpenML-Python. From version -0.15, functions that had the option to return a pandas DataFrame will return a pandas DataFrame -by default. This version (0.14) emits a warning if you still use the old access functionality. -More concretely: - -* In 0.15 we will drop the ability to return dictionaries in listing calls and only provide - pandas DataFrames. To disable warnings in 0.14 you have to request a pandas DataFrame - (using ``output_format="dataframe"``). -* In 0.15 we will drop the ability to return datasets as numpy arrays and only provide - pandas DataFrames. To disable warnings in 0.14 you have to request a pandas DataFrame - (using ``dataset_format="dataframe"``). - -Furthermore, from version 0.15, OpenML-Python will no longer download datasets and dataset metadata -by default. This version (0.14) emits a warning if you don't explicitly specifiy the desired behavior. - -Please see the pull requests #1258 and #1260 for further information. - -* ADD #1081: New flag that allows disabling downloading dataset features. -* ADD #1132: New flag that forces a redownload of cached data. -* FIX #1244: Fixes a rare bug where task listing could fail when the server returned invalid data. -* DOC #1229: Fixes a comment string for the main example. -* DOC #1241: Fixes a comment in an example. -* MAINT #1124: Improve naming of helper functions that govern the cache directories. -* MAINT #1223, #1250: Update tools used in pre-commit to the latest versions (``black==23.30``, ``mypy==1.3.0``, ``flake8==6.0.0``). -* MAINT #1253: Update the citation request to the JMLR paper. -* MAINT #1246: Add a warning that warns the user that checking for duplicate runs on the server cannot be done without an API key. - -0.13.1 -~~~~~~ - -* ADD #1081 #1132: Add additional options for (not) downloading datasets ``openml.datasets.get_dataset`` and cache management. -* ADD #1028: Add functions to delete runs, flows, datasets, and tasks (e.g., ``openml.datasets.delete_dataset``). -* ADD #1144: Add locally computed results to the ``OpenMLRun`` object's representation if the run was created locally and not downloaded from the server. -* ADD #1180: Improve the error message when the checksum of a downloaded dataset does not match the checksum provided by the API. -* ADD #1201: Make ``OpenMLTraceIteration`` a dataclass. -* DOC #1069: Add argument documentation for the ``OpenMLRun`` class. -* DOC #1241 #1229 #1231: Minor documentation fixes and resolve documentation examples not working. -* FIX #1197 #559 #1131: Fix the order of ground truth and predictions in the ``OpenMLRun`` object and in ``format_prediction``. -* FIX #1198: Support numpy 1.24 and higher. -* FIX #1216: Allow unknown task types on the server. This is only relevant when new task types are added to the test server. -* FIX #1223: Fix mypy errors for implicit optional typing. -* MAINT #1155: Add dependabot github action to automatically update other github actions. -* MAINT #1199: Obtain pre-commit's flake8 from github.com instead of gitlab.com. -* MAINT #1215: Support latest numpy version. -* MAINT #1218: Test Python3.6 on Ubuntu 20.04 instead of the latest Ubuntu (which is 22.04). -* MAINT #1221 #1212 #1206 #1211: Update github actions to the latest versions. - -0.13.0 -~~~~~~ - - * FIX #1030: ``pre-commit`` hooks now no longer should issue a warning. - * FIX #1058, #1100: Avoid ``NoneType`` error when printing task without ``class_labels`` attribute. - * FIX #1110: Make arguments to ``create_study`` and ``create_suite`` that are defined as optional by the OpenML XSD actually optional. - * FIX #1147: ``openml.flow.flow_exists`` no longer requires an API key. - * FIX #1184: Automatically resolve proxies when downloading from minio. Turn this off by setting environment variable ``no_proxy="*"``. - * MAINT #1088: Do CI for Windows on Github Actions instead of Appveyor. - * MAINT #1104: Fix outdated docstring for ``list_task``. - * MAINT #1146: Update the pre-commit dependencies. - * ADD #1103: Add a ``predictions`` property to OpenMLRun for easy accessibility of prediction data. - * ADD #1188: EXPERIMENTAL. Allow downloading all files from a minio bucket with ``download_all_files=True`` for ``get_dataset``. - - -0.12.2 -~~~~~~ - -* ADD #1065: Add a ``retry_policy`` configuration option that determines the frequency and number of times to attempt to retry server requests. -* ADD #1075: A docker image is now automatically built on a push to develop. It can be used to build docs or run tests in an isolated environment. -* ADD: You can now avoid downloading 'qualities' meta-data when downloading a task with the ``download_qualities`` parameter of ``openml.tasks.get_task[s]`` functions. -* DOC: Fixes a few broken links in the documentation. -* DOC #1061: Improve examples to always show a warning when they switch to the test server. -* DOC #1067: Improve documentation on the scikit-learn extension interface. -* DOC #1068: Create dedicated extensions page. -* FIX #1075: Correctly convert `y` to a pandas series when downloading sparse data. -* MAINT: Rename `master` brach to ` main` branch. -* MAINT/DOC: Automatically check for broken external links when building the documentation. -* MAINT/DOC: Fail documentation building on warnings. This will make the documentation building - fail if a reference cannot be found (i.e. an internal link is broken). - -0.12.1 -~~~~~~ - -* ADD #895/#1038: Measure runtimes of scikit-learn runs also for models which are parallelized - via the joblib. -* DOC #1050: Refer to the webpage instead of the XML file in the main example. -* DOC #1051: Document existing extensions to OpenML-Python besides the shipped scikit-learn - extension. -* FIX #1035: Render class attributes and methods again. -* ADD #1049: Add a command line tool for configuration openml-python. -* FIX #1042: Fixes a rare concurrency issue with OpenML-Python and joblib which caused the joblib - worker pool to fail. -* FIX #1053: Fixes a bug which could prevent importing the package in a docker container. - -0.12.0 -~~~~~~ -* ADD #964: Validate ``ignore_attribute``, ``default_target_attribute``, ``row_id_attribute`` are set to attributes that exist on the dataset when calling ``create_dataset``. -* ADD #979: Dataset features and qualities are now also cached in pickle format. -* ADD #982: Add helper functions for column transformers. -* ADD #989: ``run_model_on_task`` will now warn the user the the model passed has already been fitted. -* ADD #1009 : Give possibility to not download the dataset qualities. The cached version is used even so download attribute is false. -* ADD #1016: Add scikit-learn 0.24 support. -* ADD #1020: Add option to parallelize evaluation of tasks with joblib. -* ADD #1022: Allow minimum version of dependencies to be listed for a flow, use more accurate minimum versions for scikit-learn dependencies. -* ADD #1023: Add admin-only calls for adding topics to datasets. -* ADD #1029: Add support for fetching dataset from a minio server in parquet format. -* ADD #1031: Generally improve runtime measurements, add them for some previously unsupported flows (e.g. BaseSearchCV derived flows). -* DOC #973 : Change the task used in the welcome page example so it no longer fails using numerical dataset. -* MAINT #671: Improved the performance of ``check_datasets_active`` by only querying the given list of datasets in contrast to querying all datasets. Modified the corresponding unit test. -* MAINT #891: Changed the way that numerical features are stored. Numerical features that range from 0 to 255 are now stored as uint8, which reduces the storage space required as well as storing and loading times. -* MAINT #975, #988: Add CI through Github Actions. -* MAINT #977: Allow ``short`` and ``long`` scenarios for unit tests. Reduce the workload for some unit tests. -* MAINT #985, #1000: Improve unit test stability and output readability, and adds load balancing. -* MAINT #1018: Refactor data loading and storage. Data is now compressed on the first call to `get_data`. -* MAINT #1024: Remove flaky decorator for study unit test. -* FIX #883 #884 #906 #972: Various improvements to the caching system. -* FIX #980: Speed up ``check_datasets_active``. -* FIX #984: Add a retry mechanism when the server encounters a database issue. -* FIX #1004: Fixed an issue that prevented installation on some systems (e.g. Ubuntu). -* FIX #1013: Fixes a bug where ``OpenMLRun.setup_string`` was not uploaded to the server, prepares for ``run_details`` being sent from the server. -* FIX #1021: Fixes an issue that could occur when running unit tests and openml-python was not in PATH. -* FIX #1037: Fixes a bug where a dataset could not be loaded if a categorical value had listed nan-like as a possible category. - -0.11.0 -~~~~~~ -* ADD #753: Allows uploading custom flows to OpenML via OpenML-Python. -* ADD #777: Allows running a flow on pandas dataframes (in addition to numpy arrays). -* ADD #888: Allow passing a `task_id` to `run_model_on_task`. -* ADD #894: Support caching of datasets using feather format as an option. -* ADD #929: Add ``edit_dataset`` and ``fork_dataset`` to allow editing and forking of uploaded datasets. -* ADD #866, #943: Add support for scikit-learn's `passthrough` and `drop` when uploading flows to - OpenML. -* ADD #879: Add support for scikit-learn's MLP hyperparameter `layer_sizes`. -* ADD #894: Support caching of datasets using feather format as an option. -* ADD #945: PEP 561 compliance for distributing Type information. -* DOC #660: Remove nonexistent argument from docstring. -* DOC #901: The API reference now documents the config file and its options. -* DOC #912: API reference now shows `create_task`. -* DOC #954: Remove TODO text from documentation. -* DOC #960: document how to upload multiple ignore attributes. -* FIX #873: Fixes an issue which resulted in incorrect URLs when printing OpenML objects after - switching the server. -* FIX #885: Logger no longer registered by default. Added utility functions to easily register - logging to console and file. -* FIX #890: Correct the scaling of data in the SVM example. -* MAINT #371: ``list_evaluations`` default ``size`` changed from ``None`` to ``10_000``. -* MAINT #767: Source distribution installation is now unit-tested. -* MAINT #781: Add pre-commit and automated code formatting with black. -* MAINT #804: Rename arguments of list_evaluations to indicate they expect lists of ids. -* MAINT #836: OpenML supports only pandas version 1.0.0 or above. -* MAINT #865: OpenML no longer bundles test files in the source distribution. -* MAINT #881: Improve the error message for too-long URIs. -* MAINT #897: Dropping support for Python 3.5. -* MAINT #916: Adding support for Python 3.8. -* MAINT #920: Improve error messages for dataset upload. -* MAINT #921: Improve hangling of the OpenML server URL in the config file. -* MAINT #925: Improve error handling and error message when loading datasets. -* MAINT #928: Restructures the contributing documentation. -* MAINT #936: Adding support for scikit-learn 0.23.X. -* MAINT #945: Make OpenML-Python PEP562 compliant. -* MAINT #951: Converts TaskType class to a TaskType enum. - -0.10.2 -~~~~~~ -* ADD #857: Adds task type ID to list_runs -* DOC #862: Added license BSD 3-Clause to each of the source files. - -0.10.1 -~~~~~~ -* ADD #175: Automatically adds the docstring of scikit-learn objects to flow and its parameters. -* ADD #737: New evaluation listing call that includes the hyperparameter settings. -* ADD #744: It is now possible to only issue a warning and not raise an exception if the package - versions for a flow are not met when deserializing it. -* ADD #783: The URL to download the predictions for a run is now stored in the run object. -* ADD #790: Adds the uploader name and id as new filtering options for ``list_evaluations``. -* ADD #792: New convenience function ``openml.flow.get_flow_id``. -* ADD #861: Debug-level log information now being written to a file in the cache directory (at most 2 MB). -* DOC #778: Introduces instructions on how to publish an extension to support other libraries - than scikit-learn. -* DOC #785: The examples section is completely restructured into simple simple examples, advanced - examples and examples showcasing the use of OpenML-Python to reproduce papers which were done - with OpenML-Python. -* DOC #788: New example on manually iterating through the split of a task. -* DOC #789: Improve the usage of dataframes in the examples. -* DOC #791: New example for the paper *Efficient and Robust Automated Machine Learning* by Feurer - et al. (2015). -* DOC #803: New example for the paper *Don’t Rule Out Simple Models Prematurely: - A Large Scale Benchmark Comparing Linear and Non-linear Classifiers in OpenML* by Benjamin - Strang et al. (2018). -* DOC #808: New example demonstrating basic use cases of a dataset. -* DOC #810: New example demonstrating the use of benchmarking studies and suites. -* DOC #832: New example for the paper *Scalable Hyperparameter Transfer Learning* by - Valerio Perrone et al. (2019) -* DOC #834: New example showing how to plot the loss surface for a support vector machine. -* FIX #305: Do not require the external version in the flow XML when loading an object. -* FIX #734: Better handling of *"old"* flows. -* FIX #736: Attach a StreamHandler to the openml logger instead of the root logger. -* FIX #758: Fixes an error which made the client API crash when loading a sparse data with - categorical variables. -* FIX #779: Do not fail on corrupt pickle -* FIX #782: Assign the study id to the correct class attribute. -* FIX #819: Automatically convert column names to type string when uploading a dataset. -* FIX #820: Make ``__repr__`` work for datasets which do not have an id. -* MAINT #796: Rename an argument to make the function ``list_evaluations`` more consistent. -* MAINT #811: Print the full error message given by the server. -* MAINT #828: Create base class for OpenML entity classes. -* MAINT #829: Reduce the number of data conversion warnings. -* MAINT #831: Warn if there's an empty flow description when publishing a flow. -* MAINT #837: Also print the flow XML if a flow fails to validate. -* FIX #838: Fix list_evaluations_setups to work when evaluations are not a 100 multiple. -* FIX #847: Fixes an issue where the client API would crash when trying to download a dataset - when there are no qualities available on the server. -* MAINT #849: Move logic of most different ``publish`` functions into the base class. -* MAINt #850: Remove outdated test code. - -0.10.0 -~~~~~~ - -* ADD #737: Add list_evaluations_setups to return hyperparameters along with list of evaluations. -* FIX #261: Test server is cleared of all files uploaded during unit testing. -* FIX #447: All files created by unit tests no longer persist in local. -* FIX #608: Fixing dataset_id referenced before assignment error in get_run function. -* FIX #447: All files created by unit tests are deleted after the completion of all unit tests. -* FIX #589: Fixing a bug that did not successfully upload the columns to ignore when creating and publishing a dataset. -* FIX #608: Fixing dataset_id referenced before assignment error in get_run function. -* DOC #639: More descriptive documention for function to convert array format. -* DOC #719: Add documentation on uploading tasks. -* ADD #687: Adds a function to retrieve the list of evaluation measures available. -* ADD #695: A function to retrieve all the data quality measures available. -* ADD #412: Add a function to trim flow names for scikit-learn flows. -* ADD #715: `list_evaluations` now has an option to sort evaluations by score (value). -* ADD #722: Automatic reinstantiation of flow in `run_model_on_task`. Clearer errors if that's not possible. -* ADD #412: The scikit-learn extension populates the short name field for flows. -* MAINT #726: Update examples to remove deprecation warnings from scikit-learn -* MAINT #752: Update OpenML-Python to be compatible with sklearn 0.21 -* ADD #790: Add user ID and name to list_evaluations - - -0.9.0 -~~~~~ -* ADD #560: OpenML-Python can now handle regression tasks as well. -* ADD #620, #628, #632, #649, #682: Full support for studies and distinguishes suites from studies. -* ADD #607: Tasks can now be created and uploaded. -* ADD #647, #673: Introduced the extension interface. This provides an easy way to create a hook for machine learning packages to perform e.g. automated runs. -* ADD #548, #646, #676: Support for Pandas DataFrame and SparseDataFrame -* ADD #662: Results of listing functions can now be returned as pandas.DataFrame. -* ADD #59: Datasets can now also be retrieved by name. -* ADD #672: Add timing measurements for runs, when possible. -* ADD #661: Upload time and error messages now displayed with `list_runs`. -* ADD #644: Datasets can now be downloaded 'lazily', retrieving only metadata at first, and the full dataset only when necessary. -* ADD #659: Lazy loading of task splits. -* ADD #516: `run_flow_on_task` flow uploading is now optional. -* ADD #680: Adds `openml.config.start_using_configuration_for_example` (and resp. stop) to easily connect to the test server. -* ADD #75, #653: Adds a pretty print for objects of the top-level classes. -* FIX #642: `check_datasets_active` now correctly also returns active status of deactivated datasets. -* FIX #304, #636: Allow serialization of numpy datatypes and list of lists of more types (e.g. bools, ints) for flows. -* FIX #651: Fixed a bug that would prevent openml-python from finding the user's config file. -* FIX #693: OpenML-Python uses liac-arff instead of scipy.io for loading task splits now. -* DOC #678: Better color scheme for code examples in documentation. -* DOC #681: Small improvements and removing list of missing functions. -* DOC #684: Add notice to examples that connect to the test server. -* DOC #688: Add new example on retrieving evaluations. -* DOC #691: Update contributing guidelines to use Github draft feature instead of tags in title. -* DOC #692: All functions are documented now. -* MAINT #184: Dropping Python2 support. -* MAINT #596: Fewer dependencies for regular pip install. -* MAINT #652: Numpy and Scipy are no longer required before installation. -* MAINT #655: Lazy loading is now preferred in unit tests. -* MAINT #667: Different tag functions now share code. -* MAINT #666: More descriptive error message for `TypeError` in `list_runs`. -* MAINT #668: Fix some type hints. -* MAINT #677: `dataset.get_data` now has consistent behavior in its return type. -* MAINT #686: Adds ignore directives for several `mypy` folders. -* MAINT #629, #630: Code now adheres to single PEP8 standard. - -0.8.0 -~~~~~ - -* ADD #440: Improved dataset upload. -* ADD #545, #583: Allow uploading a dataset from a pandas DataFrame. -* ADD #528: New functions to update the status of a dataset. -* ADD #523: Support for scikit-learn 0.20's new ColumnTransformer. -* ADD #459: Enhanced support to store runs on disk prior to uploading them to - OpenML. -* ADD #564: New helpers to access the structure of a flow (and find its - subflows). -* ADD #618: The software will from now on retry to connect to the server if a - connection failed. The number of retries can be configured. -* FIX #538: Support loading clustering tasks. -* FIX #464: Fixes a bug related to listing functions (returns correct listing - size). -* FIX #580: Listing function now works properly when there are less results - than requested. -* FIX #571: Fixes an issue where tasks could not be downloaded in parallel. -* FIX #536: Flows can now be printed when the flow name is None. -* FIX #504: Better support for hierarchical hyperparameters when uploading - scikit-learn's grid and random search. -* FIX #569: Less strict checking of flow dependencies when loading flows. -* FIX #431: Pickle of task splits are no longer cached. -* DOC #540: More examples for dataset uploading. -* DOC #554: Remove the doubled progress entry from the docs. -* MAINT #613: Utilize the latest updates in OpenML evaluation listings. -* MAINT #482: Cleaner interface for handling search traces. -* MAINT #557: Continuous integration works for scikit-learn 0.18-0.20. -* MAINT #542: Continuous integration now runs python3.7 as well. -* MAINT #535: Continuous integration now enforces PEP8 compliance for new code. -* MAINT #527: Replace deprecated nose by pytest. -* MAINT #510: Documentation is now built by travis-ci instead of circle-ci. -* MAINT: Completely re-designed documentation built on sphinx gallery. -* MAINT #462: Appveyor CI support. -* MAINT #477: Improve error handling for issue - `#479 `_: - the OpenML connector fails earlier and with a better error message when - failing to create a flow from the OpenML description. -* MAINT #561: Improve documentation on running specific unit tests. - -0.4.-0.7 -~~~~~~~~ - -There is no changelog for these versions. - -0.3.0 -~~~~~ - -* Add this changelog -* 2nd example notebook PyOpenML.ipynb -* Pagination support for list datasets and list tasks - -Prior -~~~~~ - -There is no changelog for prior versions. diff --git a/doc/test_server_usage_warning.txt b/doc/test_server_usage_warning.txt deleted file mode 100644 index 2b7eb696b..000000000 --- a/doc/test_server_usage_warning.txt +++ /dev/null @@ -1,3 +0,0 @@ -This example uploads data. For that reason, this example connects to the test server at test.openml.org. -This prevents the main server from crowding with example datasets, tasks, runs, and so on. -The use of this test server can affect behaviour and performance of the OpenML-Python API. \ No newline at end of file diff --git a/doc/usage.rst b/doc/usage.rst deleted file mode 100644 index f6476407e..000000000 --- a/doc/usage.rst +++ /dev/null @@ -1,182 +0,0 @@ -:orphan: - -.. _usage: - -.. role:: bash(code) - :language: bash - -.. role:: python(code) - :language: python - -********** -User Guide -********** - -This document will guide you through the most important use cases, functions -and classes in the OpenML Python API. Throughout this document, we will use -`pandas `_ to format and filter tables. - -.. _installation: - -~~~~~~~~~~~~~~~~~~~~~ -Installation & Set up -~~~~~~~~~~~~~~~~~~~~~ - -The OpenML Python package is a connector to `OpenML `_. -It allows you to use and share datasets and tasks, run -machine learning algorithms on them and then share the results online. - -The following tutorial gives a short introduction on how to install and set up -the OpenML Python connector, followed up by a simple example. - -* :ref:`sphx_glr_examples_20_basic_introduction_tutorial.py` - -~~~~~~~~~~~~~ -Configuration -~~~~~~~~~~~~~ - -The configuration file resides in a directory ``.config/openml`` in the home -directory of the user and is called config (More specifically, it resides in the -`configuration directory specified by the XDGB Base Directory Specification -`_). -It consists of ``key = value`` pairs which are separated by newlines. -The following keys are defined: - -* apikey: - * required to access the server. The :ref:`sphx_glr_examples_20_basic_introduction_tutorial.py` - describes how to obtain an API key. - -* server: - * default: ``http://www.openml.org``. Alternatively, use ``test.openml.org`` for the test server. - -* cachedir: - * if not given, will default to ``~/.openml/cache`` - -* avoid_duplicate_runs: - * if set to ``True``, when ``run_flow_on_task`` or similar methods are called a lookup is performed to see if there already exists such a run on the server. If so, download those results instead. - * if not given, will default to ``True``. - -* retry_policy: - * Defines how to react when the server is unavailable or experiencing high load. It determines both how often to attempt to reconnect and how quickly to do so. Please don't use ``human`` in an automated script that you run more than one instance of, it might increase the time to complete your jobs and that of others. - * human (default): For people running openml in interactive fashion. Try only a few times, but in quick succession. - * robot: For people using openml in an automated fashion. Keep trying to reconnect for a longer time, quickly increasing the time between retries. - -* connection_n_retries: - * number of connection retries - * default depends on retry_policy (5 for ``human``, 50 for ``robot``) - -* verbosity: - * 0: normal output - * 1: info output - * 2: debug output - -This file is easily configurable by the ``openml`` command line interface. -To see where the file is stored, and what its values are, use `openml configure none`. -Set any field with ``openml configure FIELD`` or even all fields with just ``openml configure``. - -~~~~~~ -Docker -~~~~~~ - -It is also possible to try out the latest development version of ``openml-python`` with docker: - -.. code:: bash - - docker run -it openml/openml-python - -See the `openml-python docker documentation `_ for more information. - -~~~~~~~~~~~~ -Key concepts -~~~~~~~~~~~~ - -OpenML contains several key concepts which it needs to make machine learning -research shareable. A machine learning experiment consists of one or several -**runs**, which describe the performance of an algorithm (called a **flow** in -OpenML), its hyperparameter settings (called a **setup**) on a **task**. A -**Task** is the combination of a **dataset**, a split and an evaluation -metric. In this user guide we will go through listing and exploring existing -**tasks** to actually running machine learning algorithms on them. In a further -user guide we will examine how to search through **datasets** in order to curate -a list of **tasks**. - -A further explanation is given in the -`OpenML user guide `_. - -~~~~~~~~~~~~~~~~~~ -Working with tasks -~~~~~~~~~~~~~~~~~~ - -You can think of a task as an experimentation protocol, describing how to apply -a machine learning model to a dataset in a way that is comparable with the -results of others (more on how to do that further down). Tasks are containers, -defining which dataset to use, what kind of task we're solving (regression, -classification, clustering, etc...) and which column to predict. Furthermore, -it also describes how to split the dataset into a train and test set, whether -to use several disjoint train and test splits (cross-validation) and whether -this should be repeated several times. Also, the task defines a target metric -for which a flow should be optimized. - -Below you can find our tutorial regarding tasks and if you want to know more -you can read the `OpenML guide `_: - -* :ref:`sphx_glr_examples_30_extended_tasks_tutorial.py` - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Running machine learning algorithms and uploading results -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In order to upload and share results of running a machine learning algorithm -on a task, we need to create an :class:`~openml.OpenMLRun`. A run object can -be created by running a :class:`~openml.OpenMLFlow` or a scikit-learn compatible -model on a task. We will focus on the simpler example of running a -scikit-learn model. - -Flows are descriptions of something runable which does the machine learning. -A flow contains all information to set up the necessary machine learning -library and its dependencies as well as all possible parameters. - -A run is the outcome of running a flow on a task. It contains all parameter -settings for the flow, a setup string (most likely a command line call) and all -predictions of that run. When a run is uploaded to the server, the server -automatically calculates several metrics which can be used to compare the -performance of different flows to each other. - -So far, the OpenML Python connector works only with estimator objects following -the `scikit-learn estimator API `_. -Those can be directly run on a task, and a flow will automatically be created or -downloaded from the server if it already exists. - -The next tutorial covers how to train different machine learning models, -how to run machine learning models on OpenML data and how to share the results: - -* :ref:`sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py` - -~~~~~~~~ -Datasets -~~~~~~~~ - -OpenML provides a large collection of datasets and the benchmark -"`OpenML100 `_" which consists of a curated -list of datasets. - -You can find the dataset that best fits your requirements by making use of the -available metadata. The tutorial which follows explains how to get a list of -datasets, how to filter the list to find the dataset that suits your -requirements and how to download a dataset: - -* :ref:`sphx_glr_examples_30_extended_datasets_tutorial.py` - -OpenML is about sharing machine learning results and the datasets they were -obtained on. Learn how to share your datasets in the following tutorial: - -* :ref:`sphx_glr_examples_30_extended_create_upload_tutorial.py` - -*********************** -Extending OpenML-Python -*********************** - -OpenML-Python provides an extension interface to connect machine learning libraries directly to -the API and ships a ``scikit-learn`` extension. You can find more information in the Section -:ref:`extensions`' - diff --git a/docs/contributing.md b/docs/contributing.md index c18de3ccc..39072d64e 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -14,10 +14,8 @@ In particular, a few ways to contribute to openml-python are: repositories and may have their own guidelines. For more information, see also [extensions](extensions.md). - Bug reports. If something doesn't work for you or is cumbersome, - please open a new issue to let us know about the problem. See - [this - section](https://github.com/openml/openml-python/blob/main/CONTRIBUTING.md). -- [Cite OpenML](https://www.openml.org/cite) if you use it in a + please open a new issue to let us know about the problem. +- [Cite OpenML](https://www.openml.org/terms) if you use it in a scientific publication. - Visit one of our [hackathons](https://www.openml.org/meet). - Contribute to another OpenML project, such as [the main OpenML diff --git a/docs/details.md b/docs/details.md new file mode 100644 index 000000000..bf4b0cd2b --- /dev/null +++ b/docs/details.md @@ -0,0 +1,76 @@ +# Advanced User Guide + +This document highlights some of the more advanced features of +`openml-python`. + +## Configuration + +The configuration file resides in a directory `.config/openml` in the +home directory of the user and is called config (More specifically, it +resides in the [configuration directory specified by the XDGB Base +Directory +Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html)). +It consists of `key = value` pairs which are separated by newlines. The +following keys are defined: + +- apikey: required to access the server. +- server: the server to connect to (default: `http://www.openml.org`). + For connection to the test server, set this to `test.openml.org`. +- cachedir: the root folder where the cache file directories should be created. + If not given, will default to `~/.openml/cache` +- avoid_duplicate_runs: if set to `True` (default), when certain functions + are called a lookup is performed to see if there already + exists such a run on the server. If so, download those + results instead. +- retry_policy: Defines how to react when the server is unavailable or + experiencing high load. It determines both how often to + attempt to reconnect and how quickly to do so. Please don't + use `human` in an automated script that you run more than + one instance of, it might increase the time to complete your + jobs and that of others. One of: + - human (default): For people running openml in interactive + fashion. Try only a few times, but in quick succession. + - robot: For people using openml in an automated fashion. Keep + trying to reconnect for a longer time, quickly increasing + the time between retries. + +- connection_n_retries: number of times to retry a request if they fail. +Default depends on retry_policy (5 for `human`, 50 for `robot`) +- verbosity: the level of output: + - 0: normal output + - 1: info output + - 2: debug output + +This file is easily configurable by the `openml` command line interface. +To see where the file is stored, and what its values are, use openml +configure none. + +## Docker + +It is also possible to try out the latest development version of +`openml-python` with docker: + +``` bash +docker run -it openml/openml-python +``` + +See the [openml-python docker +documentation](https://github.com/openml/openml-python/blob/main/docker/readme.md) +for more information. + +## Key concepts + +OpenML contains several key concepts which it needs to make machine +learning research shareable. A machine learning experiment consists of +one or several **runs**, which describe the performance of an algorithm +(called a **flow** in OpenML), its hyperparameter settings (called a +**setup**) on a **task**. A **Task** is the combination of a +**dataset**, a split and an evaluation metric. In this user guide we +will go through listing and exploring existing **tasks** to actually +running machine learning algorithms on them. In a further user guide we +will examine how to search through **datasets** in order to curate a +list of **tasks**. + +A further explanation is given in the [OpenML user +guide](https://docs.openml.org/concepts/). + diff --git a/docs/extensions.md b/docs/extensions.md index f2aa230f5..858447440 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -2,17 +2,14 @@ OpenML-Python provides an extension interface to connect other machine learning libraries than scikit-learn to OpenML. Please check the -`api_extensions`{.interpreted-text role="ref"} and use the scikit-learn -extension in -`openml.extensions.sklearn.SklearnExtension`{.interpreted-text -role="class"} as a starting point. +[`api_extensions`](../reference/extensions/extension_interface/) and use the scikit-learn +extension as a starting point. ## List of extensions Here is a list of currently maintained OpenML extensions: -- `openml.extensions.sklearn.SklearnExtension`{.interpreted-text - role="class"} +- [openml-sklearn](https://github.com/openml/openml-sklearn) - [openml-keras](https://github.com/openml/openml-keras) - [openml-pytorch](https://github.com/openml/openml-pytorch) - [openml-tensorflow (for tensorflow @@ -34,43 +31,33 @@ extension interface to allows others to contribute back. Building a suitable extension for therefore requires an understanding of the current OpenML-Python support. -The -`sphx_glr_examples_20_basic_simple_flows_and_runs_tutorial.py`{.interpreted-text -role="ref"} tutorial shows how scikit-learn currently works with -OpenML-Python as an extension. The *sklearn* extension packaged with the -[openml-python](https://github.com/openml/openml-python) repository can -be used as a template/benchmark to build the new extension. +[This tutorial](../examples/Basics/simple_flows_and_runs_tutorial) shows how the scikit-learn +extension works with OpenML-Python. #### API -- The extension scripts must import the [openml]{.title-ref} package - and be able to interface with any function from the OpenML-Python - `api`{.interpreted-text role="ref"}. +- The extension scripts must import the openml-python package + and be able to interface with any function from the API. - The extension has to be defined as a Python class and must inherit - from `openml.extensions.Extension`{.interpreted-text role="class"}. -- This class needs to have all the functions from [class - Extension]{.title-ref} overloaded as required. + from [`openml.extensions.Extension`](../reference/extensions/extension_interface/#openml.extensions.extension_interface.Extension). +- This class needs to have all the functions from `openml.extensions.Extension` overloaded as required. - The redefined functions should have adequate and appropriate - docstrings. The [Sklearn Extension API - :class:\`openml.extensions.sklearn.SklearnExtension.html]{.title-ref} - is a good example to follow. + docstrings. The sklearn Extension API is a good example to follow. #### Interfacing with OpenML-Python Once the new extension class has been defined, the openml-python module -to `openml.extensions.register_extension`{.interpreted-text role="meth"} +to [`openml.extensions.register_extension`](../reference/extensions/functions/#openml.extensions.functions.register_extension) must be called to allow OpenML-Python to interface the new extension. The following methods should get implemented. Although the documentation -in the [Extension]{.title-ref} interface should always be leading, here -we list some additional information and best practices. The [Sklearn -Extension API -:class:\`openml.extensions.sklearn.SklearnExtension.html]{.title-ref} is -a good example to follow. Note that most methods are relatively simple +in the extension interface should always be leading, here +we list some additional information and best practices. +Note that most methods are relatively simple and can be implemented in several lines of code. - General setup (required) - - `can_handle_flow`{.interpreted-text role="meth"}: Takes as + - `can_handle_flow`: Takes as argument an OpenML flow, and checks whether this can be handled by the current extension. The OpenML database consists of many flows, from various workbenches (e.g., scikit-learn, Weka, mlr). @@ -78,16 +65,16 @@ and can be implemented in several lines of code. Typically, the flow-dependency field is used to check whether the specific library is present, and no unknown libraries are present there. - - `can_handle_model`{.interpreted-text role="meth"}: Similar as - `can_handle_flow`{.interpreted-text role="meth"}, except that in + - `can_handle_model`: Similar as + `can_handle_flow`:, except that in this case a Python object is given. As such, in many cases, this method can be implemented by checking whether this adheres to a certain base class. - Serialization and De-serialization (required) - - `flow_to_model`{.interpreted-text role="meth"}: deserializes the + - `flow_to_model`: deserializes the OpenML Flow into a model (if the library can indeed handle the flow). This method has an important interplay with - `model_to_flow`{.interpreted-text role="meth"}. Running these + `model_to_flow`. Running these two methods in succession should result in exactly the same model (or flow). This property can be used for unit testing (e.g., build a model with hyperparameters, make predictions on a @@ -97,22 +84,20 @@ and can be implemented in several lines of code. might seem daunting, but note that here some complicated design choices were made, that allow for all sorts of interesting research questions. It is probably good practice to start easy. - - `model_to_flow`{.interpreted-text role="meth"}: The inverse of - `flow_to_model`{.interpreted-text role="meth"}. Serializes a + - `model_to_flow`: The inverse of `flow_to_model`. Serializes a model into an OpenML Flow. The flow should preserve the class, the library version, and the tunable hyperparameters. - - `get_version_information`{.interpreted-text role="meth"}: Return + - `get_version_information`: Return a tuple with the version information of the important libraries. - - `create_setup_string`{.interpreted-text role="meth"}: No longer + - `create_setup_string`: No longer used, and will be deprecated soon. - Performing runs (required) - - `is_estimator`{.interpreted-text role="meth"}: Gets as input a + - `is_estimator`: Gets as input a class, and checks whether it has the status of estimator in the library (typically, whether it has a train method and a predict method). - - `seed_model`{.interpreted-text role="meth"}: Sets a random seed - to the model. - - `_run_model_on_fold`{.interpreted-text role="meth"}: One of the + - `seed_model`: Sets a random seed to the model. + - `_run_model_on_fold`: One of the main requirements for a library to generate run objects for the OpenML server. Obtains a train split (with labels) and a test split (without labels) and the goal is to train a model on the @@ -125,39 +110,35 @@ and can be implemented in several lines of code. user-defined measures (such as runtime information, as this can not be inferred on the server). Additionally, information about a hyperparameter optimization trace can be provided. - - `obtain_parameter_values`{.interpreted-text role="meth"}: + - `obtain_parameter_values`: Obtains the hyperparameters of a given model and the current values. Please note that in the case of a hyperparameter optimization procedure (e.g., random search), you only should return the hyperparameters of this procedure (e.g., the hyperparameter grid, budget, etc) and that the chosen model will be inferred from the optimization trace. - - `check_if_model_fitted`{.interpreted-text role="meth"}: Check + - `check_if_model_fitted`: Check whether the train method of the model has been called (and as such, whether the predict method can be used). - Hyperparameter optimization (optional) - - `instantiate_model_from_hpo_class`{.interpreted-text - role="meth"}: If a given run has recorded the hyperparameter + - `instantiate_model_from_hpo_class`: If a given run has recorded the hyperparameter optimization trace, then this method can be used to reinstantiate the model with hyperparameters of a given hyperparameter optimization iteration. Has some similarities - with `flow_to_model`{.interpreted-text role="meth"} (as this + with `flow_to_model` (as this method also sets the hyperparameters of a model). Note that although this method is required, it is not necessary to implement any logic if hyperparameter optimization is not - implemented. Simply raise a [NotImplementedError]{.title-ref} + implemented. Simply raise a `NotImplementedError` then. ### Hosting the library Each extension created should be a stand-alone repository, compatible -with the [OpenML-Python -repository](https://github.com/openml/openml-python). The extension -repository should work off-the-shelf with *OpenML-Python* installed. +with the [OpenML-Python repository](https://github.com/openml/openml-python). +The extension repository should work off-the-shelf with *OpenML-Python* installed. -Create a [public Github -repo](https://docs.github.com/en/github/getting-started-with-github/create-a-repo) -with the following directory structure: +Create a public Github repo with the following directory structure: | [repo name] | |-- [extension name] @@ -168,7 +149,7 @@ with the following directory structure: ### Recommended - Test cases to keep the extension up to date with the - [openml-python]{.title-ref} upstream changes. + Openml-Python upstream changes. - Documentation of the extension API, especially if any new functionality added to OpenML-Python\'s extension design. - Examples to show how the new extension interfaces and works with diff --git a/docs/index.md b/docs/index.md index cda5bcb4b..1058c3956 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,65 +1,79 @@ # OpenML -**Collaborative Machine Learning in Python** +**The Python API for a World of Data and More** Welcome to the documentation of the OpenML Python API, a connector to the collaborative machine learning platform -[OpenML.org](https://www.openml.org). The OpenML Python package allows -to use datasets and tasks from OpenML together with scikit-learn and -share the results online. +[OpenML.org](https://www.openml.org). +OpenML-Python can download or upload data from OpenML, such as datasets +and machine learning experiment results. -## Example +If you are new to OpenML, we recommend checking out the [OpenML documentation](https://docs.openml.org/) +to get familiar with the concepts and features of OpenML. In particular, we recommend +reading more about the [OpenML concepts](https://docs.openml.org/concepts/). + +## :joystick: Minimal Examples + +Use the following code to get the [credit-g](https://www.openml.org/search?type=data&sort=runs&status=active&id=31) [dataset](https://docs.openml.org/concepts/data/): ```python import openml -from sklearn import impute, tree, pipeline - -# Define a scikit-learn classifier or pipeline -clf = pipeline.Pipeline( - steps=[ - ('imputer', impute.SimpleImputer()), - ('estimator', tree.DecisionTreeClassifier()) - ] -) -# Download the OpenML task for the pendigits dataset with 10-fold -# cross-validation. -task = openml.tasks.get_task(32) -# Run the scikit-learn model on the task. -run = openml.runs.run_model_on_task(clf, task) -# Publish the experiment on OpenML (optional, requires an API key. -# You can get your own API key by signing up to OpenML.org) -run.publish() -print(f'View the run online: {run.openml_url}') + +dataset = openml.datasets.get_dataset("credit-g") # or by ID get_dataset(31) +X, y, categorical_indicator, attribute_names = dataset.get_data(target="class") ``` -Find more examples in the sidebar on the left. +Get a [task](https://docs.openml.org/concepts/tasks/) for [supervised classification on credit-g](https://www.openml.org/search?type=task&id=31&source_data.data_id=31): -## How to get OpenML for python +```python +import openml -You can install the OpenML package via `pip` (we recommend using a virtual environment): +task = openml.tasks.get_task(31) +dataset = task.get_dataset() +X, y, categorical_indicator, attribute_names = dataset.get_data(target=task.target_name) +# get splits for the first fold of 10-fold cross-validation +train_indices, test_indices = task.get_train_test_split_indices(fold=0) +``` + +Use an [OpenML benchmarking suite](https://docs.openml.org/concepts/benchmarking/) to get a curated list of machine-learning tasks: +```python +import openml + +suite = openml.study.get_suite("amlb-classification-all") # Get a curated list of tasks for classification +for task_id in suite.tasks: + task = openml.tasks.get_task(task_id) +``` +Find more examples in the navbar at the top. + +## :magic_wand: Installation + +OpenML-Python is available on Linux, MacOS, and Windows. + +You can install OpenML-Python with: ```bash -python -m pip install openml +pip install openml ``` For more advanced installation information, please see the -["Introduction"](../examples/20_basic/introduction_tutorial.py) example. +["Introduction"](../examples/Basics/introduction_tutorial) example. ## Further information - [OpenML documentation](https://docs.openml.org/) - [OpenML client APIs](https://docs.openml.org/APIs/) -- [OpenML developer guide](https://docs.openml.org/Contributing/) +- [OpenML developer guide](https://docs.openml.org/contributing/) - [Contact information](https://www.openml.org/contact) - [Citation request](https://www.openml.org/cite) - [OpenML blog](https://medium.com/open-machine-learning) - [OpenML twitter account](https://twitter.com/open_ml) + ## Contributing -Contribution to the OpenML package is highly appreciated. Please see the -["Contributing"][contributing] page for more information. +Contributing to the OpenML package is highly appreciated. Please see the +["Contributing"](contributing.md) page for more information. ## Citing OpenML-Python diff --git a/docs/progress.md b/docs/progress.md deleted file mode 100644 index c2923576b..000000000 --- a/docs/progress.md +++ /dev/null @@ -1,489 +0,0 @@ -# Changelog {#progress} - -## next - -> - MAINT #1340: Add Numpy 2.0 support. Update tests to work with -> scikit-learn \<= 1.5. -> - ADD #1342: Add HTTP header to requests to indicate they are from -> openml-python. - -## 0.14.2 - -> - MAINT #1280: Use the server-provided `parquet_url` instead of -> `minio_url` to determine the location of the parquet file. -> - ADD #716: add documentation for remaining attributes of classes -> and functions. -> - ADD #1261: more annotations for type hints. -> - MAINT #1294: update tests to new tag specification. -> - FIX #1314: Update fetching a bucket from MinIO. -> - FIX #1315: Make class label retrieval more lenient. -> - ADD #1316: add feature descriptions ontologies support. -> - MAINT #1310/#1307: switch to ruff and resolve all mypy errors. - -## 0.14.1 - -> - FIX: Fallback on downloading ARFF when failing to download parquet -> from MinIO due to a ServerError. - -## 0.14.0 - -**IMPORTANT:** This release paves the way towards a breaking update of -OpenML-Python. From version 0.15, functions that had the option to -return a pandas DataFrame will return a pandas DataFrame by default. -This version (0.14) emits a warning if you still use the old access -functionality. More concretely: - -- In 0.15 we will drop the ability to return dictionaries in listing - calls and only provide pandas DataFrames. To disable warnings in - 0.14 you have to request a pandas DataFrame (using - `output_format="dataframe"`). -- In 0.15 we will drop the ability to return datasets as numpy arrays - and only provide pandas DataFrames. To disable warnings in 0.14 you - have to request a pandas DataFrame (using - `dataset_format="dataframe"`). - -Furthermore, from version 0.15, OpenML-Python will no longer download -datasets and dataset metadata by default. This version (0.14) emits a -warning if you don\'t explicitly specifiy the desired behavior. - -Please see the pull requests #1258 and #1260 for further information. - -- ADD #1081: New flag that allows disabling downloading dataset - features. -- ADD #1132: New flag that forces a redownload of cached data. -- FIX #1244: Fixes a rare bug where task listing could fail when the - server returned invalid data. -- DOC #1229: Fixes a comment string for the main example. -- DOC #1241: Fixes a comment in an example. -- MAINT #1124: Improve naming of helper functions that govern the - cache directories. -- MAINT #1223, #1250: Update tools used in pre-commit to the latest - versions (`black==23.30`, `mypy==1.3.0`, `flake8==6.0.0`). -- MAINT #1253: Update the citation request to the JMLR paper. -- MAINT #1246: Add a warning that warns the user that checking for - duplicate runs on the server cannot be done without an API key. - -## 0.13.1 - -- ADD #1081 #1132: Add additional options for (not) downloading - datasets `openml.datasets.get_dataset` and cache management. -- ADD #1028: Add functions to delete runs, flows, datasets, and tasks - (e.g., `openml.datasets.delete_dataset`). -- ADD #1144: Add locally computed results to the `OpenMLRun` object\'s - representation if the run was created locally and not downloaded - from the server. -- ADD #1180: Improve the error message when the checksum of a - downloaded dataset does not match the checksum provided by the API. -- ADD #1201: Make `OpenMLTraceIteration` a dataclass. -- DOC #1069: Add argument documentation for the `OpenMLRun` class. -- DOC #1241 #1229 #1231: Minor documentation fixes and resolve - documentation examples not working. -- FIX #1197 #559 #1131: Fix the order of ground truth and predictions - in the `OpenMLRun` object and in `format_prediction`. -- FIX #1198: Support numpy 1.24 and higher. -- FIX #1216: Allow unknown task types on the server. This is only - relevant when new task types are added to the test server. -- FIX #1223: Fix mypy errors for implicit optional typing. -- MAINT #1155: Add dependabot github action to automatically update - other github actions. -- MAINT #1199: Obtain pre-commit\'s flake8 from github.com instead of - gitlab.com. -- MAINT #1215: Support latest numpy version. -- MAINT #1218: Test Python3.6 on Ubuntu 20.04 instead of the latest - Ubuntu (which is 22.04). -- MAINT #1221 #1212 #1206 #1211: Update github actions to the latest - versions. - -## 0.13.0 - -> - FIX #1030: `pre-commit` hooks now no longer should issue a -> warning. -> - FIX #1058, #1100: Avoid `NoneType` error when printing task -> without `class_labels` attribute. -> - FIX #1110: Make arguments to `create_study` and `create_suite` -> that are defined as optional by the OpenML XSD actually optional. -> - FIX #1147: `openml.flow.flow_exists` no longer requires an API -> key. -> - FIX #1184: Automatically resolve proxies when downloading from -> minio. Turn this off by setting environment variable -> `no_proxy="*"`. -> - MAINT #1088: Do CI for Windows on Github Actions instead of -> Appveyor. -> - MAINT #1104: Fix outdated docstring for `list_task`. -> - MAINT #1146: Update the pre-commit dependencies. -> - ADD #1103: Add a `predictions` property to OpenMLRun for easy -> accessibility of prediction data. -> - ADD #1188: EXPERIMENTAL. Allow downloading all files from a minio -> bucket with `download_all_files=True` for `get_dataset`. - -## 0.12.2 - -- ADD #1065: Add a `retry_policy` configuration option that determines - the frequency and number of times to attempt to retry server - requests. -- ADD #1075: A docker image is now automatically built on a push to - develop. It can be used to build docs or run tests in an isolated - environment. -- ADD: You can now avoid downloading \'qualities\' meta-data when - downloading a task with the `download_qualities` parameter of - `openml.tasks.get_task[s]` functions. -- DOC: Fixes a few broken links in the documentation. -- DOC #1061: Improve examples to always show a warning when they - switch to the test server. -- DOC #1067: Improve documentation on the scikit-learn extension - interface. -- DOC #1068: Create dedicated extensions page. -- FIX #1075: Correctly convert [y]{.title-ref} to a pandas series when - downloading sparse data. -- MAINT: Rename [master]{.title-ref} brach to [ main]{.title-ref} - branch. -- MAINT/DOC: Automatically check for broken external links when - building the documentation. -- MAINT/DOC: Fail documentation building on warnings. This will make - the documentation building fail if a reference cannot be found (i.e. - an internal link is broken). - -## 0.12.1 - -- ADD #895/#1038: Measure runtimes of scikit-learn runs also for - models which are parallelized via the joblib. -- DOC #1050: Refer to the webpage instead of the XML file in the main - example. -- DOC #1051: Document existing extensions to OpenML-Python besides the - shipped scikit-learn extension. -- FIX #1035: Render class attributes and methods again. -- ADD #1049: Add a command line tool for configuration openml-python. -- FIX #1042: Fixes a rare concurrency issue with OpenML-Python and - joblib which caused the joblib worker pool to fail. -- FIX #1053: Fixes a bug which could prevent importing the package in - a docker container. - -## 0.12.0 - -- ADD #964: Validate `ignore_attribute`, `default_target_attribute`, - `row_id_attribute` are set to attributes that exist on the dataset - when calling `create_dataset`. -- ADD #979: Dataset features and qualities are now also cached in - pickle format. -- ADD #982: Add helper functions for column transformers. -- ADD #989: `run_model_on_task` will now warn the user the the model - passed has already been fitted. -- ADD #1009 : Give possibility to not download the dataset qualities. - The cached version is used even so download attribute is false. -- ADD #1016: Add scikit-learn 0.24 support. -- ADD #1020: Add option to parallelize evaluation of tasks with - joblib. -- ADD #1022: Allow minimum version of dependencies to be listed for a - flow, use more accurate minimum versions for scikit-learn - dependencies. -- ADD #1023: Add admin-only calls for adding topics to datasets. -- ADD #1029: Add support for fetching dataset from a minio server in - parquet format. -- ADD #1031: Generally improve runtime measurements, add them for some - previously unsupported flows (e.g. BaseSearchCV derived flows). -- DOC #973 : Change the task used in the welcome page example so it no - longer fails using numerical dataset. -- MAINT #671: Improved the performance of `check_datasets_active` by - only querying the given list of datasets in contrast to querying all - datasets. Modified the corresponding unit test. -- MAINT #891: Changed the way that numerical features are stored. - Numerical features that range from 0 to 255 are now stored as uint8, - which reduces the storage space required as well as storing and - loading times. -- MAINT #975, #988: Add CI through Github Actions. -- MAINT #977: Allow `short` and `long` scenarios for unit tests. - Reduce the workload for some unit tests. -- MAINT #985, #1000: Improve unit test stability and output - readability, and adds load balancing. -- MAINT #1018: Refactor data loading and storage. Data is now - compressed on the first call to [get_data]{.title-ref}. -- MAINT #1024: Remove flaky decorator for study unit test. -- FIX #883 #884 #906 #972: Various improvements to the caching system. -- FIX #980: Speed up `check_datasets_active`. -- FIX #984: Add a retry mechanism when the server encounters a - database issue. -- FIX #1004: Fixed an issue that prevented installation on some - systems (e.g. Ubuntu). -- FIX #1013: Fixes a bug where `OpenMLRun.setup_string` was not - uploaded to the server, prepares for `run_details` being sent from - the server. -- FIX #1021: Fixes an issue that could occur when running unit tests - and openml-python was not in PATH. -- FIX #1037: Fixes a bug where a dataset could not be loaded if a - categorical value had listed nan-like as a possible category. - -## 0.11.0 - -- ADD #753: Allows uploading custom flows to OpenML via OpenML-Python. -- ADD #777: Allows running a flow on pandas dataframes (in addition to - numpy arrays). -- ADD #888: Allow passing a [task_id]{.title-ref} to - [run_model_on_task]{.title-ref}. -- ADD #894: Support caching of datasets using feather format as an - option. -- ADD #929: Add `edit_dataset` and `fork_dataset` to allow editing and - forking of uploaded datasets. -- ADD #866, #943: Add support for scikit-learn\'s - [passthrough]{.title-ref} and [drop]{.title-ref} when uploading - flows to OpenML. -- ADD #879: Add support for scikit-learn\'s MLP hyperparameter - [layer_sizes]{.title-ref}. -- ADD #894: Support caching of datasets using feather format as an - option. -- ADD #945: PEP 561 compliance for distributing Type information. -- DOC #660: Remove nonexistent argument from docstring. -- DOC #901: The API reference now documents the config file and its - options. -- DOC #912: API reference now shows [create_task]{.title-ref}. -- DOC #954: Remove TODO text from documentation. -- DOC #960: document how to upload multiple ignore attributes. -- FIX #873: Fixes an issue which resulted in incorrect URLs when - printing OpenML objects after switching the server. -- FIX #885: Logger no longer registered by default. Added utility - functions to easily register logging to console and file. -- FIX #890: Correct the scaling of data in the SVM example. -- MAINT #371: `list_evaluations` default `size` changed from `None` to - `10_000`. -- MAINT #767: Source distribution installation is now unit-tested. -- MAINT #781: Add pre-commit and automated code formatting with black. -- MAINT #804: Rename arguments of list_evaluations to indicate they - expect lists of ids. -- MAINT #836: OpenML supports only pandas version 1.0.0 or above. -- MAINT #865: OpenML no longer bundles test files in the source - distribution. -- MAINT #881: Improve the error message for too-long URIs. -- MAINT #897: Dropping support for Python 3.5. -- MAINT #916: Adding support for Python 3.8. -- MAINT #920: Improve error messages for dataset upload. -- MAINT #921: Improve hangling of the OpenML server URL in the config - file. -- MAINT #925: Improve error handling and error message when loading - datasets. -- MAINT #928: Restructures the contributing documentation. -- MAINT #936: Adding support for scikit-learn 0.23.X. -- MAINT #945: Make OpenML-Python PEP562 compliant. -- MAINT #951: Converts TaskType class to a TaskType enum. - -## 0.10.2 - -- ADD #857: Adds task type ID to list_runs -- DOC #862: Added license BSD 3-Clause to each of the source files. - -## 0.10.1 - -- ADD #175: Automatically adds the docstring of scikit-learn objects - to flow and its parameters. -- ADD #737: New evaluation listing call that includes the - hyperparameter settings. -- ADD #744: It is now possible to only issue a warning and not raise - an exception if the package versions for a flow are not met when - deserializing it. -- ADD #783: The URL to download the predictions for a run is now - stored in the run object. -- ADD #790: Adds the uploader name and id as new filtering options for - `list_evaluations`. -- ADD #792: New convenience function `openml.flow.get_flow_id`. -- ADD #861: Debug-level log information now being written to a file in - the cache directory (at most 2 MB). -- DOC #778: Introduces instructions on how to publish an extension to - support other libraries than scikit-learn. -- DOC #785: The examples section is completely restructured into - simple simple examples, advanced examples and examples showcasing - the use of OpenML-Python to reproduce papers which were done with - OpenML-Python. -- DOC #788: New example on manually iterating through the split of a - task. -- DOC #789: Improve the usage of dataframes in the examples. -- DOC #791: New example for the paper *Efficient and Robust Automated - Machine Learning* by Feurer et al. (2015). -- DOC #803: New example for the paper *Don't Rule Out Simple Models - Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear - Classifiers in OpenML* by Benjamin Strang et al. (2018). -- DOC #808: New example demonstrating basic use cases of a dataset. -- DOC #810: New example demonstrating the use of benchmarking studies - and suites. -- DOC #832: New example for the paper *Scalable Hyperparameter - Transfer Learning* by Valerio Perrone et al. (2019) -- DOC #834: New example showing how to plot the loss surface for a - support vector machine. -- FIX #305: Do not require the external version in the flow XML when - loading an object. -- FIX #734: Better handling of *\"old\"* flows. -- FIX #736: Attach a StreamHandler to the openml logger instead of the - root logger. -- FIX #758: Fixes an error which made the client API crash when - loading a sparse data with categorical variables. -- FIX #779: Do not fail on corrupt pickle -- FIX #782: Assign the study id to the correct class attribute. -- FIX #819: Automatically convert column names to type string when - uploading a dataset. -- FIX #820: Make `__repr__` work for datasets which do not have an id. -- MAINT #796: Rename an argument to make the function - `list_evaluations` more consistent. -- MAINT #811: Print the full error message given by the server. -- MAINT #828: Create base class for OpenML entity classes. -- MAINT #829: Reduce the number of data conversion warnings. -- MAINT #831: Warn if there\'s an empty flow description when - publishing a flow. -- MAINT #837: Also print the flow XML if a flow fails to validate. -- FIX #838: Fix list_evaluations_setups to work when evaluations are - not a 100 multiple. -- FIX #847: Fixes an issue where the client API would crash when - trying to download a dataset when there are no qualities available - on the server. -- MAINT #849: Move logic of most different `publish` functions into - the base class. -- MAINt #850: Remove outdated test code. - -## 0.10.0 - -- ADD #737: Add list_evaluations_setups to return hyperparameters - along with list of evaluations. -- FIX #261: Test server is cleared of all files uploaded during unit - testing. -- FIX #447: All files created by unit tests no longer persist in - local. -- FIX #608: Fixing dataset_id referenced before assignment error in - get_run function. -- FIX #447: All files created by unit tests are deleted after the - completion of all unit tests. -- FIX #589: Fixing a bug that did not successfully upload the columns - to ignore when creating and publishing a dataset. -- FIX #608: Fixing dataset_id referenced before assignment error in - get_run function. -- DOC #639: More descriptive documention for function to convert array - format. -- DOC #719: Add documentation on uploading tasks. -- ADD #687: Adds a function to retrieve the list of evaluation - measures available. -- ADD #695: A function to retrieve all the data quality measures - available. -- ADD #412: Add a function to trim flow names for scikit-learn flows. -- ADD #715: [list_evaluations]{.title-ref} now has an option to sort - evaluations by score (value). -- ADD #722: Automatic reinstantiation of flow in - [run_model_on_task]{.title-ref}. Clearer errors if that\'s not - possible. -- ADD #412: The scikit-learn extension populates the short name field - for flows. -- MAINT #726: Update examples to remove deprecation warnings from - scikit-learn -- MAINT #752: Update OpenML-Python to be compatible with sklearn 0.21 -- ADD #790: Add user ID and name to list_evaluations - -## 0.9.0 - -- ADD #560: OpenML-Python can now handle regression tasks as well. -- ADD #620, #628, #632, #649, #682: Full support for studies and - distinguishes suites from studies. -- ADD #607: Tasks can now be created and uploaded. -- ADD #647, #673: Introduced the extension interface. This provides an - easy way to create a hook for machine learning packages to perform - e.g. automated runs. -- ADD #548, #646, #676: Support for Pandas DataFrame and - SparseDataFrame -- ADD #662: Results of listing functions can now be returned as - pandas.DataFrame. -- ADD #59: Datasets can now also be retrieved by name. -- ADD #672: Add timing measurements for runs, when possible. -- ADD #661: Upload time and error messages now displayed with - [list_runs]{.title-ref}. -- ADD #644: Datasets can now be downloaded \'lazily\', retrieving only - metadata at first, and the full dataset only when necessary. -- ADD #659: Lazy loading of task splits. -- ADD #516: [run_flow_on_task]{.title-ref} flow uploading is now - optional. -- ADD #680: Adds - [openml.config.start_using_configuration_for_example]{.title-ref} - (and resp. stop) to easily connect to the test server. -- ADD #75, #653: Adds a pretty print for objects of the top-level - classes. -- FIX #642: [check_datasets_active]{.title-ref} now correctly also - returns active status of deactivated datasets. -- FIX #304, #636: Allow serialization of numpy datatypes and list of - lists of more types (e.g. bools, ints) for flows. -- FIX #651: Fixed a bug that would prevent openml-python from finding - the user\'s config file. -- FIX #693: OpenML-Python uses liac-arff instead of scipy.io for - loading task splits now. -- DOC #678: Better color scheme for code examples in documentation. -- DOC #681: Small improvements and removing list of missing functions. -- DOC #684: Add notice to examples that connect to the test server. -- DOC #688: Add new example on retrieving evaluations. -- DOC #691: Update contributing guidelines to use Github draft feature - instead of tags in title. -- DOC #692: All functions are documented now. -- MAINT #184: Dropping Python2 support. -- MAINT #596: Fewer dependencies for regular pip install. -- MAINT #652: Numpy and Scipy are no longer required before - installation. -- MAINT #655: Lazy loading is now preferred in unit tests. -- MAINT #667: Different tag functions now share code. -- MAINT #666: More descriptive error message for - [TypeError]{.title-ref} in [list_runs]{.title-ref}. -- MAINT #668: Fix some type hints. -- MAINT #677: [dataset.get_data]{.title-ref} now has consistent - behavior in its return type. -- MAINT #686: Adds ignore directives for several [mypy]{.title-ref} - folders. -- MAINT #629, #630: Code now adheres to single PEP8 standard. - -## 0.8.0 - -- ADD #440: Improved dataset upload. -- ADD #545, #583: Allow uploading a dataset from a pandas DataFrame. -- ADD #528: New functions to update the status of a dataset. -- ADD #523: Support for scikit-learn 0.20\'s new ColumnTransformer. -- ADD #459: Enhanced support to store runs on disk prior to uploading - them to OpenML. -- ADD #564: New helpers to access the structure of a flow (and find - its subflows). -- ADD #618: The software will from now on retry to connect to the - server if a connection failed. The number of retries can be - configured. -- FIX #538: Support loading clustering tasks. -- FIX #464: Fixes a bug related to listing functions (returns correct - listing size). -- FIX #580: Listing function now works properly when there are less - results than requested. -- FIX #571: Fixes an issue where tasks could not be downloaded in - parallel. -- FIX #536: Flows can now be printed when the flow name is None. -- FIX #504: Better support for hierarchical hyperparameters when - uploading scikit-learn\'s grid and random search. -- FIX #569: Less strict checking of flow dependencies when loading - flows. -- FIX #431: Pickle of task splits are no longer cached. -- DOC #540: More examples for dataset uploading. -- DOC #554: Remove the doubled progress entry from the docs. -- MAINT #613: Utilize the latest updates in OpenML evaluation - listings. -- MAINT #482: Cleaner interface for handling search traces. -- MAINT #557: Continuous integration works for scikit-learn 0.18-0.20. -- MAINT #542: Continuous integration now runs python3.7 as well. -- MAINT #535: Continuous integration now enforces PEP8 compliance for - new code. -- MAINT #527: Replace deprecated nose by pytest. -- MAINT #510: Documentation is now built by travis-ci instead of - circle-ci. -- MAINT: Completely re-designed documentation built on sphinx gallery. -- MAINT #462: Appveyor CI support. -- MAINT #477: Improve error handling for issue - [#479](https://github.com/openml/openml-python/pull/479): the OpenML - connector fails earlier and with a better error message when failing - to create a flow from the OpenML description. -- MAINT #561: Improve documentation on running specific unit tests. - -## 0.4.-0.7 - -There is no changelog for these versions. - -## 0.3.0 - -- Add this changelog -- 2nd example notebook PyOpenML.ipynb -- Pagination support for list datasets and list tasks - -## Prior - -There is no changelog for prior versions. diff --git a/docs/usage.md b/docs/usage.md deleted file mode 100644 index 7c733fedc..000000000 --- a/docs/usage.md +++ /dev/null @@ -1,155 +0,0 @@ -# User Guide - -This document will guide you through the most important use cases, -functions and classes in the OpenML Python API. Throughout this -document, we will use [pandas](https://pandas.pydata.org/) to format and -filter tables. - -## Installation - -The OpenML Python package is a connector to -[OpenML](https://www.openml.org/). It allows you to use and share -datasets and tasks, run machine learning algorithms on them and then -share the results online. - -The ["intruduction tutorial and setup"][intro] tutorial gives a short introduction on how to install and -set up the OpenML Python connector, followed up by a simple example. - -## Configuration - -The configuration file resides in a directory `.config/openml` in the -home directory of the user and is called config (More specifically, it -resides in the [configuration directory specified by the XDGB Base -Directory -Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html)). -It consists of `key = value` pairs which are separated by newlines. The -following keys are defined: - -- apikey: required to access the server. The [introduction tutorial][intro] describes how to obtain an API key. -- server: the server to connect to (default: `http://www.openml.org`). - For connection to the test server, set this to `test.openml.org`. -- cachedir: the root folder where the cache file directories should be created. - If not given, will default to `~/.openml/cache` -- avoid_duplicate_runs: if set to `True` (default), when `run_flow_on_task` or similar methods - are called a lookup is performed to see if there already - exists such a run on the server. If so, download those - results instead. -- retry_policy: Defines how to react when the server is unavailable or - experiencing high load. It determines both how often to - attempt to reconnect and how quickly to do so. Please don't - use `human` in an automated script that you run more than - one instance of, it might increase the time to complete your - jobs and that of others. One of: - - human (default): For people running openml in interactive - fashion. Try only a few times, but in quick succession. - - robot: For people using openml in an automated fashion. Keep - trying to reconnect for a longer time, quickly increasing - the time between retries. - -- connection_n_retries: number of times to retry a request if they fail. -Default depends on retry_policy (5 for `human`, 50 for `robot`) -- verbosity: the level of output: - - 0: normal output - - 1: info output - - 2: debug output - -This file is easily configurable by the `openml` command line interface. -To see where the file is stored, and what its values are, use openml -configure none. - -## Docker - -It is also possible to try out the latest development version of -`openml-python` with docker: - -``` bash -docker run -it openml/openml-python -``` - -See the [openml-python docker -documentation](https://github.com/openml/openml-python/blob/main/docker/readme.md) -for more information. - -## Key concepts - -OpenML contains several key concepts which it needs to make machine -learning research shareable. A machine learning experiment consists of -one or several **runs**, which describe the performance of an algorithm -(called a **flow** in OpenML), its hyperparameter settings (called a -**setup**) on a **task**. A **Task** is the combination of a -**dataset**, a split and an evaluation metric. In this user guide we -will go through listing and exploring existing **tasks** to actually -running machine learning algorithms on them. In a further user guide we -will examine how to search through **datasets** in order to curate a -list of **tasks**. - -A further explanation is given in the [OpenML user -guide](https://openml.github.io/OpenML/#concepts). - -## Working with tasks - -You can think of a task as an experimentation protocol, describing how -to apply a machine learning model to a dataset in a way that is -comparable with the results of others (more on how to do that further -down). Tasks are containers, defining which dataset to use, what kind of -task we\'re solving (regression, classification, clustering, etc\...) -and which column to predict. Furthermore, it also describes how to split -the dataset into a train and test set, whether to use several disjoint -train and test splits (cross-validation) and whether this should be -repeated several times. Also, the task defines a target metric for which -a flow should be optimized. - -If you want to know more about tasks, try the ["Task tutorial"](../examples/30_extended/tasks_tutorial) - -## Running machine learning algorithms and uploading results - -In order to upload and share results of running a machine learning -algorithm on a task, we need to create an -[openml.runs.OpenMLRun][]. A run object can be -created by running a [openml.flows.OpenMLFlow][] or a scikit-learn compatible model on a task. We will -focus on the simpler example of running a scikit-learn model. - -Flows are descriptions of something runnable which does the machine -learning. A flow contains all information to set up the necessary -machine learning library and its dependencies as well as all possible -parameters. - -A run is the outcome of running a flow on a task. It contains all -parameter settings for the flow, a setup string (most likely a command -line call) and all predictions of that run. When a run is uploaded to -the server, the server automatically calculates several metrics which -can be used to compare the performance of different flows to each other. - -So far, the OpenML Python connector works only with estimator objects -following the [scikit-learn estimator -API](https://scikit-learn.org/stable/developers/develop.html#apis-of-scikit-learn-objects). -Those can be directly run on a task, and a flow will automatically be -created or downloaded from the server if it already exists. - -See ["Simple Flows and Runs"](../examples/20_basic/simple_flows_and_runs_tutorial) for a tutorial covers how to train different machine learning models, -how to run machine learning models on OpenML data and how to share the -results. - -## Datasets - -OpenML provides a large collection of datasets and the benchmark -[OpenML100](https://docs.openml.org/benchmark/) which consists of a -curated list of datasets. - -You can find the dataset that best fits your requirements by making use -of the available metadata. The tutorial ["extended datasets"](../examples/30_extended/datasets_tutorial) which follows explains how to -get a list of datasets, how to filter the list to find the dataset that -suits your requirements and how to download a dataset. - -OpenML is about sharing machine learning results and the datasets they -were obtained on. Learn how to share your datasets in the following -tutorial ["Upload"](../examples/30_extended/create_upload_tutorial) tutorial. - -# Extending OpenML-Python - -OpenML-Python provides an extension interface to connect machine -learning libraries directly to the API and ships a `scikit-learn` -extension. Read more about them in the ["Extensions"](extensions.md) section. - -[intro]: examples/20_basic/introduction_tutorial/ - diff --git a/examples/20_basic/README.txt b/examples/20_basic/README.txt deleted file mode 100644 index 29c787116..000000000 --- a/examples/20_basic/README.txt +++ /dev/null @@ -1,4 +0,0 @@ -Introductory Examples -===================== - -Introductory examples to the usage of the OpenML python connector. diff --git a/examples/20_basic/introduction_tutorial.py b/examples/20_basic/introduction_tutorial.py deleted file mode 100644 index a850a0792..000000000 --- a/examples/20_basic/introduction_tutorial.py +++ /dev/null @@ -1,115 +0,0 @@ -# %% [markdown] -# # Introduction tutorial & Setup -# An example how to set up OpenML-Python followed up by a simple example. - -# %% [markdown] -# OpenML is an online collaboration platform for machine learning which allows -# you to: -# -# * Find or share interesting, well-documented datasets -# * Define research / modelling goals (tasks) -# * Explore large amounts of machine learning algorithms, with APIs in Java, R, Python -# * Log and share reproducible experiments, models, results -# * Works seamlessly with scikit-learn and other libraries -# * Large scale benchmarking, compare to state of the art -# - -# %% [markdown] -# # Installation -# Installation is done via ``pip``: -# -# ```bash -# pip install openml -# ``` - -# %% [markdown] -# # Authentication -# -# The OpenML server can only be accessed by users who have signed up on the -# OpenML platform. If you don’t have an account yet, sign up now. -# You will receive an API key, which will authenticate you to the server -# and allow you to download and upload datasets, tasks, runs and flows. -# -# * Create an OpenML account (free) on https://www.openml.org. -# * After logging in, open your account page (avatar on the top right) -# * Open 'Account Settings', then 'API authentication' to find your API key. -# -# There are two ways to permanently authenticate: -# -# * Use the ``openml`` CLI tool with ``openml configure apikey MYKEY``, -# replacing **MYKEY** with your API key. -# * Create a plain text file **~/.openml/config** with the line -# **'apikey=MYKEY'**, replacing **MYKEY** with your API key. The config -# file must be in the directory ~/.openml/config and exist prior to -# importing the openml module. -# -# Alternatively, by running the code below and replacing 'YOURKEY' with your API key, -# you authenticate for the duration of the python process. - - -# %% - -import openml -from sklearn import neighbors - -# %% [markdown] -#
-#

Warning

-#

-# This example uploads data. For that reason, this example connects to the -# test server at test.openml.org.
-# This prevents the main server from becoming overloaded with example datasets, tasks, -# runs, and other submissions.
-# Using this test server may affect the behavior and performance of the -# OpenML-Python API. -#

-#
- -# %% -# openml.config.start_using_configuration_for_example() - -# %% [markdown] -# When using the main server instead, make sure your apikey is configured. -# This can be done with the following line of code (uncomment it!). -# Never share your apikey with others. - -# %% -# openml.config.apikey = 'YOURKEY' - -# %% [markdown] -# # Caching -# When downloading datasets, tasks, runs and flows, they will be cached to -# retrieve them without calling the server later. As with the API key, -# the cache directory can be either specified through the config file or -# through the API: -# -# * Add the line **cachedir = 'MYDIR'** to the config file, replacing -# 'MYDIR' with the path to the cache directory. By default, OpenML -# will use **~/.openml/cache** as the cache directory. -# * Run the code below, replacing 'YOURDIR' with the path to the cache directory. - -# %% -# Uncomment and set your OpenML cache directory -# import os -# openml.config.cache_directory = os.path.expanduser('YOURDIR') -openml.config.set_root_cache_directory("YOURDIR") - -# %% [markdown] -# # Simple Example -# Download the OpenML task for the eeg-eye-state. - -# %% -task = openml.tasks.get_task(403) -clf = neighbors.KNeighborsClassifier(n_neighbors=5) -openml.config.start_using_configuration_for_example() - -run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False) -# Publish the experiment on OpenML (optional, requires an API key). -# For this tutorial, our configuration publishes to the test server -# as to not crowd the main server with runs created by examples. -myrun = run.publish() - -# %% -openml.config.stop_using_configuration_for_example() -# License: BSD 3-Clause diff --git a/examples/20_basic/simple_datasets_tutorial.py b/examples/20_basic/simple_datasets_tutorial.py deleted file mode 100644 index f855184c0..000000000 --- a/examples/20_basic/simple_datasets_tutorial.py +++ /dev/null @@ -1,78 +0,0 @@ -# %% [markdown] -# # Datasets -# A basic tutorial on how to list, load and visualize datasets. -# -# In general, we recommend working with tasks, so that the results can -# be easily reproduced. Furthermore, the results can be compared to existing results -# at OpenML. However, for the purposes of this tutorial, we are going to work with -# the datasets directly. - -# %% - -import openml - -# %% [markdown] -# ## List datasets - -# %% -datasets_df = openml.datasets.list_datasets(output_format="dataframe") -print(datasets_df.head(n=10)) - -# %% [markdown] -# ## Download a dataset - -# %% -# Iris dataset https://www.openml.org/d/61 -dataset = openml.datasets.get_dataset(dataset_id=61, version=1) - -# Print a summary -print( - f"This is dataset '{dataset.name}', the target feature is " - f"'{dataset.default_target_attribute}'" -) -print(f"URL: {dataset.url}") -print(dataset.description[:500]) - -# %% [markdown] -# ## Load a dataset -# X - An array/dataframe where each row represents one example with -# the corresponding feature values. -# -# y - the classes for each example -# -# categorical_indicator - an array that indicates which feature is categorical -# -# attribute_names - the names of the features for the examples (X) and -# target feature (y) - -# %% -X, y, categorical_indicator, attribute_names = dataset.get_data( - target=dataset.default_target_attribute -) - -# %% [markdown] -# Visualize the dataset - -<<<<<<< docs/mkdoc -- Incoming Change -# %% -======= -import matplotlib.pyplot as plt ->>>>>>> develop -- Current Change -import pandas as pd -import seaborn as sns - -sns.set_style("darkgrid") - - -def hide_current_axis(*args, **kwds): - plt.gca().set_visible(False) - - -# We combine all the data so that we can map the different -# examples to different colors according to the classes. -combined_data = pd.concat([X, y], axis=1) -iris_plot = sns.pairplot(combined_data, hue="class") -iris_plot.map_upper(hide_current_axis) -plt.show() - -# License: BSD 3-Clause diff --git a/examples/20_basic/simple_flows_and_runs_tutorial.py b/examples/20_basic/simple_flows_and_runs_tutorial.py deleted file mode 100644 index 9f35e8bc1..000000000 --- a/examples/20_basic/simple_flows_and_runs_tutorial.py +++ /dev/null @@ -1,65 +0,0 @@ -# %% [markdown] -# # Flows and Runs -# A simple tutorial on how to train/run a model and how to upload the results. - -# %% -import openml -from sklearn import ensemble, neighbors - -from openml.utils import thread_safe_if_oslo_installed - - -# %% [markdown] -#
-#

Warning

-#

-# This example uploads data. For that reason, this example connects to the -# test server at test.openml.org.
-# This prevents the main server from becoming overloaded with example datasets, tasks, -# runs, and other submissions.
-# Using this test server may affect the behavior and performance of the -# OpenML-Python API. -#

-#
- -# %% -openml.config.start_using_configuration_for_example() - -# %% [markdown] -# ## Train a machine learning model - -# NOTE: We are using dataset 20 from the test server: https://test.openml.org/d/20 - -# %% -dataset = openml.datasets.get_dataset(20) -X, y, categorical_indicator, attribute_names = dataset.get_data( - dataset_format="dataframe", target=dataset.default_target_attribute -) -if y is None: - y = X["class"] - X = X.drop(columns=["class"], axis=1) -clf = neighbors.KNeighborsClassifier(n_neighbors=3) -clf.fit(X, y) - -# %% [markdown] -# ## Running a model on a task - -# %% -task = openml.tasks.get_task(119) - -clf = ensemble.RandomForestClassifier() -run = openml.runs.run_model_on_task(clf, task) -print(run) - -# %% [markdown] -# ## Publishing the run - -# %% -myrun = run.publish() -print(f"Run was uploaded to {myrun.openml_url}") -print(f"The flow can be found at {myrun.flow.openml_url}") - -# %% -openml.config.stop_using_configuration_for_example() -# License: BSD 3-Clause diff --git a/examples/30_extended/README.txt b/examples/30_extended/README.txt deleted file mode 100644 index 432fa68f0..000000000 --- a/examples/30_extended/README.txt +++ /dev/null @@ -1,4 +0,0 @@ -In-Depth Examples -================= - -Extended examples for the usage of the OpenML python connector. \ No newline at end of file diff --git a/examples/40_paper/2018_kdd_rijn_example.py b/examples/40_paper/2018_kdd_rijn_example.py deleted file mode 100644 index 315c27dc3..000000000 --- a/examples/40_paper/2018_kdd_rijn_example.py +++ /dev/null @@ -1,213 +0,0 @@ -# %% [markdown] -# # van Rijn and Hutter (2018) -# -# A tutorial on how to reproduce the paper *Hyperparameter Importance Across Datasets*. -# -# This is a Unix-only tutorial, as the requirements can not be satisfied on a Windows machine (Untested on other -# systems). -# -# ## Publication -# -# | Hyperparameter importance across datasets -# | Jan N. van Rijn and Frank Hutter -# | In *Proceedings of the 24th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining*, 2018 -# | Available at https://dl.acm.org/doi/10.1145/3219819.3220058 - -import sys -# DEPRECATED EXAMPLE -- Avoid running this code in our CI/CD pipeline -print("This example is deprecated, remove this code to use it manually.") -if not run_code: - print("Exiting...") - sys.exit() - -import json - -import fanova -import matplotlib.pyplot as plt -import pandas as pd -import seaborn as sns - -import openml - -############################################################################## -# With the advent of automated machine learning, automated hyperparameter -# optimization methods are by now routinely used in data mining. However, this -# progress is not yet matched by equal progress on automatic analyses that -# yield information beyond performance-optimizing hyperparameter settings. -# In this example, we aim to answer the following two questions: Given an -# algorithm, what are generally its most important hyperparameters? -# -# This work is carried out on the OpenML-100 benchmark suite, which can be -# obtained by ``openml.study.get_suite('OpenML100')``. In this example, we -# conduct the experiment on the Support Vector Machine (``flow_id=7707``) -# with specific kernel (we will perform a post-process filter operation for -# this). We should set some other experimental parameters (number of results -# per task, evaluation measure and the number of trees of the internal -# functional Anova) before the fun can begin. -# -# Note that we simplify the example in several ways: -# -# 1) We only consider numerical hyperparameters -# 2) We consider all hyperparameters that are numerical (in reality, some -# hyperparameters might be inactive (e.g., ``degree``) or irrelevant -# (e.g., ``random_state``) -# 3) We assume all hyperparameters to be on uniform scale -# -# Any difference in conclusion between the actual paper and the presented -# results is most likely due to one of these simplifications. For example, -# the hyperparameter C looks rather insignificant, whereas it is quite -# important when it is put on a log-scale. All these simplifications can be -# addressed by defining a ConfigSpace. For a more elaborated example that uses -# this, please see: -# https://github.com/janvanrijn/openml-pimp/blob/d0a14f3eb480f2a90008889f00041bdccc7b9265/examples/plot/plot_fanova_aggregates.py - -suite = openml.study.get_suite("OpenML100") -flow_id = 7707 -parameter_filters = {"sklearn.svm.classes.SVC(17)_kernel": "sigmoid"} -evaluation_measure = "predictive_accuracy" -limit_per_task = 500 -limit_nr_tasks = 15 -n_trees = 16 - -fanova_results = [] -# we will obtain all results from OpenML per task. Practice has shown that this places the bottleneck on the -# communication with OpenML, and for iterated experimenting it is better to cache the results in a local file. -for idx, task_id in enumerate(suite.tasks): - if limit_nr_tasks is not None and idx >= limit_nr_tasks: - continue - print( - "Starting with task %d (%d/%d)" - % (task_id, idx + 1, len(suite.tasks) if limit_nr_tasks is None else limit_nr_tasks) - ) - # note that we explicitly only include tasks from the benchmark suite that was specified (as per the for-loop) - evals = openml.evaluations.list_evaluations_setups( - evaluation_measure, - flows=[flow_id], - tasks=[task_id], - size=limit_per_task, - ) - -# %% [markdown] -# With the advent of automated machine learning, automated hyperparameter -# optimization methods are by now routinely used in data mining. However, this -# progress is not yet matched by equal progress on automatic analyses that -# yield information beyond performance-optimizing hyperparameter settings. -# In this example, we aim to answer the following two questions: Given an -# algorithm, what are generally its most important hyperparameters? -# -# This work is carried out on the OpenML-100 benchmark suite, which can be -# obtained by ``openml.study.get_suite('OpenML100')``. In this example, we -# conduct the experiment on the Support Vector Machine (``flow_id=7707``) -# with specific kernel (we will perform a post-process filter operation for -# this). We should set some other experimental parameters (number of results -# per task, evaluation measure and the number of trees of the internal -# functional Anova) before the fun can begin. -# -# Note that we simplify the example in several ways: -# -# 1) We only consider numerical hyperparameters -# 2) We consider all hyperparameters that are numerical (in reality, some -# hyperparameters might be inactive (e.g., ``degree``) or irrelevant -# (e.g., ``random_state``) -# 3) We assume all hyperparameters to be on uniform scale -# -# Any difference in conclusion between the actual paper and the presented -# results is most likely due to one of these simplifications. For example, -# the hyperparameter C looks rather insignificant, whereas it is quite -# important when it is put on a log-scale. All these simplifications can be -# addressed by defining a ConfigSpace. For a more elaborated example that uses -# this, please see: -# https://github.com/janvanrijn/openml-pimp/blob/d0a14f3eb480f2a90008889f00041bdccc7b9265/examples/plot/plot_fanova_aggregates.py # noqa F401 - -# %% - suite = openml.study.get_suite("OpenML100") - flow_id = 7707 - parameter_filters = {"sklearn.svm.classes.SVC(17)_kernel": "sigmoid"} - evaluation_measure = "predictive_accuracy" - limit_per_task = 500 - limit_nr_tasks = 15 - n_trees = 16 - - fanova_results = [] - # we will obtain all results from OpenML per task. Practice has shown that this places the bottleneck on the - # communication with OpenML, and for iterated experimenting it is better to cache the results in a local file. - for idx, task_id in enumerate(suite.tasks): - if limit_nr_tasks is not None and idx >= limit_nr_tasks: - continue - print( - "Starting with task %d (%d/%d)" - % (task_id, idx + 1, len(suite.tasks) if limit_nr_tasks is None else limit_nr_tasks) - ) - # note that we explicitly only include tasks from the benchmark suite that was specified (as per the for-loop) - evals = openml.evaluations.list_evaluations_setups( - evaluation_measure, - flows=[flow_id], - tasks=[task_id], - size=limit_per_task, - output_format="dataframe", - ) - except json.decoder.JSONDecodeError as e: - print("Task %d error: %s" % (task_id, e)) - continue - # apply our filters, to have only the setups that comply to the hyperparameters we want - for filter_key, filter_value in parameter_filters.items(): - setups_evals = setups_evals[setups_evals[filter_key] == filter_value] - # in this simplified example, we only display numerical and float hyperparameters. For categorical hyperparameters, - # the fanova library needs to be informed by using a configspace object. - setups_evals = setups_evals.select_dtypes(include=["int64", "float64"]) - # drop rows with unique values. These are by definition not an interesting hyperparameter, e.g., ``axis``, - # ``verbose``. - setups_evals = setups_evals[ - [ - c - for c in list(setups_evals) - if len(setups_evals[c].unique()) > 1 or c == performance_column - ] - ] - # We are done with processing ``setups_evals``. Note that we still might have some irrelevant hyperparameters, e.g., - # ``random_state``. We have dropped some relevant hyperparameters, i.e., several categoricals. Let's check it out: - - # determine x values to pass to fanova library - parameter_names = [ - pname for pname in setups_evals.columns.to_numpy() if pname != performance_column - ] - evaluator = fanova.fanova.fANOVA( - X=setups_evals[parameter_names].to_numpy(), - Y=setups_evals[performance_column].to_numpy(), - n_trees=n_trees, - ) - for idx, pname in enumerate(parameter_names): - try: - fanova_results.append( - { - "hyperparameter": pname.split(".")[-1], - "fanova": evaluator.quantify_importance([idx])[(idx,)][ - "individual importance" - ], - } - ) - except RuntimeError as e: - # functional ANOVA sometimes crashes with a RuntimeError, e.g., on tasks where the performance is constant - # for all configurations (there is no variance). We will skip these tasks (like the authors did in the - # paper). - print("Task %d error: %s" % (task_id, e)) - continue - - # transform ``fanova_results`` from a list of dicts into a DataFrame - fanova_results = pd.DataFrame(fanova_results) - -# %% [markdown] -# make the boxplot of the variance contribution. Obviously, we can also use -# this data to make the Nemenyi plot, but this relies on the rather complex -# ``Orange`` dependency (``pip install Orange3``). For the complete example, -# the reader is referred to the more elaborate script (referred to earlier) - - # %% - fig, ax = plt.subplots() - sns.boxplot(x="hyperparameter", y="fanova", data=fanova_results, ax=ax) - ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha="right") - ax.set_ylabel("Variance Contribution") - ax.set_xlabel(None) - plt.tight_layout() - plt.show() - # License: BSD 3-Clause diff --git a/examples/40_paper/README.txt b/examples/40_paper/README.txt deleted file mode 100644 index 9b571d55b..000000000 --- a/examples/40_paper/README.txt +++ /dev/null @@ -1,5 +0,0 @@ -Usage in research papers -======================== - -These examples demonstrate how OpenML-Python can be used for research purposes by re-implementing -its use in recent publications. diff --git a/examples/30_extended/configure_logging.py b/examples/Advanced/configure_logging.py similarity index 95% rename from examples/30_extended/configure_logging.py rename to examples/Advanced/configure_logging.py index 0191253e9..60b789846 100644 --- a/examples/30_extended/configure_logging.py +++ b/examples/Advanced/configure_logging.py @@ -1,5 +1,4 @@ # %% [markdown] -# # Logging # This tutorial explains openml-python logging, and shows how to configure it. # Openml-python uses the [Python logging module](https://docs.python.org/3/library/logging.html) # to provide users with log messages. Each log message is assigned a level of importance, see @@ -9,7 +8,7 @@ # By default, openml-python will print log messages of level `WARNING` and above to console. # All log messages (including `DEBUG` and `INFO`) are also saved in a file, which can be # found in your cache directory (see also the -# [introduction tutorial](../20_basic/introduction_tutorial). +# [introduction tutorial](../Basics/introduction_tutorial). # These file logs are automatically deleted if needed, and use at most 2MB of space. # # It is possible to configure what log levels to send to console and file. @@ -49,5 +48,3 @@ # * 0: `logging.WARNING` and up. # * 1: `logging.INFO` and up. # * 2: `logging.DEBUG` and up (i.e. all messages). -# -# License: BSD 3-Clause diff --git a/examples/30_extended/create_upload_tutorial.py b/examples/Advanced/create_upload_tutorial.py similarity index 97% rename from examples/30_extended/create_upload_tutorial.py rename to examples/Advanced/create_upload_tutorial.py index 2b010401c..46ec96319 100644 --- a/examples/30_extended/create_upload_tutorial.py +++ b/examples/Advanced/create_upload_tutorial.py @@ -1,5 +1,4 @@ # %% [markdown] -# # Dataset upload tutorial # A tutorial on how to create and upload a dataset to OpenML. # %% @@ -11,10 +10,6 @@ import openml from openml.datasets.functions import create_dataset -# %% [markdown] -# .. warning:: -# .. include:: ../../test_server_usage_warning.txt - # %% openml.config.start_using_configuration_for_example() @@ -28,8 +23,7 @@ # * A pandas sparse dataframe # %% [markdown] -# Dataset is a numpy array -# ======================== +# ## Dataset is a numpy array # A numpy array can contain lists in the case of dense data or it can contain # OrderedDicts in the case of sparse data. # @@ -66,7 +60,7 @@ paper_url = "https://web.stanford.edu/~hastie/Papers/LARS/LeastAngle_2002.pdf" # %% [markdown] -# # Create the dataset object +# ## Create the dataset object # The definition of all fields can be found in the XSD files describing the # expected format: # @@ -237,8 +231,7 @@ print(f"URL for dataset: {weather_dataset.openml_url}") # %% [markdown] -# Dataset is a sparse matrix -# ========================== +# ## Dataset is a sparse matrix # %% sparse_data = coo_matrix( @@ -308,4 +301,3 @@ # %% openml.config.stop_using_configuration_for_example() -# License: BSD 3-Clause diff --git a/examples/30_extended/datasets_tutorial.py b/examples/Advanced/datasets_tutorial.py similarity index 95% rename from examples/30_extended/datasets_tutorial.py rename to examples/Advanced/datasets_tutorial.py index d7c74b843..cc57686d0 100644 --- a/examples/30_extended/datasets_tutorial.py +++ b/examples/Advanced/datasets_tutorial.py @@ -1,7 +1,7 @@ # %% [markdown] -# # Datasets # How to list and download datasets. +# %% import pandas as pd import openml @@ -11,6 +11,8 @@ # ## Exercise 0 # # * List datasets and return a dataframe + +# %% datalist = openml.datasets.list_datasets() datalist = datalist[["did", "name", "NumberOfInstances", "NumberOfFeatures", "NumberOfClasses"]] @@ -46,8 +48,7 @@ # Print a summary print( - f"This is dataset '{dataset.name}', the target feature is " - f"'{dataset.default_target_attribute}'" + f"This is dataset '{dataset.name}', the target feature is '{dataset.default_target_attribute}'" ) print(f"URL: {dataset.url}") print(dataset.description[:500]) @@ -106,9 +107,6 @@ # %% [markdown] # ## Edit a created dataset # This example uses the test server, to avoid editing a dataset on the main server. -# -# .. warning:: -# .. include:: ../../test_server_usage_warning.txt # %% openml.config.start_using_configuration_for_example() @@ -165,4 +163,3 @@ # %% openml.config.stop_using_configuration_for_example() -# License: BSD 3-Clauses diff --git a/examples/30_extended/fetch_evaluations_tutorial.py b/examples/Advanced/fetch_evaluations_tutorial.py similarity index 97% rename from examples/30_extended/fetch_evaluations_tutorial.py rename to examples/Advanced/fetch_evaluations_tutorial.py index 21f36a194..1b759423b 100644 --- a/examples/30_extended/fetch_evaluations_tutorial.py +++ b/examples/Advanced/fetch_evaluations_tutorial.py @@ -1,6 +1,4 @@ # %% [markdown] -# # Fetching Evaluations - # Evaluations contain a concise summary of the results of all runs made. Each evaluation # provides information on the dataset used, the flow applied, the setup used, the metric # evaluated, and the result obtained on the metric, for each such run made. These collection @@ -27,9 +25,7 @@ # We shall retrieve a small set (only 10 entries) to test the listing function for evaluations # %% -openml.evaluations.list_evaluations( - function="predictive_accuracy", size=10 -) +openml.evaluations.list_evaluations(function="predictive_accuracy", size=10) # Using other evaluation metrics, 'precision' in this case evals = openml.evaluations.list_evaluations( @@ -182,6 +178,4 @@ def plot_flow_compare(evaluations, top_n=10, metric="predictive_accuracy"): function="predictive_accuracy", flows=[6767], size=100, parameters_in_separate_columns=True ) -print(evals_setups.head(10)) - -# License: BSD 3-Clause +print(evals_setups.head(10)) \ No newline at end of file diff --git a/examples/30_extended/study_tutorial.py b/examples/Advanced/study_tutorial.py similarity index 85% rename from examples/30_extended/study_tutorial.py rename to examples/Advanced/study_tutorial.py index 416e543bb..6912efd06 100644 --- a/examples/30_extended/study_tutorial.py +++ b/examples/Advanced/study_tutorial.py @@ -1,5 +1,4 @@ # %% [markdown] -# # Benchmark studies # How to list, download and upload benchmark studies. # In contrast to # [benchmark suites](https://docs.openml.org/benchmark/#benchmarking-suites) which @@ -13,7 +12,6 @@ import openml - # %% [markdown] # ## Listing studies # @@ -22,14 +20,12 @@ # easier-to-work-with data structure # %% -studies = openml.study.list_studies(output_format="dataframe", status="all") +studies = openml.study.list_studies(status="all") print(studies.head(n=10)) # %% [markdown] # ## Downloading studies - -# %% [markdown] # This is done based on the study ID. # %% @@ -62,9 +58,6 @@ # %% [markdown] # We'll use the test server for the rest of this tutorial. -# -# .. warning:: -# .. include:: ../../test_server_usage_warning.txt # %% openml.config.start_using_configuration_for_example() @@ -76,7 +69,20 @@ # In this examples we'll create a few runs for the OpenML-100 benchmark # suite which is available on the OpenML test server. +#
+#

Warning

+#

+# For the rest of this tutorial, we will require the `openml-sklearn` package. +# Install it with `pip install openml-sklearn`. +#

+#
+ # %% +# Get sklearn extension to run sklearn models easily on OpenML tasks. +from openml_sklearn import SklearnExtension + +extension = SklearnExtension() + # Model to be used clf = RandomForestClassifier() @@ -112,4 +118,3 @@ # %% openml.config.stop_using_configuration_for_example() -# License: BSD 3-Clause diff --git a/examples/30_extended/suites_tutorial.py b/examples/Advanced/suites_tutorial.py similarity index 77% rename from examples/30_extended/suites_tutorial.py rename to examples/Advanced/suites_tutorial.py index a92c1cdb5..7ca42079d 100644 --- a/examples/30_extended/suites_tutorial.py +++ b/examples/Advanced/suites_tutorial.py @@ -1,11 +1,5 @@ # %% [markdown] -# # Benchmark suites -# # How to list, download and upload benchmark suites. -# -# If you want to learn more about benchmark suites, check out our -# brief introductory tutorial ["Simple suites tutorial"](../20_basic/simple_suites_tutorial) or the -# [OpenML benchmark docs](https://docs.openml.org/benchmark/#benchmarking-suites). # %% import uuid @@ -14,7 +8,6 @@ import openml - # %% [markdown] # ## Listing suites # @@ -23,13 +16,11 @@ # easier-to-work-with data structure # %% -suites = openml.study.list_suites(output_format="dataframe", status="all") +suites = openml.study.list_suites(status="all") print(suites.head(n=10)) # %% [markdown] # ## Downloading suites - -# %% [markdown] # This is done based on the dataset ID. # %% @@ -52,7 +43,7 @@ # And we can use the task listing functionality to learn more about them: # %% -tasks = openml.tasks.list_tasks(output_format="dataframe") +tasks = openml.tasks.list_tasks() # %% [markdown] # Using ``@`` in @@ -65,9 +56,6 @@ # %% [markdown] # We'll use the test server for the rest of this tutorial. -# -# .. warning:: -# .. include:: ../../test_server_usage_warning.txt # %% openml.config.start_using_configuration_for_example() @@ -83,7 +71,7 @@ # the test server: # %% -all_tasks = list(openml.tasks.list_tasks(output_format="dataframe")["tid"]) +all_tasks = list(openml.tasks.list_tasks()["tid"]) task_ids_for_suite = sorted(np.random.choice(all_tasks, replace=False, size=20)) # The study needs a machine-readable and unique alias. To obtain this, @@ -102,4 +90,3 @@ # %% openml.config.stop_using_configuration_for_example() -# License: BSD 3-Clause diff --git a/examples/30_extended/task_manual_iteration_tutorial.py b/examples/Advanced/task_manual_iteration_tutorial.py similarity index 62% rename from examples/30_extended/task_manual_iteration_tutorial.py rename to examples/Advanced/task_manual_iteration_tutorial.py index 8b35633a2..1e630e213 100644 --- a/examples/30_extended/task_manual_iteration_tutorial.py +++ b/examples/Advanced/task_manual_iteration_tutorial.py @@ -1,13 +1,5 @@ # %% [markdown] -# # Tasks: retrieving splits - -# Tasks define a target and a train/test split. Normally, they are the input to the function -# ``openml.runs.run_model_on_task`` which automatically runs the model on all splits of the task. -# However, sometimes it is necessary to manually split a dataset to perform experiments outside of -# the functions provided by OpenML. One such example is in the benchmark library -# [HPOBench](https://github.com/automl/HPOBench) which extensively uses data from OpenML, -# but not OpenML's functionality to conduct runs. - +# Tasks define a target and a train/test split, which we can use for benchmarking. # %% import openml @@ -45,12 +37,7 @@ # %% print( - "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format( - task_id, - n_repeats, - n_folds, - n_samples, - ) + f"Task {task_id}: number of repeats: {n_repeats}, number of folds: {n_folds}, number of samples {n_samples}." ) # %% [markdown] @@ -72,19 +59,14 @@ # And then split the data based on this: # %% -X, y = task.get_X_and_y(dataset_format="dataframe") +X, y = task.get_X_and_y() X_train = X.iloc[train_indices] y_train = y.iloc[train_indices] X_test = X.iloc[test_indices] y_test = y.iloc[test_indices] print( - "X_train.shape: {}, y_train.shape: {}, X_test.shape: {}, y_test.shape: {}".format( - X_train.shape, - y_train.shape, - X_test.shape, - y_test.shape, - ) + f"X_train.shape: {X_train.shape}, y_train.shape: {y_train.shape}, X_test.shape: {X_test.shape}, y_test.shape: {y_test.shape}" ) # %% [markdown] @@ -96,12 +78,7 @@ X, y = task.get_X_and_y() n_repeats, n_folds, n_samples = task.get_split_dimensions() print( - "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format( - task_id, - n_repeats, - n_folds, - n_samples, - ) + f"Task {task_id}: number of repeats: {n_repeats}, number of folds: {n_folds}, number of samples {n_samples}." ) # %% [markdown] @@ -122,16 +99,8 @@ y_test = y.iloc[test_indices] print( - "Repeat #{}, fold #{}, samples {}: X_train.shape: {}, " - "y_train.shape {}, X_test.shape {}, y_test.shape {}".format( - repeat_idx, - fold_idx, - sample_idx, - X_train.shape, - y_train.shape, - X_test.shape, - y_test.shape, - ) + f"Repeat #{repeat_idx}, fold #{fold_idx}, samples {sample_idx}: X_train.shape: {X_train.shape}, " + f"y_train.shape {y_train.shape}, X_test.shape {X_test.shape}, y_test.shape {y_test.shape}" ) # %% [markdown] @@ -143,12 +112,7 @@ X, y = task.get_X_and_y() n_repeats, n_folds, n_samples = task.get_split_dimensions() print( - "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format( - task_id, - n_repeats, - n_folds, - n_samples, - ) + f"Task {task_id}: number of repeats: {n_repeats}, number of folds: {n_folds}, number of samples {n_samples}." ) # %% [markdown] @@ -169,16 +133,8 @@ y_test = y.iloc[test_indices] print( - "Repeat #{}, fold #{}, samples {}: X_train.shape: {}, " - "y_train.shape {}, X_test.shape {}, y_test.shape {}".format( - repeat_idx, - fold_idx, - sample_idx, - X_train.shape, - y_train.shape, - X_test.shape, - y_test.shape, - ) + f"Repeat #{repeat_idx}, fold #{fold_idx}, samples {sample_idx}: X_train.shape: {X_train.shape}, " + f"y_train.shape {y_train.shape}, X_test.shape {X_test.shape}, y_test.shape {y_test.shape}" ) # %% [markdown] @@ -190,12 +146,7 @@ X, y = task.get_X_and_y() n_repeats, n_folds, n_samples = task.get_split_dimensions() print( - "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format( - task_id, - n_repeats, - n_folds, - n_samples, - ) + f"Task {task_id}: number of repeats: {n_repeats}, number of folds: {n_folds}, number of samples {n_samples}." ) # %% [markdown] @@ -216,15 +167,6 @@ y_test = y.iloc[test_indices] print( - "Repeat #{}, fold #{}, samples {}: X_train.shape: {}, " - "y_train.shape {}, X_test.shape {}, y_test.shape {}".format( - repeat_idx, - fold_idx, - sample_idx, - X_train.shape, - y_train.shape, - X_test.shape, - y_test.shape, - ) + f"Repeat #{repeat_idx}, fold #{fold_idx}, samples {sample_idx}: X_train.shape: {X_train.shape}, " + f"y_train.shape {y_train.shape}, X_test.shape {X_test.shape}, y_test.shape {y_test.shape}" ) -# License: BSD 3-Clause diff --git a/examples/30_extended/tasks_tutorial.py b/examples/Advanced/tasks_tutorial.py similarity index 87% rename from examples/30_extended/tasks_tutorial.py rename to examples/Advanced/tasks_tutorial.py index 54a373fca..dff7293ad 100644 --- a/examples/30_extended/tasks_tutorial.py +++ b/examples/Advanced/tasks_tutorial.py @@ -1,5 +1,4 @@ # %% [markdown] -# # Tasks # A tutorial on how to list and download tasks. # %% @@ -31,9 +30,7 @@ # instead to have better visualization capabilities and easier access: # %% -tasks = openml.tasks.list_tasks( - task_type=TaskType.SUPERVISED_CLASSIFICATION, output_format="dataframe" -) +tasks = openml.tasks.list_tasks(task_type=TaskType.SUPERVISED_CLASSIFICATION) print(tasks.columns) print(f"First 5 of {len(tasks)} tasks:") print(tasks.head()) @@ -69,7 +66,7 @@ # Similar to listing tasks by task type, we can list tasks by tags: # %% -tasks = openml.tasks.list_tasks(tag="OpenML100", output_format="dataframe") +tasks = openml.tasks.list_tasks(tag="OpenML100") print(f"First 5 of {len(tasks)} tasks:") print(tasks.head()) @@ -77,7 +74,7 @@ # Furthermore, we can list tasks based on the dataset id: # %% -tasks = openml.tasks.list_tasks(data_id=1471, output_format="dataframe") +tasks = openml.tasks.list_tasks(data_id=1471) print(f"First 5 of {len(tasks)} tasks:") print(tasks.head()) @@ -85,7 +82,7 @@ # In addition, a size limit and an offset can be applied both separately and simultaneously: # %% -tasks = openml.tasks.list_tasks(size=10, offset=50, output_format="dataframe") +tasks = openml.tasks.list_tasks(size=10, offset=50) print(tasks) # %% [markdown] @@ -101,7 +98,7 @@ # Finally, it is also possible to list all tasks on OpenML with: # %% -tasks = openml.tasks.list_tasks(output_format="dataframe") +tasks = openml.tasks.list_tasks() print(len(tasks)) # %% [markdown] @@ -163,9 +160,7 @@ # %% [markdown] # We'll use the test server for the rest of this tutorial. -# -# .. warning:: -# .. include:: ../../test_server_usage_warning.txt + # %% openml.config.start_using_configuration_for_example() @@ -203,13 +198,4 @@ print("Task already exists. Task ID is", task_id) # %% -# reverting to prod server openml.config.stop_using_configuration_for_example() - - -# %% [markdown] -# * [Complete list of task types](https://www.openml.org/search?type=task_type). -# * [Complete list of model estimation procedures](https://www.openml.org/search?q=%2520measure_type%3Aestimation_procedure&type=measure). -# * [Complete list of evaluation measures](https://www.openml.org/search?q=measure_type%3Aevaluation_measure&type=measure). -# -# License: BSD 3-Clause diff --git a/examples/Basics/introduction_tutorial.py b/examples/Basics/introduction_tutorial.py new file mode 100644 index 000000000..c864772f5 --- /dev/null +++ b/examples/Basics/introduction_tutorial.py @@ -0,0 +1,55 @@ +# %% [markdown] +# ## Installation +# Installation is done via ``pip``: +# +# ```bash +# pip install openml +# ``` + +# %% [markdown] +# ## Authentication +# +# For certain functionality, such as uploading tasks or datasets, users have to +# sign up. Only accessing the data on OpenML does not require an account! +# +# If you don’t have an account yet, sign up now. +# You will receive an API key, which will authenticate you to the server +# and allow you to download and upload datasets, tasks, runs and flows. +# +# * Create an OpenML account (free) on https://www.openml.org. +# * After logging in, open your account page (avatar on the top right) +# * Open 'Account Settings', then 'API authentication' to find your API key. +# +# There are two ways to permanently authenticate: +# +# * Use the ``openml`` CLI tool with ``openml configure apikey MYKEY``, +# replacing **MYKEY** with your API key. +# * Create a plain text file **~/.openml/config** with the line +# **'apikey=MYKEY'**, replacing **MYKEY** with your API key. The config +# file must be in the directory ~/.openml/config and exist prior to +# importing the openml module. +# +# Alternatively, by running the code below and replacing 'YOURKEY' with your API key, +# you authenticate for the duration of the Python process. + +# %% +import openml + +openml.config.apikey = "YOURKEY" + +# %% [markdown] +# ## Caching +# When downloading datasets, tasks, runs and flows, they will be cached to +# retrieve them without calling the server later. As with the API key, +# the cache directory can be either specified through the config file or +# through the API: +# +# * Add the line **cachedir = 'MYDIR'** to the config file, replacing +# 'MYDIR' with the path to the cache directory. By default, OpenML +# will use **~/.openml/cache** as the cache directory. +# * Run the code below, replacing 'YOURDIR' with the path to the cache directory. + +# %% +import openml + +openml.config.set_root_cache_directory("YOURDIR") \ No newline at end of file diff --git a/examples/Basics/simple_datasets_tutorial.py b/examples/Basics/simple_datasets_tutorial.py new file mode 100644 index 000000000..75d36ed0f --- /dev/null +++ b/examples/Basics/simple_datasets_tutorial.py @@ -0,0 +1,57 @@ +# %% [markdown] +# A basic tutorial on how to list, load and visualize datasets. +# +# In general, we recommend working with tasks, so that the results can +# be easily reproduced. Furthermore, the results can be compared to existing results +# at OpenML. However, for the purposes of this tutorial, we are going to work with +# the datasets directly. + +# %% + +import openml + +# %% [markdown] +# ## List datasets stored on OpenML + +# %% +datasets_df = openml.datasets.list_datasets() +print(datasets_df.head(n=10)) + +# %% [markdown] +# ## Download a dataset + +# %% +# Iris dataset https://www.openml.org/d/61 +dataset = openml.datasets.get_dataset(dataset_id=61) + +# Print a summary +print( + f"This is dataset '{dataset.name}', the target feature is '{dataset.default_target_attribute}'" +) +print(f"URL: {dataset.url}") +print(dataset.description[:500]) + +# %% [markdown] +# ## Load a dataset +# * `X` - A dataframe where each row represents one example with +# the corresponding feature values. +# * `y` - the classes for each example +# * `categorical_indicator` - a list that indicates which feature is categorical +# * `attribute_names` - the names of the features for the examples (X) and +# target feature (y) + +# %% +X, y, categorical_indicator, attribute_names = dataset.get_data( + target=dataset.default_target_attribute +) + +# %% [markdown] +# Visualize the dataset + +# %% +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns + +iris_plot = sns.pairplot(pd.concat([X, y], axis=1), hue="class") +plt.show() diff --git a/examples/Basics/simple_flows_and_runs_tutorial.py b/examples/Basics/simple_flows_and_runs_tutorial.py new file mode 100644 index 000000000..41eed9234 --- /dev/null +++ b/examples/Basics/simple_flows_and_runs_tutorial.py @@ -0,0 +1,122 @@ +# %% [markdown] +# A simple tutorial on how to upload results from a machine learning experiment to OpenML. + +# %% +import sklearn +from sklearn.neighbors import KNeighborsClassifier + +import openml + +# %% [markdown] +#
+#

Warning

+#

+# This example uploads data. For that reason, this example connects to the +# test server at test.openml.org.
+# This prevents the main server from becoming overloaded with example datasets, tasks, +# runs, and other submissions.
+# Using this test server may affect the behavior and performance of the +# OpenML-Python API. +#

+#
+ +# %% +openml.config.start_using_configuration_for_example() + +# %% [markdown] +# ## Train a machine learning model and evaluate it +# NOTE: We are using task 119 from the test server: https://test.openml.org/d/20 + +# %% +task = openml.tasks.get_task(119) + +# Get the data +dataset = task.get_dataset() +X, y, categorical_indicator, attribute_names = dataset.get_data( + target=dataset.default_target_attribute +) + +# Get the holdout split from the task +train_indices, test_indices = task.get_train_test_split_indices(fold=0, repeat=0) +X_train, X_test = X.iloc[train_indices], X.iloc[test_indices] +y_train, y_test = y.iloc[train_indices], y.iloc[test_indices] + +knn_parameters = { + "n_neighbors": 3, +} +clf = KNeighborsClassifier(**knn_parameters) +clf.fit(X_train, y_train) + +# Get experiment results +y_pred = clf.predict(X_test) +y_pred_proba = clf.predict_proba(X_test) + +# %% [markdown] +# ## Upload the machine learning experiments to OpenML +# First, create a fow and fill it with metadata about the machine learning model. + +# %% +knn_flow = openml.flows.OpenMLFlow( + # Metadata + model=clf, # or None, if you do not want to upload the model object. + name="CustomKNeighborsClassifier", + description="A custom KNeighborsClassifier flow for OpenML.", + external_version=f"{sklearn.__version__}", + language="English", + tags=["openml_tutorial_knn"], + dependencies=f"{sklearn.__version__}", + # Hyperparameters + parameters={k: str(v) for k, v in knn_parameters.items()}, + parameters_meta_info={ + "n_neighbors": {"description": "number of neighbors to use", "data_type": "int"} + }, + # If you have a pipeline with subcomponents, such as preprocessing, add them here. + components={}, +) +knn_flow.publish() +print(f"knn_flow was published with the ID {knn_flow.flow_id}") + +# %% [markdown] +# Second, we create a run to store the results associated with the flow. + +# %% + +# Format the predictions for OpenML +predictions = [] +for test_index, y_true_i, y_pred_i, y_pred_proba_i in zip( + test_indices, y_test, y_pred, y_pred_proba +): + predictions.append( + openml.runs.functions.format_prediction( + task=task, + repeat=0, + fold=0, + index=test_index, + prediction=y_pred_i, + truth=y_true_i, + proba=dict(zip(task.class_labels, y_pred_proba_i)), + ) + ) + +# Format the parameters for OpenML +oml_knn_parameters = [ + {"oml:name": k, "oml:value": v, "oml:component": knn_flow.flow_id} + for k, v in knn_parameters.items() +] + +knn_run = openml.runs.OpenMLRun( + task_id=task.task_id, + flow_id=knn_flow.flow_id, + dataset_id=dataset.dataset_id, + parameter_settings=oml_knn_parameters, + data_content=predictions, + tags=["openml_tutorial_knn"], + description_text="Run generated by the tutorial.", +) +knn_run = knn_run.publish() +print(f"Run was uploaded to {knn_run.openml_url}") +print(f"The flow can be found at {knn_run.flow.openml_url}") + +# %% +openml.config.stop_using_configuration_for_example() diff --git a/examples/20_basic/simple_suites_tutorial.py b/examples/Basics/simple_suites_tutorial.py similarity index 67% rename from examples/20_basic/simple_suites_tutorial.py rename to examples/Basics/simple_suites_tutorial.py index 5a1b429b1..cc3c7b1cf 100644 --- a/examples/20_basic/simple_suites_tutorial.py +++ b/examples/Basics/simple_suites_tutorial.py @@ -1,5 +1,4 @@ # %% [markdown] -# # Benchmark suites # This is a brief showcase of OpenML benchmark suites, which were introduced by # [Bischl et al. (2019)](https://arxiv.org/abs/1708.03731v2). Benchmark suites standardize the # datasets and splits to be used in an experiment or paper. They are fully integrated into OpenML @@ -9,13 +8,11 @@ import openml # %% [markdown] -# OpenML-CC18 -# =========== +# ## OpenML-CC18 # # As an example we have a look at the OpenML-CC18, which is a suite of 72 classification datasets -# from OpenML which were carefully selected to be usable by many algorithms and also represent -# datasets commonly used in machine learning research. These are all datasets from mid-2018 that -# satisfy a large set of clear requirements for thorough yet practical benchmarking: +# from OpenML which were carefully selected to be usable by many algorithms. These are all datasets +# from mid-2018 that satisfy a large set of clear requirements for thorough yet practical benchmarking: # # 1. the number of observations are between 500 and 100,000 to focus on medium-sized datasets, # 2. the number of features does not exceed 5,000 features to keep the runtime of the algorithms @@ -28,11 +25,10 @@ # A full description can be found in the # [OpenML benchmarking docs](https://docs.openml.org/benchmark/#openml-cc18). # -# In this example we'll focus on how to use benchmark suites in practice. +# In this example, we'll focus on how to use benchmark suites in practice. # %% [markdown] -# Downloading benchmark suites -# ============================ +# ## Downloading benchmark suites # %% suite = openml.study.get_suite(99) @@ -49,19 +45,9 @@ print(tasks) # %% [markdown] -# and iterated over for benchmarking. For speed reasons we only iterate over the first three tasks: +# and iterated over for benchmarking. For speed reasons, we only iterate over the first three tasks: # %% for task_id in tasks[:3]: task = openml.tasks.get_task(task_id) print(task) - -# %% [markdown] -# Further examples -# ================ -# -# * [Suites Tutorial](../../30_extended/suites_tutorial) -# * [Study Tutoral](../../30_extended/study_tutorial) -# * [Paper example: Strang et al.](../../40_paper/2018_ida_strang_example.py) - -# License: BSD 3-Clause diff --git a/examples/Basics/simple_tasks_tutorial.py b/examples/Basics/simple_tasks_tutorial.py new file mode 100644 index 000000000..598ce4e71 --- /dev/null +++ b/examples/Basics/simple_tasks_tutorial.py @@ -0,0 +1,27 @@ +# %% [markdown] +# A brief example on how to use tasks from OpenML. + +# %% + +import openml + +# %% [markdown] +# Get a [task](https://docs.openml.org/concepts/tasks/) for +# [supervised classification on credit-g](https://www.openml.org/search?type=task&id=31&source_data.data_id=31): + +# %% +task = openml.tasks.get_task(31) + +# %% [markdown] +# Get the dataset and its data from the task. + +# %% +dataset = task.get_dataset() +X, y, categorical_indicator, attribute_names = dataset.get_data(target=task.target_name) + +# %% [markdown] +# Get the first out of the 10 cross-validation splits from the task. + +# %% +train_indices, test_indices = task.get_train_test_split_indices(fold=0) +print(train_indices[:10]) # print the first 10 indices of the training set diff --git a/examples/README.txt b/examples/README.txt deleted file mode 100644 index d10746bcb..000000000 --- a/examples/README.txt +++ /dev/null @@ -1,5 +0,0 @@ -.. _examples-index: - -================ -Examples Gallery -================ diff --git a/examples/40_paper/2015_neurips_feurer_example.py b/examples/_external_or_deprecated/2015_neurips_feurer_example.py similarity index 74% rename from examples/40_paper/2015_neurips_feurer_example.py rename to examples/_external_or_deprecated/2015_neurips_feurer_example.py index 8b1ac02f9..ae59c9ced 100644 --- a/examples/40_paper/2015_neurips_feurer_example.py +++ b/examples/_external_or_deprecated/2015_neurips_feurer_example.py @@ -1,27 +1,30 @@ -# %% [markdown] -# # Feurer et al. (2015) +""" +Feurer et al. (2015) +==================== -# A tutorial on how to get the datasets used in the paper introducing *Auto-sklearn* by Feurer et al.. -# -# Auto-sklearn website: https://automl.github.io/auto-sklearn/ -# -# ## Publication -# -# | Efficient and Robust Automated Machine Learning -# | Matthias Feurer, Aaron Klein, Katharina Eggensperger, Jost Springenberg, Manuel Blum and Frank Hutter -# | In *Advances in Neural Information Processing Systems 28*, 2015 -# | Available at https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning.pdf +A tutorial on how to get the datasets used in the paper introducing *Auto-sklearn* by Feurer et al.. + +Auto-sklearn website: https://automl.github.io/auto-sklearn/ + +Publication +~~~~~~~~~~~ + +| Efficient and Robust Automated Machine Learning +| Matthias Feurer, Aaron Klein, Katharina Eggensperger, Jost Springenberg, Manuel Blum and Frank Hutter +| In *Advances in Neural Information Processing Systems 28*, 2015 +| Available at https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning.pdf +""" # noqa F401 + +# License: BSD 3-Clause -# %% import pandas as pd import openml -# %% [markdown] +#################################################################################################### # List of dataset IDs given in the supplementary material of Feurer et al.: # https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning-supplemental.zip - -# %% +# fmt: off dataset_ids = [ 3, 6, 12, 14, 16, 18, 21, 22, 23, 24, 26, 28, 30, 31, 32, 36, 38, 44, 46, 57, 60, 179, 180, 181, 182, 184, 185, 273, 293, 300, 351, 354, 357, 389, @@ -34,8 +37,9 @@ 1056, 1067, 1068, 1069, 1111, 1112, 1114, 1116, 1119, 1120, 1128, 1130, 1134, 1138, 1139, 1142, 1146, 1161, 1166, ] +# fmt: on -# %% [markdown] +#################################################################################################### # The dataset IDs could be used directly to load the dataset and split the data into a training set # and a test set. However, to be reproducible, we will first obtain the respective tasks from # OpenML, which define both the target feature and the train/test split. @@ -48,16 +52,15 @@ # Please check the `OpenML documentation of tasks `_ if you # want to learn more about them. -# %% [markdown] +#################################################################################################### # This lists both active and inactive tasks (because of ``status='all'``). Unfortunately, # this is necessary as some of the datasets contain issues found after the publication and became # deactivated, which also deactivated the tasks on them. More information on active or inactive -# datasets can be found in the [online docs](https://docs.openml.org/#dataset-status). - -# %% +# datasets can be found in the `online docs `_. tasks = openml.tasks.list_tasks( task_type=openml.tasks.TaskType.SUPERVISED_CLASSIFICATION, status="all", + output_format="dataframe", ) # Query only those with holdout as the resampling startegy. @@ -65,7 +68,7 @@ task_ids = [] for did in dataset_ids: - tasks_ = list(tasks.query(f"did == {did}").tid) + tasks_ = list(tasks.query("did == {}".format(did)).tid) if len(tasks_) >= 1: # if there are multiple task, take the one with lowest ID (oldest). task_id = min(tasks_) else: @@ -88,5 +91,3 @@ # These are the tasks to work with: print(task_ids) - -# License: BSD 3-Clause diff --git a/examples/40_paper/2018_ida_strang_example.py b/examples/_external_or_deprecated/2018_ida_strang_example.py similarity index 76% rename from examples/40_paper/2018_ida_strang_example.py rename to examples/_external_or_deprecated/2018_ida_strang_example.py index 1a873a01c..8b225125b 100644 --- a/examples/40_paper/2018_ida_strang_example.py +++ b/examples/_external_or_deprecated/2018_ida_strang_example.py @@ -1,22 +1,26 @@ -# %% [markdown] -# # Strang et al. (2018) -# -# A tutorial on how to reproduce the analysis conducted for *Don't Rule Out Simple Models -# Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear Classifiers in OpenML*. -# -# ## Publication -# -# | Don't Rule Out Simple Models Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear Classifiers in OpenML -# | Benjamin Strang, Peter van der Putten, Jan N. van Rijn and Frank Hutter -# | In *Advances in Intelligent Data Analysis XVII 17th International Symposium*, 2018 -# | Available at https://link.springer.com/chapter/10.1007%2F978-3-030-01768-2_25 +""" +Strang et al. (2018) +==================== -# %% -import matplotlib.pyplot as plt +A tutorial on how to reproduce the analysis conducted for *Don't Rule Out Simple Models +Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear Classifiers in OpenML*. + +Publication +~~~~~~~~~~~ + +| Don't Rule Out Simple Models Prematurely: A Large Scale Benchmark Comparing Linear and Non-linear Classifiers in OpenML +| Benjamin Strang, Peter van der Putten, Jan N. van Rijn and Frank Hutter +| In *Advances in Intelligent Data Analysis XVII 17th International Symposium*, 2018 +| Available at https://link.springer.com/chapter/10.1007%2F978-3-030-01768-2_25 +""" +# License: BSD 3-Clause + +import matplotlib.pyplot as plt import openml +import pandas as pd -# %% [markdown] +############################################################################## # A basic step for each data-mining or machine learning task is to determine # which model to choose based on the problem and the data at hand. In this # work we investigate when non-linear classifiers outperform linear @@ -31,7 +35,6 @@ # more effort to distinguish the same flow with different hyperparameter # values. -# %% study_id = 123 # for comparing svms: flow_ids = [7754, 7756] # for comparing nns: flow_ids = [7722, 7729] @@ -44,17 +47,13 @@ # Downloads all evaluation records related to this study evaluations = openml.evaluations.list_evaluations( - measure, - size=None, - flows=flow_ids, - study=study_id, - output_format="dataframe", + measure, size=None, flows=flow_ids, study=study_id, output_format="dataframe" ) # gives us a table with columns data_id, flow1_value, flow2_value evaluations = evaluations.pivot(index="data_id", columns="flow_id", values="value").dropna() # downloads all data qualities (for scatter plot) data_qualities = openml.datasets.list_datasets( - data_id=list(evaluations.index.values), + data_id=list(evaluations.index.values), output_format="dataframe" ) # removes irrelevant data qualities data_qualities = data_qualities[meta_features] @@ -66,10 +65,10 @@ # adds column that indicates the difference between the two classifiers evaluations["diff"] = evaluations[flow_ids[0]] - evaluations[flow_ids[1]] -# %% [markdown] + +############################################################################## # makes the s-plot -# %% fig_splot, ax_splot = plt.subplots() ax_splot.plot(range(len(evaluations)), sorted(evaluations["diff"])) ax_splot.set_title(classifier_family) @@ -79,18 +78,18 @@ plt.show() -# %% [markdown] +############################################################################## # adds column that indicates the difference between the two classifiers, # needed for the scatter plot -# %% def determine_class(val_lin, val_nonlin): if val_lin < val_nonlin: return class_values[0] - if val_nonlin < val_lin: + elif val_nonlin < val_lin: return class_values[1] - return class_values[2] + else: + return class_values[2] evaluations["class"] = evaluations.apply( @@ -110,11 +109,10 @@ def determine_class(val_lin, val_nonlin): ax_scatter.set_yscale("log") plt.show() -# %% [markdown] +############################################################################## # makes a scatter plot where each data point represents the performance of the # two algorithms on various axis (not in the paper) -# %% fig_diagplot, ax_diagplot = plt.subplots() ax_diagplot.grid(linestyle="--") ax_diagplot.plot([0, 1], ls="-", color="black") @@ -124,4 +122,3 @@ def determine_class(val_lin, val_nonlin): ax_diagplot.set_xlabel(measure) ax_diagplot.set_ylabel(measure) plt.show() -# License: BSD 3-Clause diff --git a/examples/_external_or_deprecated/2018_kdd_rijn_example.py b/examples/_external_or_deprecated/2018_kdd_rijn_example.py new file mode 100644 index 000000000..6522013e3 --- /dev/null +++ b/examples/_external_or_deprecated/2018_kdd_rijn_example.py @@ -0,0 +1,188 @@ +""" +van Rijn and Hutter (2018) +========================== + +A tutorial on how to reproduce the paper *Hyperparameter Importance Across Datasets*. + +Example Deprecation Warning! +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This example is not supported anymore by the OpenML-Python developers. The example is kept for reference purposes but not tested anymore. + +Publication +~~~~~~~~~~~ + +| Hyperparameter importance across datasets +| Jan N. van Rijn and Frank Hutter +| In *Proceedings of the 24th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining*, 2018 +| Available at https://dl.acm.org/doi/10.1145/3219819.3220058 + +Requirements +~~~~~~~~~~~~ + +This is a Unix-only tutorial, as the requirements can not be satisfied on a Windows machine (Untested on other +systems). + +The following Python packages are required: + +pip install openml[examples,docs] fanova ConfigSpace<1.0 +""" + +# License: BSD 3-Clause + +import sys + +if sys.platform == "win32": # noqa + print( + "The pyrfr library (requirement of fanova) can currently not be installed on Windows systems" + ) + exit() + +# DEPRECATED EXAMPLE -- Avoid running this code in our CI/CD pipeline +print("This example is deprecated, remove the `if False` in this code to use it manually.") +if False: + import json + import fanova + import matplotlib.pyplot as plt + import pandas as pd + import seaborn as sns + + import openml + + + ############################################################################## + # With the advent of automated machine learning, automated hyperparameter + # optimization methods are by now routinely used in data mining. However, this + # progress is not yet matched by equal progress on automatic analyses that + # yield information beyond performance-optimizing hyperparameter settings. + # In this example, we aim to answer the following two questions: Given an + # algorithm, what are generally its most important hyperparameters? + # + # This work is carried out on the OpenML-100 benchmark suite, which can be + # obtained by ``openml.study.get_suite('OpenML100')``. In this example, we + # conduct the experiment on the Support Vector Machine (``flow_id=7707``) + # with specific kernel (we will perform a post-process filter operation for + # this). We should set some other experimental parameters (number of results + # per task, evaluation measure and the number of trees of the internal + # functional Anova) before the fun can begin. + # + # Note that we simplify the example in several ways: + # + # 1) We only consider numerical hyperparameters + # 2) We consider all hyperparameters that are numerical (in reality, some + # hyperparameters might be inactive (e.g., ``degree``) or irrelevant + # (e.g., ``random_state``) + # 3) We assume all hyperparameters to be on uniform scale + # + # Any difference in conclusion between the actual paper and the presented + # results is most likely due to one of these simplifications. For example, + # the hyperparameter C looks rather insignificant, whereas it is quite + # important when it is put on a log-scale. All these simplifications can be + # addressed by defining a ConfigSpace. For a more elaborated example that uses + # this, please see: + # https://github.com/janvanrijn/openml-pimp/blob/d0a14f3eb480f2a90008889f00041bdccc7b9265/examples/plot/plot_fanova_aggregates.py # noqa F401 + + suite = openml.study.get_suite("OpenML100") + flow_id = 7707 + parameter_filters = {"sklearn.svm.classes.SVC(17)_kernel": "sigmoid"} + evaluation_measure = "predictive_accuracy" + limit_per_task = 500 + limit_nr_tasks = 15 + n_trees = 16 + + fanova_results = [] + # we will obtain all results from OpenML per task. Practice has shown that this places the bottleneck on the + # communication with OpenML, and for iterated experimenting it is better to cache the results in a local file. + for idx, task_id in enumerate(suite.tasks): + if limit_nr_tasks is not None and idx >= limit_nr_tasks: + continue + print( + "Starting with task %d (%d/%d)" + % (task_id, idx + 1, len(suite.tasks) if limit_nr_tasks is None else limit_nr_tasks) + ) + # note that we explicitly only include tasks from the benchmark suite that was specified (as per the for-loop) + evals = openml.evaluations.list_evaluations_setups( + evaluation_measure, + flows=[flow_id], + tasks=[task_id], + size=limit_per_task, + output_format="dataframe", + ) + + performance_column = "value" + # make a DataFrame consisting of all hyperparameters (which is a dict in setup['parameters']) and the performance + # value (in setup['value']). The following line looks a bit complicated, but combines 2 tasks: a) combine + # hyperparameters and performance data in a single dict, b) cast hyperparameter values to the appropriate format + # Note that the ``json.loads(...)`` requires the content to be in JSON format, which is only the case for + # scikit-learn setups (and even there some legacy setups might violate this requirement). It will work for the + # setups that belong to the flows embedded in this example though. + try: + setups_evals = pd.DataFrame( + [ + dict( + **{name: json.loads(value) for name, value in setup["parameters"].items()}, + **{performance_column: setup[performance_column]} + ) + for _, setup in evals.iterrows() + ] + ) + except json.decoder.JSONDecodeError as e: + print("Task %d error: %s" % (task_id, e)) + continue + # apply our filters, to have only the setups that comply to the hyperparameters we want + for filter_key, filter_value in parameter_filters.items(): + setups_evals = setups_evals[setups_evals[filter_key] == filter_value] + # in this simplified example, we only display numerical and float hyperparameters. For categorical hyperparameters, + # the fanova library needs to be informed by using a configspace object. + setups_evals = setups_evals.select_dtypes(include=["int64", "float64"]) + # drop rows with unique values. These are by definition not an interesting hyperparameter, e.g., ``axis``, + # ``verbose``. + setups_evals = setups_evals[ + [ + c + for c in list(setups_evals) + if len(setups_evals[c].unique()) > 1 or c == performance_column + ] + ] + # We are done with processing ``setups_evals``. Note that we still might have some irrelevant hyperparameters, e.g., + # ``random_state``. We have dropped some relevant hyperparameters, i.e., several categoricals. Let's check it out: + + # determine x values to pass to fanova library + parameter_names = [ + pname for pname in setups_evals.columns.to_numpy() if pname != performance_column + ] + evaluator = fanova.fanova.fANOVA( + X=setups_evals[parameter_names].to_numpy(), + Y=setups_evals[performance_column].to_numpy(), + n_trees=n_trees, + ) + for idx, pname in enumerate(parameter_names): + try: + fanova_results.append( + { + "hyperparameter": pname.split(".")[-1], + "fanova": evaluator.quantify_importance([idx])[(idx,)]["individual importance"], + } + ) + except RuntimeError as e: + # functional ANOVA sometimes crashes with a RuntimeError, e.g., on tasks where the performance is constant + # for all configurations (there is no variance). We will skip these tasks (like the authors did in the + # paper). + print("Task %d error: %s" % (task_id, e)) + continue + + # transform ``fanova_results`` from a list of dicts into a DataFrame + fanova_results = pd.DataFrame(fanova_results) + + ############################################################################## + # make the boxplot of the variance contribution. Obviously, we can also use + # this data to make the Nemenyi plot, but this relies on the rather complex + # ``Orange`` dependency (``pip install Orange3``). For the complete example, + # the reader is referred to the more elaborate script (referred to earlier) + fig, ax = plt.subplots() + sns.boxplot(x="hyperparameter", y="fanova", data=fanova_results, ax=ax) + ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha="right") + ax.set_ylabel("Variance Contribution") + ax.set_xlabel(None) + plt.tight_layout() + plt.show() diff --git a/examples/40_paper/2018_neurips_perrone_example.py b/examples/_external_or_deprecated/2018_neurips_perrone_example.py similarity index 79% rename from examples/40_paper/2018_neurips_perrone_example.py rename to examples/_external_or_deprecated/2018_neurips_perrone_example.py index feb107cba..0d72846ac 100644 --- a/examples/40_paper/2018_neurips_perrone_example.py +++ b/examples/_external_or_deprecated/2018_neurips_perrone_example.py @@ -1,48 +1,49 @@ -# %% [markdown] -# # Perrone et al. (2018) -# -# A tutorial on how to build a surrogate model based on OpenML data as done for *Scalable -# Hyperparameter Transfer Learning* by Perrone et al.. -# -# ## Publication -# -# | Scalable Hyperparameter Transfer Learning -# | Valerio Perrone and Rodolphe Jenatton and Matthias Seeger and Cedric Archambeau -# | In *Advances in Neural Information Processing Systems 31*, 2018 -# | Available at https://papers.nips.cc/paper/7917-scalable-hyperparameter-transfer-learning.pdf -# -# This example demonstrates how OpenML runs can be used to construct a surrogate model. -# -# In the following section, we shall do the following: -# -# * Retrieve tasks and flows as used in the experiments by Perrone et al. (2018). -# * Build a tabular data by fetching the evaluations uploaded to OpenML. -# * Impute missing values and handle categorical data before building a Random Forest model that -# maps hyperparameter values to the area under curve score. +""" +Perrone et al. (2018) +===================== + +A tutorial on how to build a surrogate model based on OpenML data as done for *Scalable +Hyperparameter Transfer Learning* by Perrone et al.. + +Publication +~~~~~~~~~~~ + +| Scalable Hyperparameter Transfer Learning +| Valerio Perrone and Rodolphe Jenatton and Matthias Seeger and Cedric Archambeau +| In *Advances in Neural Information Processing Systems 31*, 2018 +| Available at https://papers.nips.cc/paper/7917-scalable-hyperparameter-transfer-learning.pdf + +This example demonstrates how OpenML runs can be used to construct a surrogate model. + +In the following section, we shall do the following: + +* Retrieve tasks and flows as used in the experiments by Perrone et al. (2018). +* Build a tabular data by fetching the evaluations uploaded to OpenML. +* Impute missing values and handle categorical data before building a Random Forest model that + maps hyperparameter values to the area under curve score. +""" + +############################################################################ +# License: BSD 3-Clause -# %% import openml import numpy as np import pandas as pd from matplotlib import pyplot as plt -from sklearn.compose import ColumnTransformer -from sklearn.ensemble import RandomForestRegressor +from sklearn.pipeline import Pipeline from sklearn.impute import SimpleImputer +from sklearn.compose import ColumnTransformer from sklearn.metrics import mean_squared_error -from sklearn.pipeline import Pipeline from sklearn.preprocessing import OneHotEncoder - -import openml +from sklearn.ensemble import RandomForestRegressor flow_type = "svm" # this example will use the smaller svm flow evaluations - -# %% [markdown] +############################################################################ # The subsequent functions are defined to fetch tasks, flows, evaluations and preprocess them into # a tabular format that can be used to build models. -# %% def fetch_evaluations(run_full=False, flow_type="svm", metric="area_under_roc_curve"): """ Fetch a list of evaluations based on the flows and tasks used in the experiments. @@ -93,6 +94,7 @@ def fetch_evaluations(run_full=False, flow_type="svm", metric="area_under_roc_cu tasks=task_ids, flows=[flow_id], uploaders=[2702], + output_format="dataframe", parameters_in_separate_columns=True, ) return eval_df, task_ids, flow_id @@ -152,26 +154,25 @@ def list_categorical_attributes(flow_type="svm"): return ["booster"] -# %% [markdown] +############################################################################# # Fetching the data from OpenML # ***************************** # Now, we read all the tasks and evaluations for them and collate into a table. # Here, we are reading all the tasks and evaluations for the SVM flow and # pre-processing all retrieved evaluations. -# %% eval_df, task_ids, flow_id = fetch_evaluations(run_full=False, flow_type=flow_type) X, y = create_table_from_evaluations(eval_df, flow_type=flow_type) print(X.head()) print("Y : ", y[:5]) -# %% [markdown] -# ## Creating pre-processing and modelling pipelines +############################################################################# +# Creating pre-processing and modelling pipelines +# *********************************************** # The two primary tasks are to impute the missing values, that is, account for the hyperparameters # that are not available with the runs from OpenML. And secondly, to handle categorical variables # using One-hot encoding prior to modelling. -# %% # Separating data into categorical and non-categorical (numeric for this example) columns cat_cols = list_categorical_attributes(flow_type=flow_type) num_cols = list(set(X.columns) - set(cat_cols)) @@ -180,18 +181,8 @@ def list_categorical_attributes(flow_type="svm"): num_imputer = SimpleImputer(missing_values=np.nan, strategy="constant", fill_value=-1) # Creating the one-hot encoder for numerical representation of categorical columns -enc = Pipeline( - [ - ( - "cat_si", - SimpleImputer( - strategy="constant", - fill_value="missing", - ), - ), - ("cat_ohe", OneHotEncoder(handle_unknown="ignore")), - ], -) +enc = OneHotEncoder(handle_unknown="ignore") + # Combining column transformers ct = ColumnTransformer([("cat", enc, cat_cols), ("num", num_imputer, num_cols)]) @@ -200,13 +191,13 @@ def list_categorical_attributes(flow_type="svm"): model = Pipeline(steps=[("preprocess", ct), ("surrogate", clf)]) -# %% [markdown] -# ## Building a surrogate model on a task's evaluation +############################################################################# +# Building a surrogate model on a task's evaluation +# ************************************************* # The same set of functions can be used for a single task to retrieve a singular table which can # be used for the surrogate model construction. We shall use the SVM flow here to keep execution # time simple and quick. -# %% # Selecting a task for the surrogate task_id = task_ids[-1] print("Task ID : ", task_id) @@ -215,10 +206,12 @@ def list_categorical_attributes(flow_type="svm"): model.fit(X, y) y_pred = model.predict(X) -print(f"Training RMSE : {mean_squared_error(y, y_pred):.5}") +print("Training RMSE : {:.5}".format(mean_squared_error(y, y_pred))) + -# %% [markdown] -# ## Evaluating the surrogate model +############################################################################# +# Evaluating the surrogate model +# ****************************** # The surrogate model built from a task's evaluations fetched from OpenML will be put into # trivial action here, where we shall randomly sample configurations and observe the trajectory # of the area under curve (auc) we can obtain from the surrogate we've built. @@ -226,7 +219,6 @@ def list_categorical_attributes(flow_type="svm"): # NOTE: This section is written exclusively for the SVM flow -# %% # Sampling random configurations def random_sample_configurations(num_samples=100): colnames = ["cost", "degree", "gamma", "kernel"] @@ -249,7 +241,7 @@ def random_sample_configurations(num_samples=100): configs = random_sample_configurations(num_samples=1000) print(configs) -# %% +############################################################################# preds = model.predict(configs) # tracking the maximum AUC obtained over the functions evaluations @@ -262,4 +254,3 @@ def random_sample_configurations(num_samples=100): plt.title("AUC regret for Random Search on surrogate") plt.xlabel("Numbe of function evaluations") plt.ylabel("Regret") -# License: BSD 3-Clause diff --git a/examples/_external_or_deprecated/README.md b/examples/_external_or_deprecated/README.md new file mode 100644 index 000000000..d25a81baa --- /dev/null +++ b/examples/_external_or_deprecated/README.md @@ -0,0 +1,5 @@ +# External or Deprecated Examples + +This directory contains examples that are either external or deprecated. They may not be maintained or updated +regularly, and their functionality might not align with the latest version of the library. Moreover, +they are not shown on the documentation website. \ No newline at end of file diff --git a/examples/30_extended/benchmark_with_optunahub.py b/examples/_external_or_deprecated/benchmark_with_optunahub.py similarity index 91% rename from examples/30_extended/benchmark_with_optunahub.py rename to examples/_external_or_deprecated/benchmark_with_optunahub.py index 67d106da3..ece3e7c40 100644 --- a/examples/30_extended/benchmark_with_optunahub.py +++ b/examples/_external_or_deprecated/benchmark_with_optunahub.py @@ -15,19 +15,30 @@ import logging import optuna - -import openml -from openml.extensions.sklearn import cat -from openml.extensions.sklearn import cont from sklearn.compose import ColumnTransformer from sklearn.ensemble import RandomForestClassifier from sklearn.impute import SimpleImputer from sklearn.pipeline import Pipeline from sklearn.preprocessing import OneHotEncoder +import openml logger = logging.Logger(name="Experiment Logger", level=1) +#
+#

Warning

+#

+# For the rest of this tutorial, we will require the `openml-sklearn` package. +# Install it with `pip install openml-sklearn`. +#

+#
+ +# %% +# Get sklearn extension to run sklearn models easily on OpenML tasks. +from openml_sklearn import SklearnExtension, cat, cont + +extension = SklearnExtension() + # Set your openml api key if you want to upload your results to OpenML (eg: # https://openml.org/search?type=run&sort=date) . To get one, simply make an # account (you don't need one for anything else, just to upload your results), diff --git a/examples/30_extended/fetch_runtimes_tutorial.py b/examples/_external_or_deprecated/fetch_runtimes_tutorial.py similarity index 100% rename from examples/30_extended/fetch_runtimes_tutorial.py rename to examples/_external_or_deprecated/fetch_runtimes_tutorial.py diff --git a/examples/30_extended/flow_id_tutorial.py b/examples/_external_or_deprecated/flow_id_tutorial.py similarity index 100% rename from examples/30_extended/flow_id_tutorial.py rename to examples/_external_or_deprecated/flow_id_tutorial.py diff --git a/examples/30_extended/flows_and_runs_tutorial.py b/examples/_external_or_deprecated/flows_and_runs_tutorial.py similarity index 100% rename from examples/30_extended/flows_and_runs_tutorial.py rename to examples/_external_or_deprecated/flows_and_runs_tutorial.py diff --git a/examples/30_extended/plot_svm_hyperparameters_tutorial.py b/examples/_external_or_deprecated/plot_svm_hyperparameters_tutorial.py similarity index 100% rename from examples/30_extended/plot_svm_hyperparameters_tutorial.py rename to examples/_external_or_deprecated/plot_svm_hyperparameters_tutorial.py diff --git a/examples/30_extended/run_setup_tutorial.py b/examples/_external_or_deprecated/run_setup_tutorial.py similarity index 100% rename from examples/30_extended/run_setup_tutorial.py rename to examples/_external_or_deprecated/run_setup_tutorial.py diff --git a/examples/30_extended/custom_flow_.py b/examples/_external_or_deprecated/upload_amlb_flows_and_runs.py similarity index 100% rename from examples/30_extended/custom_flow_.py rename to examples/_external_or_deprecated/upload_amlb_flows_and_runs.py diff --git a/examples/introduction.py b/examples/introduction.py new file mode 100644 index 000000000..630c72f9d --- /dev/null +++ b/examples/introduction.py @@ -0,0 +1,22 @@ +# %% [markdown] +# +# We provide a set of examples here to get started with OpenML-Python. These examples cover various aspects of using the +# OpenML API, including downloading datasets, uploading results, and working with tasks. +# +# ## Basics +# +# 1. [Installing and setting up OpenML-Python](../Basics/introduction_tutorial/) +# 2. [Downloading datasets](../Basics/simple_datasets_tutorial/) +# 3. [Using tasks](../Basics/simple_tasks_tutorial/) +# 3. [Uploading experiment results](../Basics/simple_flows_and_runs_tutorial/) +# 4. [Working with collections of tasks](../Basics/simple_suites_tutorial/) +# +# ## Advanced +# 1. [Getting splits for datasets from tasks](../Advanced/task_manual_iteration_tutorial/) +# 2. [Creating and uploading datasets](../Advanced/create_upload_tutorial/) +# 3. [Searching and editing datasets](../Advanced/datasets_tutorial/) +# 4. [Searching and creating tasks](../Advanced/task_tutorial/) +# 5. [Listing, downloading, and uploading suites](../Advanced/suites_tutorial/) +# 6. [Listing, downloading, and uploading studies](../Advanced/study_tutorial/) +# 7. [Downloading evaluation results](../Advanced/fetch_evaluations_tutorial/) +# 8. [Configuring logging](../Advanced/configure_logging/) diff --git a/examples/test_server_usage_warning.txt b/examples/test_server_usage_warning.txt deleted file mode 100644 index c551480b6..000000000 --- a/examples/test_server_usage_warning.txt +++ /dev/null @@ -1,3 +0,0 @@ -This example uploads data. For that reason, this example connects to the test server at test.openml.org. -This prevents the main server from crowding with example datasets, tasks, runs, and so on. -The use of this test server can affect behaviour and performance of the OpenML-Python API. diff --git a/mkdocs.yml b/mkdocs.yml index 38d9fe05a..92ba3c851 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -42,12 +42,29 @@ extra_css: nav: - index.md - - Code Reference: reference/ - - Examples: examples/ - - Usage: usage.md - - Contributing: contributing.md + - Examples: + - Overview: examples/introduction.py + - Basics: + - Setup: examples/Basics/introduction_tutorial.py + - Datasets: examples/Basics/simple_datasets_tutorial.py + - Tasks: examples/Basics/simple_tasks_tutorial.py + - Flows and Runs: examples/Basics/simple_flows_and_runs_tutorial.py + - Suites: examples/Basics/simple_suites_tutorial.py + - Advanced: + - Dataset Splits from Tasks: examples/Advanced/task_manual_iteration_tutorial.py + - Creating and Uploading Datasets: examples/Advanced/create_upload_tutorial.py + - Searching and Editing Datasets: examples/Advanced/datasets_tutorial.py + - Searching and Creating Tasks: examples/Advanced/tasks_tutorial.py + - List, Download, and Upload Suites: examples/Advanced/suites_tutorial.py + - List, Download, and Upload Studies: examples/Advanced/study_tutorial.py + - Downloading Evaluation Results: examples/Advanced/fetch_evaluations_tutorial.py + - Configuring Logging: examples/Advanced/configure_logging.py + + - Extensions: extensions.md - - Changelog: progress.md + - Advanced User Guide: details.md + - API: reference/ + - Contributing: contributing.md markdown_extensions: - pymdownx.highlight: diff --git a/openml/_api_calls.py b/openml/_api_calls.py index 3509f18e7..81296b3da 100644 --- a/openml/_api_calls.py +++ b/openml/_api_calls.py @@ -37,7 +37,7 @@ FILE_ELEMENTS_TYPE = Dict[str, Union[str, Tuple[str, str]]] DATABASE_CONNECTION_ERRCODE = 107 -API_TOKEN_HELP_LINK = "https://openml.github.io/openml-python/main/examples/20_basic/introduction_tutorial.html#authentication" # noqa: S105 +API_TOKEN_HELP_LINK = "https://openml.github.io/openml-python/latest/examples/Basics/introduction_tutorial/#authentication" # noqa: S105 def _robot_delay(n: int) -> float: @@ -519,7 +519,7 @@ def __parse_server_exception( msg = ( f"The API call {url} requires authentication via an API key.\nPlease configure " "OpenML-Python to use your API as described in this example:" - "\nhttps://openml.github.io/openml-python/main/examples/20_basic/introduction_tutorial.html#authentication" + "\nhttps://openml.github.io/openml-python/latest/examples/Basics/introduction_tutorial/#authentication" ) return OpenMLNotAuthorizedError(message=msg) diff --git a/scripts/gen_ref_pages.py b/scripts/gen_ref_pages.py index 730a98024..22a873a4a 100644 --- a/scripts/gen_ref_pages.py +++ b/scripts/gen_ref_pages.py @@ -5,8 +5,9 @@ """ +from __future__ import annotations + from pathlib import Path -import shutil import mkdocs_gen_files @@ -44,6 +45,8 @@ examples_dir = root / "examples" examples_doc_dir = root / "docs" / "examples" for path in sorted(examples_dir.rglob("*.py")): + if "_external_or_deprecated" in path.parts: + continue dest_path = Path("examples") / path.relative_to(examples_dir) with mkdocs_gen_files.open(dest_path, "w") as dest_file: print(path.read_text(), file=dest_file)