From 7c393ff6fa2bc484099efa2792234f4726163a30 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Wed, 26 Jul 2017 07:36:36 +0500 Subject: [PATCH 01/27] Add a cross-validation wrapper for optimizers --- src/mlpack/core/CMakeLists.txt | 1 + src/mlpack/core/hpt/CMakeLists.txt | 12 +++ src/mlpack/core/hpt/bind.hpp | 45 ++++++++ src/mlpack/core/hpt/cv_function.hpp | 130 +++++++++++++++++++++++ src/mlpack/core/hpt/cv_function_impl.hpp | 103 ++++++++++++++++++ src/mlpack/tests/CMakeLists.txt | 1 + src/mlpack/tests/hpt_test.cpp | 54 ++++++++++ 7 files changed, 346 insertions(+) create mode 100644 src/mlpack/core/hpt/CMakeLists.txt create mode 100644 src/mlpack/core/hpt/bind.hpp create mode 100644 src/mlpack/core/hpt/cv_function.hpp create mode 100644 src/mlpack/core/hpt/cv_function_impl.hpp create mode 100644 src/mlpack/tests/hpt_test.cpp diff --git a/src/mlpack/core/CMakeLists.txt b/src/mlpack/core/CMakeLists.txt index 6d9194cc2f7..bdc15a79b68 100644 --- a/src/mlpack/core/CMakeLists.txt +++ b/src/mlpack/core/CMakeLists.txt @@ -5,6 +5,7 @@ set(DIRS cv data dists + hpt kernels math metrics diff --git a/src/mlpack/core/hpt/CMakeLists.txt b/src/mlpack/core/hpt/CMakeLists.txt new file mode 100644 index 00000000000..a6efa9f458d --- /dev/null +++ b/src/mlpack/core/hpt/CMakeLists.txt @@ -0,0 +1,12 @@ +set(SOURCES + bind.hpp + cv_function.hpp + cv_function_impl.hpp +) + +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() + +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/hpt/bind.hpp b/src/mlpack/core/hpt/bind.hpp new file mode 100644 index 00000000000..a397f09d9df --- /dev/null +++ b/src/mlpack/core/hpt/bind.hpp @@ -0,0 +1,45 @@ +/** + * @file bind.hpp + * @author Kirill Mishchenko + * + * Facilities for supporting bound arguments. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_HPT_BIND_HPP +#define MLPACK_CORE_HPT_BIND_HPP + +#include + +#include + +namespace mlpack { +namespace hpt { + +/** + * A struct for storing information about a bound argument. Objects of this type + * are supposed to be passed into the CVFunction constructor. + * + * This struct is not meant to be used directly by users. Rather use the + * mlpack::hpt::Bind function. + * + * @tparam T The type of the bound argument. + * @tparam I The index of the bound argument. + */ +template +struct BoundArg +{ + //! The index of the bound argument. + static const size_t index = I; + + //! The value of the bound argument. + const T& value; +}; + +} // namespace hpt +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/hpt/cv_function.hpp b/src/mlpack/core/hpt/cv_function.hpp new file mode 100644 index 00000000000..3a4337184b3 --- /dev/null +++ b/src/mlpack/core/hpt/cv_function.hpp @@ -0,0 +1,130 @@ +/** + * @file cv_function.hpp + * @author Kirill Mishchenko + * + * A cross-validation wrapper for optimizers. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_HPT_CV_FUNCTION_HPP +#define MLPACK_CORE_HPT_CV_FUNCTION_HPP + +#include + +namespace mlpack { +namespace hpt { + +/** + * This wrapper serves for adapting the interface of the cross-validation + * classes to the one that can be utilized by the mlpack optimizers. + * + * This class is not supposed to be used directly by users. To tune + * hyper-parameters see HyperParameterTuner. + * + * @tparam CVType A cross-validation strategy. + * @tparam TotalArgs The total number of arguments that are supposed to be + * passed to the Evaluate method of a CVType object. + * @tparam BoundArgs Types of arguments (wrapped into the BoundArg struct) that + * should be passed into the Evaluate method of a CVType object but are not + * going to be passed into the Evaluate method of a CVFunction object. + */ +template +class CVFunction +{ + public: + /** + * Initialize a CVFunction object. + * + * @param cv A cross-validation object. + * @param BoundArgs Arguments that should be passed into the Evaluate method + * of the CVType object but are not going to be passed into the Evaluate + * method of this object. + */ + CVFunction(CVType& cv, const BoundArgs&... args); + + /** + * Run cross-validation with the bound and passed parameters. + * + * @param parameters Arguments (rather than the bound arguments) that should + * be passed into the Evaluate method of the CVType object. + */ + double Evaluate(const arma::mat& parameters); + + private: + //! The type of tuples of BoundArgs. + using BoundArgsTupleType = std::tuple; + + //! The amount of bound arguments. + static const size_t BoundArgsAmount = + std::tuple_size::value; + + /** + * A struct that finds out whether the next argument for the Evaluate method + * of a CVType object should be a bound argument at the position BAIndex + * rather than an element of parameters at the position PIndex. + */ + template + struct UseBoundArg; + + //! A reference to the cross-validation object. + CVType& cv; + + //! The bound arguments. + BoundArgsTupleType boundArgs; + + /** + * Collect all arguments and run cross-validation. + */ + template::type> + inline double Evaluate(const arma::mat& parameters, const Args&... args); + + /** + * Run cross-validation with the collected arguments. + */ + template::type, + typename = void> + inline double Evaluate(const arma::mat& parameters, const Args&... args); + + /** + * Put the bound argument (at the BAIndex position) as the next one. + */ + template::value>::type> + inline double PutNextArg(const arma::mat& parameters, const Args&... args); + + /** + * Put the element (at the PIndex position) of the parameters as the next one. + */ + template::value>::type, + typename = void> + inline double PutNextArg(const arma::mat& parameters, const Args&... args); +}; + + +} // namespace hpt +} // namespace mlpack + +// Include implementation +#include "cv_function_impl.hpp" + +#endif diff --git a/src/mlpack/core/hpt/cv_function_impl.hpp b/src/mlpack/core/hpt/cv_function_impl.hpp new file mode 100644 index 00000000000..0e02f238c41 --- /dev/null +++ b/src/mlpack/core/hpt/cv_function_impl.hpp @@ -0,0 +1,103 @@ +/** + * @file cv_function_impl.hpp + * @author Kirill Mishchenko + * + * The implementation of the class CVFunction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_HPT_CV_FUNCTION_IMPL_HPP +#define MLPACK_CORE_HPT_CV_FUNCTION_IMPL_HPP + +namespace mlpack { +namespace hpt { + +template +template +struct CVFunction::UseBoundArg< + BAIndex, PIndex, true> +{ + using BoundArgType = + typename std::tuple_element::type; + + static const bool value = BoundArgType::index == BAIndex + PIndex; +}; + +template +template +struct CVFunction::UseBoundArg< + BAIndex, PIndex, false> +{ + static const bool value = false; +}; + +template +CVFunction::CVFunction( + CVType& cv, const BoundArgs&... args) : cv(cv), boundArgs(args...) {} + +template +double CVFunction::Evaluate( + const arma::mat& parameters) +{ + return Evaluate<0, 0>(parameters); +} + +template +template +double CVFunction::Evaluate( + const arma::mat& parameters, + const Args&... args) +{ + return PutNextArg(parameters, args...); +} + +template +template +double CVFunction::Evaluate( + const arma::mat& /* parameters */, + const Args&... args) +{ + return cv.Evaluate(args...); +} + +template +template +double CVFunction::PutNextArg( + const arma::mat& parameters, + const Args&... args) +{ + return Evaluate( + parameters, args..., std::get(boundArgs).value); +} + +template +template +double CVFunction::PutNextArg( + const arma::mat& parameters, + const Args&... args) +{ + return Evaluate( + parameters, args..., parameters(PIndex, 0)); +} + +} // namespace hpt +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index d31aeed2259..3acefc542bf 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -32,6 +32,7 @@ add_executable(mlpack_test gradient_descent_test.cpp hmm_test.cpp hoeffding_tree_test.cpp + hpt_test.cpp hyperplane_test.cpp imputation_test.cpp init_rules_test.cpp diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp new file mode 100644 index 00000000000..6b2ad95419a --- /dev/null +++ b/src/mlpack/tests/hpt_test.cpp @@ -0,0 +1,54 @@ +/** + * @file hpt_test.cpp + * + * Tests for the hyper-parameter tuning module. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#include +#include +#include +#include +#include + +#include + +using namespace mlpack::cv; +using namespace mlpack::hpt; +using namespace mlpack::regression; + +BOOST_AUTO_TEST_SUITE(HPTTest); + +/** + * Test CVFunction runs cross-validation in according with specified bound + * arguments and passed parameters. + */ +BOOST_AUTO_TEST_CASE(CVFunctionTest) +{ + arma::mat xs = arma::randn(5, 100); + arma::vec beta = arma::randn(5, 1); + arma::mat ys = beta.t() * xs + 0.1 * arma::randn(5, 1); + + SimpleCV cv(0.2, xs, ys); + + bool transposeData = true; + bool useCholesky = false; + double lambda1 = 1.0; + double lambda2 = 2.0; + + BoundArg boundUseCholesky{useCholesky}; + BoundArg boundLambda1{lambda2}; + CVFunction, BoundArg> + cvFun(cv, boundUseCholesky, boundLambda1); + + double expected = cv.Evaluate(transposeData, useCholesky, lambda1, lambda2); + double actual = cvFun.Evaluate(arma::vec{double(transposeData), lambda1}); + + BOOST_REQUIRE_CLOSE(expected, actual, 1e-5); +} + +BOOST_AUTO_TEST_SUITE_END(); From f2d11fc0c4b23bc1fc9719ddaaa86550b1498428 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Wed, 26 Jul 2017 09:06:23 +0500 Subject: [PATCH 02/27] Add grid-search optimization --- src/mlpack/core/optimizers/CMakeLists.txt | 1 + .../optimizers/grid_search/CMakeLists.txt | 11 +++ .../optimizers/grid_search/grid_search.hpp | 93 +++++++++++++++++++ .../grid_search/grid_search_impl.hpp | 72 ++++++++++++++ src/mlpack/tests/hpt_test.cpp | 88 ++++++++++++++++++ 5 files changed, 265 insertions(+) create mode 100644 src/mlpack/core/optimizers/grid_search/CMakeLists.txt create mode 100644 src/mlpack/core/optimizers/grid_search/grid_search.hpp create mode 100644 src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp diff --git a/src/mlpack/core/optimizers/CMakeLists.txt b/src/mlpack/core/optimizers/CMakeLists.txt index 072fd27b32e..ce218252827 100644 --- a/src/mlpack/core/optimizers/CMakeLists.txt +++ b/src/mlpack/core/optimizers/CMakeLists.txt @@ -4,6 +4,7 @@ set(DIRS adam aug_lagrangian gradient_descent + grid_search lbfgs minibatch_sgd rmsprop diff --git a/src/mlpack/core/optimizers/grid_search/CMakeLists.txt b/src/mlpack/core/optimizers/grid_search/CMakeLists.txt new file mode 100644 index 00000000000..ecfd1aff8de --- /dev/null +++ b/src/mlpack/core/optimizers/grid_search/CMakeLists.txt @@ -0,0 +1,11 @@ +set(SOURCES + grid_search.hpp + grid_search_impl.hpp +) + +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() + +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/optimizers/grid_search/grid_search.hpp b/src/mlpack/core/optimizers/grid_search/grid_search.hpp new file mode 100644 index 00000000000..cda39b8e601 --- /dev/null +++ b/src/mlpack/core/optimizers/grid_search/grid_search.hpp @@ -0,0 +1,93 @@ +/** + * @file grid_search.hpp + * @author Kirill Mishchenko + * + * Grid-search optimization. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_GRID_SEARCH_GRID_SEARCH_HPP +#define MLPACK_CORE_OPTIMIZERS_GRID_SEARCH_GRID_SEARCH_HPP + +#include + +namespace mlpack { +namespace optimization { + +/** + * An optimizer that finds the minimum of a given function by iterating through + * points on a multidimensional grid. + * + * For GridSearch to work, a FunctionType template parameter is required. This + * class must implement the following function: + * + * double Evaluate(const arma::mat& coordinates); + */ +class GridSearch +{ + public: + /** + * Initialize a GridSearch object. + * + * @param collections Collections of values (one for each parameter). Each + * collection should be an STL-compatible container (it should provide + * begin() and end() methods returning iterators). + */ + template + GridSearch(const Collections&... collections); + + /** + * Optimize (minimize) the given function by iterating through the all + * possible combinations of values for the parameters. + */ + template + double Optimize(FunctionType& function, arma::mat& bestParameters); + + private: + //! Collections of parameter values (one for each parameter). + std::vector> parameterValueCollections; + + /** + * Iterate through the last (parameterValueCollections.size() - i) dimensions + * of the grid and change the arguments bestObjective and bestParameters if + * there is something better. The values for the first i dimensions + * (parameters) are specified in the first i rows of the currentParameters + * argument. + */ + template + void Optimize(FunctionType& function, + double& bestObjective, + arma::mat& bestParameters, + arma::vec& currentParameters, + size_t i); + + /** + * Convert each specified collection into a std::vector container and + * put results into parameterValueCollections. + */ + template + void InitParameterValueCollections(const Collection& collection, + const Collections&... collections) + { + parameterValueCollections.push_back( + std::vector(collection.begin(), collection.end())); + + InitParameterValueCollections(collections...); + } + + /** + * Finish initialization. + */ + void InitParameterValueCollections() {} +}; + +} // namespace optimization +} // namespace mlpack + +// Include implementation +#include "grid_search_impl.hpp" + +#endif diff --git a/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp b/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp new file mode 100644 index 00000000000..5d2e01ef44d --- /dev/null +++ b/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp @@ -0,0 +1,72 @@ +/** + * @file grid_search_impl.hpp + * @author Kirill Mishchenko + * + * Implementation of the grid-search optimization. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_GRID_SEARCH_GRID_SEARCH_IMPL_HPP +#define MLPACK_CORE_OPTIMIZERS_GRID_SEARCH_GRID_SEARCH_IMPL_HPP + +#include + +namespace mlpack { +namespace optimization { + +template +GridSearch::GridSearch(const Collections&... collections) +{ + InitParameterValueCollections(collections...); +} + +template +double GridSearch::Optimize(FunctionType& function, arma::mat& bestParameters) +{ + double bestObjective = std::numeric_limits::max(); + bestParameters = arma::mat(parameterValueCollections.size(), 1); + arma::vec currentParameters = arma::vec(parameterValueCollections.size()); + + /* Initialize best parameters for the case (very unlikely though) when no set + * of parameters gives an objective value better than + * std::numeric_limits::max() */ + for (size_t i = 0; i < parameterValueCollections.size(); ++i) + bestParameters(i, 0) = parameterValueCollections[i][0]; + + Optimize(function, bestObjective, bestParameters, currentParameters, 0); + + return bestObjective; +} + +template +void GridSearch::Optimize(FunctionType& function, + double& bestObjective, + arma::mat& bestParameters, + arma::vec& currentParameters, + size_t i) +{ + if (i < parameterValueCollections.size()) + for (double value : parameterValueCollections[i]) + { + currentParameters(i) = value; + Optimize(function, bestObjective, bestParameters, currentParameters, + i + 1); + } + else + { + double objective = function.Evaluate(currentParameters); + if (objective < bestObjective) + { + bestObjective = objective; + bestParameters = currentParameters; + } + } +} + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp index 6b2ad95419a..bf0a5882233 100644 --- a/src/mlpack/tests/hpt_test.cpp +++ b/src/mlpack/tests/hpt_test.cpp @@ -13,12 +13,14 @@ #include #include #include +#include #include #include using namespace mlpack::cv; using namespace mlpack::hpt; +using namespace mlpack::optimization; using namespace mlpack::regression; BOOST_AUTO_TEST_SUITE(HPTTest); @@ -51,4 +53,90 @@ BOOST_AUTO_TEST_CASE(CVFunctionTest) BOOST_REQUIRE_CLOSE(expected, actual, 1e-5); } +void InitProneToOverfittingData(arma::mat& xs, + arma::rowvec& ys, + double& validationSize) +{ + // Total number of data points. + size_t N = 10; + // Total number of features (all except the first one are redundant). + size_t M = 5; + + arma::rowvec data = arma::linspace(0.0, 10.0, N); + xs = data; + for (size_t i = 2; i <= M; ++i) + xs = arma::join_cols(xs, arma::pow(data, i)); + + // Responses that approximately follow the function y = 2 * x. Adding noise to + // avoid having a polynomial of degree 1 that exactly fits the points. + ys = 2 * data + 0.05 * arma::randn(1, N); + + validationSize = 0.3; +} + +template +void FindLARSBestLambdas(arma::mat& xs, + arma::rowvec& ys, + double& validationSize, + bool transposeData, + bool useCholesky, + const T1& lambda1Set, + const T2& lambda2Set, + double& bestLambda1, + double& bestLambda2, + double& bestObjective) +{ + SimpleCV cv(validationSize, xs, ys); + + bestObjective = std::numeric_limits::max(); + + for (double lambda1 : lambda1Set) + for (double lambda2 : lambda2Set) + { + double objective = + cv.Evaluate(transposeData, useCholesky, lambda1, lambda2); + if (objective < bestObjective) + { + bestObjective = objective; + bestLambda1 = lambda1; + bestLambda2 = lambda2; + } + } +} + + /** + * Test grid-search optimization leads to the best parameters from the specified + * ones. + */ +BOOST_AUTO_TEST_CASE(GridSearchTest) +{ + arma::mat xs; + arma::rowvec ys; + double validationSize; + InitProneToOverfittingData(xs, ys, validationSize); + + bool transposeData = true; + bool useCholesky = false; + arma::vec lambda1Set = + arma::join_rows(arma::vec{0}, arma::logspace(-3, 2, 6)); + std::array lambda2Set{{0.0, 0.05, 0.5, 5.0}}; + + double expectedLambda1, expectedLambda2, expectedObjective; + FindLARSBestLambdas(xs, ys, validationSize, transposeData, useCholesky, + lambda1Set, lambda2Set, expectedLambda1, expectedLambda2, + expectedObjective); + + SimpleCV cv(validationSize, xs, ys); + + GridSearch optimizer(lambda1Set, lambda2Set); + CVFunction, BoundArg> + cvFun(cv, {transposeData}, {useCholesky}); + arma::mat actualParameters; + double actualObjective = optimizer.Optimize(cvFun, actualParameters); + + BOOST_REQUIRE_CLOSE(expectedObjective, actualObjective, 1e-5); + BOOST_REQUIRE_CLOSE(expectedLambda1, actualParameters(0, 0), 1e-5); + BOOST_REQUIRE_CLOSE(expectedLambda2, actualParameters(1, 0), 1e-5); +} + BOOST_AUTO_TEST_SUITE_END(); From 15e35b4f878f74014fb7256a5d226363f1b67455 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Wed, 26 Jul 2017 12:14:08 +0500 Subject: [PATCH 03/27] Add hyper-parameter tuning --- src/mlpack/core/hpt/CMakeLists.txt | 3 + src/mlpack/core/hpt/bind.hpp | 66 +++++++ src/mlpack/core/hpt/deduce_hp_types.hpp | 89 +++++++++ src/mlpack/core/hpt/hpt.hpp | 246 ++++++++++++++++++++++++ src/mlpack/core/hpt/hpt_impl.hpp | 243 +++++++++++++++++++++++ src/mlpack/tests/hpt_test.cpp | 33 ++++ 6 files changed, 680 insertions(+) create mode 100644 src/mlpack/core/hpt/deduce_hp_types.hpp create mode 100644 src/mlpack/core/hpt/hpt.hpp create mode 100644 src/mlpack/core/hpt/hpt_impl.hpp diff --git a/src/mlpack/core/hpt/CMakeLists.txt b/src/mlpack/core/hpt/CMakeLists.txt index a6efa9f458d..e5758bfeced 100644 --- a/src/mlpack/core/hpt/CMakeLists.txt +++ b/src/mlpack/core/hpt/CMakeLists.txt @@ -2,6 +2,9 @@ set(SOURCES bind.hpp cv_function.hpp cv_function_impl.hpp + deduce_hp_types.hpp + hpt.hpp + hpt_impl.hpp ) set(DIR_SRCS) diff --git a/src/mlpack/core/hpt/bind.hpp b/src/mlpack/core/hpt/bind.hpp index a397f09d9df..24a87be796e 100644 --- a/src/mlpack/core/hpt/bind.hpp +++ b/src/mlpack/core/hpt/bind.hpp @@ -19,6 +19,25 @@ namespace mlpack { namespace hpt { +template +struct PreBoundArg; + +/** + * Mark the given argument as one that should be bound. It can be applied to + * arguments that are passed to the Optimize method of HyperParameterTuner. + * + * The implementation avoids data coping. If the passed argument is an l-value + * reference, we store it as a const l-value rerefence inside the returned + * PreBoundArg object. If the passed argument is an r-value reference, + * ligth-weight coping (by taking possesion of the r-value) will be made during + * the initialization of the returned PreBoundArg object. + */ +template +PreBoundArg Bind(T&& value) +{ + return PreBoundArg{std::forward(value)}; +} + /** * A struct for storing information about a bound argument. Objects of this type * are supposed to be passed into the CVFunction constructor. @@ -39,6 +58,53 @@ struct BoundArg const T& value; }; +/** + * A struct for marking arguments as ones that should be bound (it can be useful + * for the Optimize method of HyperParameterTuner). Arguments of this type are + * supposed to be converted into structs of the type BoundArg by adding + * information about argument positions. + * + * This struct is not meant to be used directly by users. Rather use the + * mlpack::hpt::Bind function. + */ +template +struct PreBoundArg +{ + using Type = T; + + const T value; +}; + +/** + * The specialization of the template for references. + * + * This struct is not meant to be used directly by users. Rather use the + * mlpack::hpt::Bind function. + */ +template +struct PreBoundArg +{ + using Type = T; + + const T& value; +}; + +/** + * A type function for checking whether the given type is PreBoundArg. + */ +template +class IsPreBoundArg +{ + template + struct Implementation : std::false_type {}; + + template + struct Implementation> : std::true_type {}; + + public: + static const bool value = Implementation::type>::value; +}; + } // namespace hpt } // namespace mlpack diff --git a/src/mlpack/core/hpt/deduce_hp_types.hpp b/src/mlpack/core/hpt/deduce_hp_types.hpp new file mode 100644 index 00000000000..1c8e32d6ca0 --- /dev/null +++ b/src/mlpack/core/hpt/deduce_hp_types.hpp @@ -0,0 +1,89 @@ +/** + * @file deduce_hp_types.hpp + * @author Kirill Mishchenko + * + * Tools to deduce types of hyper-parameters from types of arguments in the + * Optimize method in HyperParameterTuner. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_HPT_DEDUCE_HP_TYPES_HPP +#define MLPACK_CORE_HPT_DEDUCE_HP_TYPES_HPP + +#include + +namespace mlpack { +namespace hpt { + +/** + * A type function for deducing types of hyper-parameters from types of + * arguments in the Optimize method in HyperParameterTuner. + * + * We start by putting all types of the arguments into Args, and then process + * each of them one by one and put results into the internal struct + * ResultHolder. By the end Args become empty, while ResultHolder holds the + * tuple type of hyper-parameters. + * + * Here we declare and define DeduceHyperParameterTypes for the end phase when + * Args are empty (all argument types have been processed). + */ +template +struct DeduceHyperParameterTypes +{ + template + struct ResultHolder + { + using TupleType = std::tuple; + }; +}; + +/** + * Defining DeduceHyperParameterTypes for the case when not all argument types + * have been processed, and the next one is a collection type. + */ +template +struct DeduceHyperParameterTypes +{ + template + struct ResultHolder + { + using TupleType = typename DeduceHyperParameterTypes::template + ResultHolder::TupleType; + }; + + using TupleType = typename ResultHolder<>::TupleType; +}; + +/** + * Defining DeduceHyperParameterTypes for the case when not all argument types + * have been processed, and the next one is the type of an argument that should + * be bound. + */ +template +struct DeduceHyperParameterTypes, Args...> +{ + template + struct ResultHolder + { + using TupleType = typename DeduceHyperParameterTypes::template + ResultHolder::TupleType; + }; + + using TupleType = typename ResultHolder<>::TupleType; +}; + +/** + * A short alias for deducing types of hyper-parameters from types of arguments + * in the Optimize method in HyperParameterTuner. + */ +template +using TupleOfHyperParameters = + typename DeduceHyperParameterTypes::TupleType; + +} // namespace hpt +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/hpt/hpt.hpp b/src/mlpack/core/hpt/hpt.hpp new file mode 100644 index 00000000000..caf0506486a --- /dev/null +++ b/src/mlpack/core/hpt/hpt.hpp @@ -0,0 +1,246 @@ +/** + * @file hpt.hpp + * @author Kirill Mishchenko + * + * Hyper-parameter tuning. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_HPT_HPT_HPP +#define MLPACK_CORE_HPT_HPT_HPP + +#include +#include +#include + +namespace mlpack { +namespace hpt { + +/** + * The class HyperParameterTuner for the given MLAlgorithm utilizes the provided + * Optimizer to find the values of hyper-parameters that optimize the value of + * the given Metric. The value of the Metric is calculated by performing + * cross-validation with the provided cross-validation strategy. + * + * To construct a HyperParameterTuner object you need to pass the same arguments + * as for construction of an object of the given CV class. For example, we can + * use the following code to try to find a good lambda value for + * LinearRegression. + * + * @code + * // 100-point 5-dimensional random dataset. + * arma::mat data = arma::randu(5, 100); + * // Noisy responses retrieved by a random linear transformation of data. + * arma::rowvec responses = arma::randu(5) * data + + * 0.1 * arma::randn(100); + * + * // Using 80% of data for training and remaining 20% for assessing MSE. + * double validationSize = 0.2; + * HyperParamterTuner hpt(validationSize, data, + * responses); + * + * // Finding the best value for lambda from the values 0.0, 0.001, 0.01, 0.1, + * // and 1.0. + * arma::vec lambdas{0.0, 0.001, 0.01, 0.1, 1.0}; + * double bestLambda; + * std::tie(bestLambda) = hpt.Optimize(lambdas); + * @endcode + * + * When some hyper-parameters should not be optimized, you can specify values + * for them with the Bind function as in the following example of finding good + * lambda1 and lambda2 values for LARS. + * + * @code + * HyperParamterTuner hpt2(validationSize, data, + * responses); + * + * bool transposeData = true; + * bool useCholesky = false; + * arma::vec lambda1Set{0.0, 0.001, 0.01, 0.1, 1.0}; + * arma::vec lambda2Set{0.0, 0.002, 0.02, 0.2, 2.0}; + * + * double bestLambda1, bestLambda2; + * std::tie(bestLambda1, bestLambda2) = hpt2.Optimize(Bind(transposeData), + * Bind(useCholesky), lambda1Set, lambda2Set); + * @endcode + * + * @tparam MLAlgorithm A machine learning algorithm. + * @tparam Metric A metric to assess the quality of a trained model. + * @tparam CV A cross-validation strategy used to assess a set of + * hyper-parameters. + * @tparam Optimizer An optimization strategy. + * @tparam MatType The type of data. + * @tparam PredictionsType The type of predictions (should be passed when the + * predictions type is a template parameter in Train methods of the given + * MLAlgorithm; arma::Row will be used otherwise). + * @tparam WeightsType The type of weights (should be passed when weighted + * learning is supported, and the weights type is a template parameter in + * Train methods of the given MLAlgorithm; arma::vec will be used + * otherwise). + */ +template class CV, + typename Optimizer = mlpack::optimization::GridSearch, + typename MatType = arma::mat, + typename PredictionsType = + typename cv::MetaInfoExtractor::PredictionsType, + typename WeightsType = + typename cv::MetaInfoExtractor::WeightsType> +class HyperParameterTuner +{ + public: + /** + * Create a HyperParameterTuner object by passing constructor arguments for + * the given cross-validation strategy (the CV class). + * + * @param args Constructor arguments for the given cross-validation + * strategy (the CV class). + */ + template + HyperParameterTuner(const CVArgs& ...args); + + /** + * Find the best hyper-parameters by using the given Optimizer. For each + * hyper-parameter one of the following should be passed as an argument. + * 1. A set of values to choose from (when using GridSearch as an optimizer). + * The set of values should be an STL-compatible container (it should + * provide begin() and end() methods returning iterators). + * 2. A starting value (when using any other optimizer than GridSearch). + * 3. A value bound by using the function mlpack::hpt::Bind. In this case the + * hyper-parameter will not be optimized. + * + * All arguments should be passed in the same order as if the corresponding + * hyper-paramters would be passed into the Evaluate method of the given CV + * class (in the order as they appear in the constructor(s) of the given + * MLAlgorithm). + * + * The method returns a tuple of values for hyper-parameters that haven't been + * bound. + * + * @param args Arguments corresponding to hyper-parameters (see the method + * description for more information). + */ + template + TupleOfHyperParameters Optimize(const Args&... args); + + //! Access the performance measurement of the best model from the last run. + double BestObjective() const { return bestObjective; } + + private: + static_assert( + std::is_same::value, + "HyperParameterTuner now supports only the GridSearch optimizer"); + + //! A short alias for the full type of the cross-validation. + using CVType = CV; + + //! The cross-validation object for assessing sets of hyper-parameters. + CVType cv; + + //! The optimizer. + Optimizer optimizer; + + //! The best objective from the last run. + double bestObjective; + + /** + * The set of methods to initialize the GridSearh optimizer. We basically need + * to go through all arguments passed to the Optimize method, gather all + * non-bound arguments (collections) and pass them into the GridSearch + * constructor. + */ + template::value>> + inline void InitGridSearch(const ArgsTuple& args, + Collections... collections); + + template::value>, + typename = std::enable_if_t::type>::value>> + inline void InitGridSearch(const ArgsTuple& args, + Collections... collections); + + template::value>, + typename = std::enable_if_t::type>::value>, + typename = void> + inline void InitGridSearch(const ArgsTuple& args, + Collections... collections); + + /** + * The set of methods to initialize a cost function (CVFunction) object and + * run optimization to find the best hyper-parameters. To initialize a + * CVFunction object we go through all arguments passed to the Optimize + * method, gather all bound values and pass them into the CVFunction + * constructor. + */ + template::value>> + inline void InitCVFunctionAndOptimize(const ArgsTuple& args, + arma::mat& bestParams, + BoundArgs... boundArgs); + + template::value>, + typename = std::enable_if_t::type>::value>> + inline void InitCVFunctionAndOptimize(const ArgsTuple& args, + arma::mat& bestParams, + BoundArgs... boundArgs); + + template::value>, + typename = std::enable_if_t::type>::value>, + typename = void> + inline void InitCVFunctionAndOptimize(const ArgsTuple& args, + arma::mat& bestParams, + BoundArgs... boundArgs); + + /** + * The set of methods to convert the given arma::vec vector to the tuple of + * the target type by gathering all elements in an argument list. + */ + template::value>> + inline TupleType VectorToTuple(const arma::vec& vector, const Args&... args); + + template::value>, + typename = void> + inline TupleType VectorToTuple(const arma::vec& vector, const Args&... args); +}; + +} // namespace hpt +} // namespace mlpack + +// Include implementation +#include "hpt_impl.hpp" + +#endif diff --git a/src/mlpack/core/hpt/hpt_impl.hpp b/src/mlpack/core/hpt/hpt_impl.hpp new file mode 100644 index 00000000000..28559eb40dc --- /dev/null +++ b/src/mlpack/core/hpt/hpt_impl.hpp @@ -0,0 +1,243 @@ +/** + * @file hpt_impl.hpp + * @author Kirill Mishchenko + * + * Implementation of hyper-parameter tuning. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_HPT_HPT_IMPL_HPP +#define MLPACK_CORE_HPT_HPT_IMPL_HPP + +#include + +namespace mlpack { +namespace hpt { + +template class CV, + typename Optimizer, + typename MatType, + typename PredictionsType, + typename WeightsType> +template +HyperParameterTuner::HyperParameterTuner(const CVArgs&... args) : + cv(args...) {} + +template class CV, + typename Optimizer, + typename MatType, + typename PredictionsType, + typename WeightsType> +template +TupleOfHyperParameters HyperParameterTuner::Optimize( + const Args&... args) +{ + arma::mat bestParameters; + const auto argsTuple = std::tie(args...); + + InitGridSearch<0>(argsTuple); + InitCVFunctionAndOptimize<0>(argsTuple, bestParameters); + + return VectorToTuple, 0>(bestParameters); +} + +template class CV, + typename Optimizer, + typename MatType, + typename PredictionsType, + typename WeightsType> +template +void HyperParameterTuner::InitGridSearch( + const ArgsTuple& /* args */, + Collections... collections) +{ + optimizer = Optimizer(collections...); +} + +template class CV, + typename Optimizer, + typename MatType, + typename PredictionsType, + typename WeightsType> +template +void HyperParameterTuner::InitGridSearch( + const ArgsTuple& args, + Collections... collections) +{ + InitGridSearch(args, collections...); +} + +template class CV, + typename Optimizer, + typename MatType, + typename PredictionsType, + typename WeightsType> +template +void HyperParameterTuner::InitGridSearch( + const ArgsTuple& args, + Collections... collections) +{ + InitGridSearch(args, collections..., std::get(args)); +} + +template class CV, + typename Optimizer, + typename MatType, + typename PredictionsType, + typename WeightsType> +template +void HyperParameterTuner::InitCVFunctionAndOptimize( + const ArgsTuple& /* args */, + arma::mat& bestParams, + BoundArgs... boundArgs) +{ + static const size_t totalArgs = std::tuple_size::value; + + CVFunction cvFunction(cv, boundArgs...); + bestObjective = optimizer.Optimize(cvFunction, bestParams); +} + +template class CV, + typename Optimizer, + typename MatType, + typename PredictionsType, + typename WeightsType> +template +void HyperParameterTuner::InitCVFunctionAndOptimize( + const ArgsTuple& args, + arma::mat& bestParams, + BoundArgs... boundArgs) +{ + using PreBoundArgT = typename std::remove_reference< + typename std::tuple_element::type>::type; + using BoundArgT = BoundArg; + + InitCVFunctionAndOptimize(args, bestParams, boundArgs..., + BoundArgT{std::get(args).value}); +} + +template class CV, + typename Optimizer, + typename MatType, + typename PredictionsType, + typename WeightsType> +template +void HyperParameterTuner::InitCVFunctionAndOptimize( + const ArgsTuple& args, + arma::mat& bestParams, + BoundArgs... boundArgs) +{ + InitCVFunctionAndOptimize(args, bestParams, boundArgs...); +} + +template class CV, + typename Optimizer, + typename MatType, + typename PredictionsType, + typename WeightsType> +template +TupleType HyperParameterTuner::VectorToTuple( + const arma::vec& vector, const Args&... args) +{ + return VectorToTuple(vector, args..., vector(I)); +} + +template class CV, + typename Optimizer, + typename MatType, + typename PredictionsType, + typename WeightsType> +template +TupleType HyperParameterTuner::VectorToTuple( + const arma::vec& /* vector */, const Args&... args) +{ + return TupleType(args...); +} + +} // namespace hpt +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp index bf0a5882233..1166ced19dd 100644 --- a/src/mlpack/tests/hpt_test.cpp +++ b/src/mlpack/tests/hpt_test.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -139,4 +140,36 @@ BOOST_AUTO_TEST_CASE(GridSearchTest) BOOST_REQUIRE_CLOSE(expectedLambda2, actualParameters(1, 0), 1e-5); } +/** + * Test HyperParameterTuner. + */ +BOOST_AUTO_TEST_CASE(HPTTest) +{ + arma::mat xs; + arma::rowvec ys; + double validationSize; + InitProneToOverfittingData(xs, ys, validationSize); + + bool transposeData = true; + bool useCholesky = false; + arma::vec lambda1Set = + arma::join_rows(arma::vec{0}, arma::logspace(-3, 2, 6)); + arma::vec lambda2Set{0.0, 0.05, 0.5, 5.0}; + + double expectedLambda1, expectedLambda2, expectedObjective; + FindLARSBestLambdas(xs, ys, validationSize, transposeData, useCholesky, + lambda1Set, lambda2Set, expectedLambda1, expectedLambda2, + expectedObjective); + + double actualLambda1, actualLambda2; + HyperParameterTuner + hpt(validationSize, xs, ys); + std::tie(actualLambda1, actualLambda2) = hpt.Optimize(Bind(transposeData), + Bind(useCholesky), lambda1Set, lambda2Set); + + BOOST_REQUIRE_CLOSE(expectedObjective, hpt.BestObjective(), 1e-5); + BOOST_REQUIRE_CLOSE(expectedLambda1, actualLambda1, 1e-5); + BOOST_REQUIRE_CLOSE(expectedLambda2, actualLambda2, 1e-5); +} + BOOST_AUTO_TEST_SUITE_END(); From fa78d079d9832b40a9f5a49d7e79dfbc72bca5ae Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Wed, 26 Jul 2017 16:47:10 +0500 Subject: [PATCH 04/27] Add support for metrics that should be maximized --- src/mlpack/core/hpt/hpt.hpp | 18 +++++++++++++++- src/mlpack/core/hpt/hpt_impl.hpp | 4 +++- src/mlpack/tests/hpt_test.cpp | 36 ++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/hpt/hpt.hpp b/src/mlpack/core/hpt/hpt.hpp index caf0506486a..38f77e4dded 100644 --- a/src/mlpack/core/hpt/hpt.hpp +++ b/src/mlpack/core/hpt/hpt.hpp @@ -137,8 +137,24 @@ class HyperParameterTuner std::is_same::value, "HyperParameterTuner now supports only the GridSearch optimizer"); + /** + * A decorator that returns negated values of the original metric. + */ + template + struct Negated + { + static double Evaluate(MLAlgorithm& model, + const MatType& xs, + const PredictionsType& ys) + { return -OriginalMetric::Evaluate(model, xs, ys); } + }; + //! A short alias for the full type of the cross-validation. - using CVType = CV; + using CVType = typename std::conditional, + CV, MatType, PredictionsType, + WeightsType>>::type; + //! The cross-validation object for assessing sets of hyper-parameters. CVType cv; diff --git a/src/mlpack/core/hpt/hpt_impl.hpp b/src/mlpack/core/hpt/hpt_impl.hpp index 28559eb40dc..7b065c13da1 100644 --- a/src/mlpack/core/hpt/hpt_impl.hpp +++ b/src/mlpack/core/hpt/hpt_impl.hpp @@ -145,7 +145,9 @@ void HyperParameterTuner::value; CVFunction cvFunction(cv, boundArgs...); - bestObjective = optimizer.Optimize(cvFunction, bestParams); + bestObjective = Metric::NeedsMinimization? + optimizer.Optimize(cvFunction, bestParams) : + -optimizer.Optimize(cvFunction, bestParams); } template +#include #include #include #include #include #include #include +#include #include @@ -172,4 +174,38 @@ BOOST_AUTO_TEST_CASE(HPTTest) BOOST_REQUIRE_CLOSE(expectedLambda2, actualLambda2, 1e-5); } +/** + * Test HyperParamterTuner maximizes Accuracy rather than minimizes it. + */ +BOOST_AUTO_TEST_CASE(HPTMaximizationTest) +{ + // Initializing a linearly separable dataset. + arma::mat xs = arma::linspace(0.0, 10.0, 50); + arma::Row ys = arma::join_rows(arma::zeros>(25), + arma::ones>(25)); + + // We will train and validate on the same dataset. + double validationSize = 0.5; + arma::mat doubledXs = arma::join_rows(xs, xs); + arma::Row doubledYs = arma::join_rows(ys, ys); + + // Defining lambdas to choose from. Zero should be preferred since big lambdas + // are likely to restrict capabilities of logistic regression. + arma::vec lambdas{0.0, 1e12}; + + // Making sure that the assumption above is true. + SimpleCV, Accuracy> + cv(validationSize, doubledXs, doubledYs); + BOOST_REQUIRE_GT(cv.Evaluate(0.0), cv.Evaluate(1e12)); + + HyperParameterTuner, Accuracy, SimpleCV> + hpt(validationSize, doubledXs, doubledYs); + + double actualLambda; + std::tie(actualLambda) = hpt.Optimize(lambdas); + + BOOST_REQUIRE_CLOSE(hpt.BestObjective(), 1.0, 1e-5); + BOOST_REQUIRE_CLOSE(actualLambda, 0.0, 1e-5); +} + BOOST_AUTO_TEST_SUITE_END(); From 42d16eef62099a77d0c5d1c506174428fe81ef86 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Thu, 27 Jul 2017 13:33:15 +0500 Subject: [PATCH 05/27] Add interface for accessing the best model --- src/mlpack/core/hpt/cv_function.hpp | 13 +++++++++++++ src/mlpack/core/hpt/cv_function_impl.hpp | 19 +++++++++++++++++-- src/mlpack/core/hpt/hpt.hpp | 6 ++++++ src/mlpack/core/hpt/hpt_impl.hpp | 1 + src/mlpack/tests/hpt_test.cpp | 8 ++++++++ 5 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/hpt/cv_function.hpp b/src/mlpack/core/hpt/cv_function.hpp index 3a4337184b3..291d756c476 100644 --- a/src/mlpack/core/hpt/cv_function.hpp +++ b/src/mlpack/core/hpt/cv_function.hpp @@ -53,6 +53,13 @@ class CVFunction */ double Evaluate(const arma::mat& parameters); + //! The used machine learning algorithm. + using MLAlgorithm = typename + std::remove_reference().Model())>::type; + + //! Access and modify the best model so far. + MLAlgorithm& BestModel() { return bestModel; } + private: //! The type of tuples of BoundArgs. using BoundArgsTupleType = std::tuple; @@ -77,6 +84,12 @@ class CVFunction //! The bound arguments. BoundArgsTupleType boundArgs; + //! The best objective so far. + double bestObjective; + + //! The best model so far. + MLAlgorithm bestModel; + /** * Collect all arguments and run cross-validation. */ diff --git a/src/mlpack/core/hpt/cv_function_impl.hpp b/src/mlpack/core/hpt/cv_function_impl.hpp index 0e02f238c41..b2ab6a3ed6d 100644 --- a/src/mlpack/core/hpt/cv_function_impl.hpp +++ b/src/mlpack/core/hpt/cv_function_impl.hpp @@ -36,7 +36,11 @@ struct CVFunction::UseBoundArg< template CVFunction::CVFunction( - CVType& cv, const BoundArgs&... args) : cv(cv), boundArgs(args...) {} + CVType& cv, const BoundArgs&... args) : + cv(cv), + boundArgs(args...), + bestObjective(std::numeric_limits::max()) +{ /* Nothing left to do. */ } template double CVFunction::Evaluate( @@ -67,7 +71,18 @@ double CVFunction::Evaluate( const arma::mat& /* parameters */, const Args&... args) { - return cv.Evaluate(args...); + double objective = cv.Evaluate(args...); + + // Change the best model if we have got a better score, or if we probably + // have not assigned any valid (trained) model yet. + if (bestObjective > objective || + bestObjective == std::numeric_limits::max()) + { + bestObjective = objective; + bestModel = std::move(cv.Model()); + } + + return objective; } template diff --git a/src/mlpack/core/hpt/hpt.hpp b/src/mlpack/core/hpt/hpt.hpp index 38f77e4dded..d495b2d1eb6 100644 --- a/src/mlpack/core/hpt/hpt.hpp +++ b/src/mlpack/core/hpt/hpt.hpp @@ -132,6 +132,9 @@ class HyperParameterTuner //! Access the performance measurement of the best model from the last run. double BestObjective() const { return bestObjective; } + //! Access and modify the best model from the last run. + MLAlgorithm& BestModel() { return bestModel; } + private: static_assert( std::is_same::value, @@ -165,6 +168,9 @@ class HyperParameterTuner //! The best objective from the last run. double bestObjective; + //! The best model from the last run. + MLAlgorithm bestModel; + /** * The set of methods to initialize the GridSearh optimizer. We basically need * to go through all arguments passed to the Optimize method, gather all diff --git a/src/mlpack/core/hpt/hpt_impl.hpp b/src/mlpack/core/hpt/hpt_impl.hpp index 7b065c13da1..6473ee54e23 100644 --- a/src/mlpack/core/hpt/hpt_impl.hpp +++ b/src/mlpack/core/hpt/hpt_impl.hpp @@ -148,6 +148,7 @@ void HyperParameterTuner Date: Thu, 27 Jul 2017 13:40:01 +0500 Subject: [PATCH 06/27] Fix lambda sets initialization --- src/mlpack/tests/hpt_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp index 8166678d09b..1690bc25d5b 100644 --- a/src/mlpack/tests/hpt_test.cpp +++ b/src/mlpack/tests/hpt_test.cpp @@ -121,7 +121,7 @@ BOOST_AUTO_TEST_CASE(GridSearchTest) bool transposeData = true; bool useCholesky = false; arma::vec lambda1Set = - arma::join_rows(arma::vec{0}, arma::logspace(-3, 2, 6)); + arma::join_cols(arma::vec{0}, arma::logspace(-3, 2, 6)); std::array lambda2Set{{0.0, 0.05, 0.5, 5.0}}; double expectedLambda1, expectedLambda2, expectedObjective; @@ -155,7 +155,7 @@ BOOST_AUTO_TEST_CASE(HPTTest) bool transposeData = true; bool useCholesky = false; arma::vec lambda1Set = - arma::join_rows(arma::vec{0}, arma::logspace(-3, 2, 6)); + arma::join_cols(arma::vec{0}, arma::logspace(-3, 2, 6)); arma::vec lambda2Set{0.0, 0.05, 0.5, 5.0}; double expectedLambda1, expectedLambda2, expectedObjective; From dd1db7c9029551438cf0613259eccffaadf2d361 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Mon, 7 Aug 2017 15:22:00 +0500 Subject: [PATCH 07/27] Fix misspellings --- src/mlpack/core/hpt/bind.hpp | 2 +- src/mlpack/core/hpt/hpt.hpp | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mlpack/core/hpt/bind.hpp b/src/mlpack/core/hpt/bind.hpp index 24a87be796e..00803b6f713 100644 --- a/src/mlpack/core/hpt/bind.hpp +++ b/src/mlpack/core/hpt/bind.hpp @@ -26,7 +26,7 @@ struct PreBoundArg; * Mark the given argument as one that should be bound. It can be applied to * arguments that are passed to the Optimize method of HyperParameterTuner. * - * The implementation avoids data coping. If the passed argument is an l-value + * The implementation avoids data copying. If the passed argument is an l-value * reference, we store it as a const l-value rerefence inside the returned * PreBoundArg object. If the passed argument is an r-value reference, * ligth-weight coping (by taking possesion of the r-value) will be made during diff --git a/src/mlpack/core/hpt/hpt.hpp b/src/mlpack/core/hpt/hpt.hpp index d495b2d1eb6..24f45ca724b 100644 --- a/src/mlpack/core/hpt/hpt.hpp +++ b/src/mlpack/core/hpt/hpt.hpp @@ -39,8 +39,8 @@ namespace hpt { * * // Using 80% of data for training and remaining 20% for assessing MSE. * double validationSize = 0.2; - * HyperParamterTuner hpt(validationSize, data, - * responses); + * HyperParameterTuner hpt(validationSize, + * data, responses); * * // Finding the best value for lambda from the values 0.0, 0.001, 0.01, 0.1, * // and 1.0. @@ -54,7 +54,7 @@ namespace hpt { * lambda1 and lambda2 values for LARS. * * @code - * HyperParamterTuner hpt2(validationSize, data, + * HyperParameterTuner hpt2(validationSize, data, * responses); * * bool transposeData = true; @@ -172,8 +172,8 @@ class HyperParameterTuner MLAlgorithm bestModel; /** - * The set of methods to initialize the GridSearh optimizer. We basically need - * to go through all arguments passed to the Optimize method, gather all + * The set of methods to initialize the GridSearch optimizer. We basically + * need to go through all arguments passed to the Optimize method, gather all * non-bound arguments (collections) and pass them into the GridSearch * constructor. */ From 92d75a129b2995b5acaaa5783a6e4fe6b8671ac8 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Mon, 7 Aug 2017 15:47:54 +0500 Subject: [PATCH 08/27] Fix CVFunctionTest --- src/mlpack/tests/hpt_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp index 1690bc25d5b..96287b076fc 100644 --- a/src/mlpack/tests/hpt_test.cpp +++ b/src/mlpack/tests/hpt_test.cpp @@ -36,7 +36,7 @@ BOOST_AUTO_TEST_CASE(CVFunctionTest) { arma::mat xs = arma::randn(5, 100); arma::vec beta = arma::randn(5, 1); - arma::mat ys = beta.t() * xs + 0.1 * arma::randn(5, 1); + arma::rowvec ys = beta.t() * xs + 0.1 * arma::randn(1, 100); SimpleCV cv(0.2, xs, ys); From 0b7690c5710cbce3ea25f2f6df8061648c28d53f Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Mon, 7 Aug 2017 16:15:31 +0500 Subject: [PATCH 09/27] Fix style --- src/mlpack/tests/hpt_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp index 96287b076fc..e90b5d92e56 100644 --- a/src/mlpack/tests/hpt_test.cpp +++ b/src/mlpack/tests/hpt_test.cpp @@ -203,7 +203,7 @@ BOOST_AUTO_TEST_CASE(HPTMaximizationTest) // Making sure that the assumption above is true. SimpleCV, Accuracy> - cv(validationSize, doubledXs, doubledYs); + cv(validationSize, doubledXs, doubledYs); BOOST_REQUIRE_GT(cv.Evaluate(0.0), cv.Evaluate(1e12)); HyperParameterTuner, Accuracy, SimpleCV> From 5eb3dc7c3c4afca682d8bea9efe9f6be98223db0 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Wed, 9 Aug 2017 12:56:12 +0500 Subject: [PATCH 10/27] Extend documentation --- src/mlpack/core/hpt/hpt.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mlpack/core/hpt/hpt.hpp b/src/mlpack/core/hpt/hpt.hpp index 24f45ca724b..9f0dd8d92a4 100644 --- a/src/mlpack/core/hpt/hpt.hpp +++ b/src/mlpack/core/hpt/hpt.hpp @@ -118,7 +118,9 @@ class HyperParameterTuner * All arguments should be passed in the same order as if the corresponding * hyper-paramters would be passed into the Evaluate method of the given CV * class (in the order as they appear in the constructor(s) of the given - * MLAlgorithm). + * MLAlgorithm). Also, arguments for all required hyper-parameters (ones that + * don't have default values in the corresponding MLAlgorithm constructor) + * should be provided. * * The method returns a tuple of values for hyper-parameters that haven't been * bound. From 56c2ff103a4590faed1932d2b47b5400ac1cfe80 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Wed, 9 Aug 2017 12:58:06 +0500 Subject: [PATCH 11/27] Fix typo --- src/mlpack/core/hpt/hpt.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/hpt/hpt.hpp b/src/mlpack/core/hpt/hpt.hpp index 9f0dd8d92a4..92215cee3b9 100644 --- a/src/mlpack/core/hpt/hpt.hpp +++ b/src/mlpack/core/hpt/hpt.hpp @@ -116,7 +116,7 @@ class HyperParameterTuner * hyper-parameter will not be optimized. * * All arguments should be passed in the same order as if the corresponding - * hyper-paramters would be passed into the Evaluate method of the given CV + * hyper-parameters would be passed into the Evaluate method of the given CV * class (in the order as they appear in the constructor(s) of the given * MLAlgorithm). Also, arguments for all required hyper-parameters (ones that * don't have default values in the corresponding MLAlgorithm constructor) From c6f897f05dd22cbe2b649b7a11a37c7f5b7d9693 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Thu, 10 Aug 2017 08:26:21 +0500 Subject: [PATCH 12/27] Make optimal lambdas in the "middle" of the space --- src/mlpack/tests/hpt_test.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp index e90b5d92e56..5e30e56aee2 100644 --- a/src/mlpack/tests/hpt_test.cpp +++ b/src/mlpack/tests/hpt_test.cpp @@ -60,6 +60,9 @@ void InitProneToOverfittingData(arma::mat& xs, arma::rowvec& ys, double& validationSize) { + // Making the function generate the same data for all callers. + arma::arma_rng::set_seed(11); + // Total number of data points. size_t N = 10; // Total number of features (all except the first one are redundant). @@ -129,6 +132,10 @@ BOOST_AUTO_TEST_CASE(GridSearchTest) lambda1Set, lambda2Set, expectedLambda1, expectedLambda2, expectedObjective); + // We should get these values (it has been found empirically). + BOOST_REQUIRE_CLOSE(expectedLambda1, 0.01, 1e-5); + BOOST_REQUIRE_CLOSE(expectedLambda2, 0.05, 1e-5); + SimpleCV cv(validationSize, xs, ys); GridSearch optimizer(lambda1Set, lambda2Set); @@ -163,6 +170,10 @@ BOOST_AUTO_TEST_CASE(HPTTest) lambda1Set, lambda2Set, expectedLambda1, expectedLambda2, expectedObjective); + // We should get these values (it has been found empirically). + BOOST_REQUIRE_CLOSE(expectedLambda1, 0.01, 1e-5); + BOOST_REQUIRE_CLOSE(expectedLambda2, 0.05, 1e-5); + double actualLambda1, actualLambda2; HyperParameterTuner hpt(validationSize, xs, ys); From 54ac86798bd6ed5222b3591ee74305e6aaaba084 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Thu, 10 Aug 2017 10:57:34 +0500 Subject: [PATCH 13/27] Add passing MLAlgorithm type to CVFunction --- src/mlpack/core/hpt/cv_function.hpp | 10 ++--- src/mlpack/core/hpt/cv_function_impl.hpp | 56 +++++++++++++++++------- src/mlpack/core/hpt/hpt_impl.hpp | 3 +- src/mlpack/tests/hpt_test.cpp | 4 +- 4 files changed, 49 insertions(+), 24 deletions(-) diff --git a/src/mlpack/core/hpt/cv_function.hpp b/src/mlpack/core/hpt/cv_function.hpp index 291d756c476..8a6638d3650 100644 --- a/src/mlpack/core/hpt/cv_function.hpp +++ b/src/mlpack/core/hpt/cv_function.hpp @@ -25,13 +25,17 @@ namespace hpt { * hyper-parameters see HyperParameterTuner. * * @tparam CVType A cross-validation strategy. + * @tparam MLAlgorithm The machine learning algorithm used in cross-validation. * @tparam TotalArgs The total number of arguments that are supposed to be * passed to the Evaluate method of a CVType object. * @tparam BoundArgs Types of arguments (wrapped into the BoundArg struct) that * should be passed into the Evaluate method of a CVType object but are not * going to be passed into the Evaluate method of a CVFunction object. */ -template +template class CVFunction { public: @@ -53,10 +57,6 @@ class CVFunction */ double Evaluate(const arma::mat& parameters); - //! The used machine learning algorithm. - using MLAlgorithm = typename - std::remove_reference().Model())>::type; - //! Access and modify the best model so far. MLAlgorithm& BestModel() { return bestModel; } diff --git a/src/mlpack/core/hpt/cv_function_impl.hpp b/src/mlpack/core/hpt/cv_function_impl.hpp index b2ab6a3ed6d..a4da8d5247c 100644 --- a/src/mlpack/core/hpt/cv_function_impl.hpp +++ b/src/mlpack/core/hpt/cv_function_impl.hpp @@ -15,9 +15,12 @@ namespace mlpack { namespace hpt { -template +template template -struct CVFunction::UseBoundArg< +struct CVFunction::UseBoundArg< BAIndex, PIndex, true> { using BoundArgType = @@ -26,48 +29,63 @@ struct CVFunction::UseBoundArg< static const bool value = BoundArgType::index == BAIndex + PIndex; }; -template +template template -struct CVFunction::UseBoundArg< +struct CVFunction::UseBoundArg< BAIndex, PIndex, false> { static const bool value = false; }; -template -CVFunction::CVFunction( +template +CVFunction::CVFunction( CVType& cv, const BoundArgs&... args) : cv(cv), boundArgs(args...), bestObjective(std::numeric_limits::max()) { /* Nothing left to do. */ } -template -double CVFunction::Evaluate( +template +double CVFunction::Evaluate( const arma::mat& parameters) { return Evaluate<0, 0>(parameters); } -template +template template -double CVFunction::Evaluate( +double CVFunction::Evaluate( const arma::mat& parameters, const Args&... args) { return PutNextArg(parameters, args...); } -template +template template -double CVFunction::Evaluate( +double CVFunction::Evaluate( const arma::mat& /* parameters */, const Args&... args) { @@ -85,12 +103,15 @@ double CVFunction::Evaluate( return objective; } -template +template template -double CVFunction::PutNextArg( +double CVFunction::PutNextArg( const arma::mat& parameters, const Args&... args) { @@ -98,13 +119,16 @@ double CVFunction::PutNextArg( parameters, args..., std::get(boundArgs).value); } -template +template template -double CVFunction::PutNextArg( +double CVFunction::PutNextArg( const arma::mat& parameters, const Args&... args) { diff --git a/src/mlpack/core/hpt/hpt_impl.hpp b/src/mlpack/core/hpt/hpt_impl.hpp index 6473ee54e23..77869a275a0 100644 --- a/src/mlpack/core/hpt/hpt_impl.hpp +++ b/src/mlpack/core/hpt/hpt_impl.hpp @@ -144,7 +144,8 @@ void HyperParameterTuner::value; - CVFunction cvFunction(cv, boundArgs...); + CVFunction + cvFunction(cv, boundArgs...); bestObjective = Metric::NeedsMinimization? optimizer.Optimize(cvFunction, bestParams) : -optimizer.Optimize(cvFunction, bestParams); diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp index 5e30e56aee2..563a07f4467 100644 --- a/src/mlpack/tests/hpt_test.cpp +++ b/src/mlpack/tests/hpt_test.cpp @@ -47,7 +47,7 @@ BOOST_AUTO_TEST_CASE(CVFunctionTest) BoundArg boundUseCholesky{useCholesky}; BoundArg boundLambda1{lambda2}; - CVFunction, BoundArg> + CVFunction, BoundArg> cvFun(cv, boundUseCholesky, boundLambda1); double expected = cv.Evaluate(transposeData, useCholesky, lambda1, lambda2); @@ -139,7 +139,7 @@ BOOST_AUTO_TEST_CASE(GridSearchTest) SimpleCV cv(validationSize, xs, ys); GridSearch optimizer(lambda1Set, lambda2Set); - CVFunction, BoundArg> + CVFunction, BoundArg> cvFun(cv, {transposeData}, {useCholesky}); arma::mat actualParameters; double actualObjective = optimizer.Optimize(cvFun, actualParameters); From 46e8c2519fd418ec342871c8ee18e28a08986d79 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Thu, 10 Aug 2017 11:43:56 +0500 Subject: [PATCH 14/27] Add braces around multiline statement --- src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp b/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp index 5d2e01ef44d..e6e704ad3fd 100644 --- a/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp +++ b/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp @@ -49,12 +49,14 @@ void GridSearch::Optimize(FunctionType& function, size_t i) { if (i < parameterValueCollections.size()) + { for (double value : parameterValueCollections[i]) { currentParameters(i) = value; Optimize(function, bestObjective, bestParameters, currentParameters, i + 1); } + } else { double objective = function.Evaluate(currentParameters); From 7b9604c7fab09de6538bc4aae83dd30684508a08 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Thu, 10 Aug 2017 12:00:21 +0500 Subject: [PATCH 15/27] Use more verbose names for BAIndex and PIndex --- src/mlpack/core/hpt/cv_function.hpp | 43 ++++++++++++------------ src/mlpack/core/hpt/cv_function_impl.hpp | 38 ++++++++++----------- 2 files changed, 41 insertions(+), 40 deletions(-) diff --git a/src/mlpack/core/hpt/cv_function.hpp b/src/mlpack/core/hpt/cv_function.hpp index 8a6638d3650..cfabcb28f2e 100644 --- a/src/mlpack/core/hpt/cv_function.hpp +++ b/src/mlpack/core/hpt/cv_function.hpp @@ -70,12 +70,12 @@ class CVFunction /** * A struct that finds out whether the next argument for the Evaluate method - * of a CVType object should be a bound argument at the position BAIndex - * rather than an element of parameters at the position PIndex. + * of a CVType object should be a bound argument at the position BoundArgIndex + * rather than an element of parameters at the position ParamIndex. */ - template + template struct UseBoundArg; //! A reference to the cross-validation object. @@ -93,42 +93,43 @@ class CVFunction /** * Collect all arguments and run cross-validation. */ - template::type> + typename = typename + std::enable_if::type> inline double Evaluate(const arma::mat& parameters, const Args&... args); /** * Run cross-validation with the collected arguments. */ - template::type, + typename = typename + std::enable_if::type, typename = void> inline double Evaluate(const arma::mat& parameters, const Args&... args); /** - * Put the bound argument (at the BAIndex position) as the next one. + * Put the bound argument (at the BoundArgIndex position) as the next one. */ - template::value>::type> + UseBoundArg::value>::type> inline double PutNextArg(const arma::mat& parameters, const Args&... args); /** - * Put the element (at the PIndex position) of the parameters as the next one. + * Put the element (at the ParamIndex position) of the parameters as the next + * one. */ - template::value>::type, + !UseBoundArg::value>::type, typename = void> inline double PutNextArg(const arma::mat& parameters, const Args&... args); }; diff --git a/src/mlpack/core/hpt/cv_function_impl.hpp b/src/mlpack/core/hpt/cv_function_impl.hpp index a4da8d5247c..5f854a10820 100644 --- a/src/mlpack/core/hpt/cv_function_impl.hpp +++ b/src/mlpack/core/hpt/cv_function_impl.hpp @@ -19,23 +19,23 @@ template -template +template struct CVFunction::UseBoundArg< - BAIndex, PIndex, true> + BoundArgIndex, ParamIndex, true> { using BoundArgType = - typename std::tuple_element::type; + typename std::tuple_element::type; - static const bool value = BoundArgType::index == BAIndex + PIndex; + static const bool value = BoundArgType::index == BoundArgIndex + ParamIndex; }; template -template +template struct CVFunction::UseBoundArg< - BAIndex, PIndex, false> + BoundArgIndex, ParamIndex, false> { static const bool value = false; }; @@ -65,23 +65,23 @@ template -template double CVFunction::Evaluate( const arma::mat& parameters, const Args&... args) { - return PutNextArg(parameters, args...); + return PutNextArg(parameters, args...); } template -template @@ -107,24 +107,24 @@ template -template double CVFunction::PutNextArg( const arma::mat& parameters, const Args&... args) { - return Evaluate( - parameters, args..., std::get(boundArgs).value); + return Evaluate( + parameters, args..., std::get(boundArgs).value); } template -template @@ -132,8 +132,8 @@ double CVFunction::PutNextArg( const arma::mat& parameters, const Args&... args) { - return Evaluate( - parameters, args..., parameters(PIndex, 0)); + return Evaluate( + parameters, args..., parameters(ParamIndex, 0)); } } // namespace hpt From cce56f5eef727db6de4d320197158efb092e7869 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Thu, 10 Aug 2017 15:26:38 +0500 Subject: [PATCH 16/27] Use the word "fixed" for "bound" --- src/mlpack/core/hpt/CMakeLists.txt | 2 +- src/mlpack/core/hpt/deduce_hp_types.hpp | 6 +-- src/mlpack/core/hpt/{bind.hpp => fixed.hpp} | 52 ++++++++++----------- src/mlpack/core/hpt/hpt.hpp | 34 +++++++------- src/mlpack/core/hpt/hpt_impl.hpp | 26 +++++------ src/mlpack/tests/hpt_test.cpp | 18 +++---- 6 files changed, 69 insertions(+), 69 deletions(-) rename src/mlpack/core/hpt/{bind.hpp => fixed.hpp} (62%) diff --git a/src/mlpack/core/hpt/CMakeLists.txt b/src/mlpack/core/hpt/CMakeLists.txt index e5758bfeced..c39f9b674e3 100644 --- a/src/mlpack/core/hpt/CMakeLists.txt +++ b/src/mlpack/core/hpt/CMakeLists.txt @@ -1,8 +1,8 @@ set(SOURCES - bind.hpp cv_function.hpp cv_function_impl.hpp deduce_hp_types.hpp + fixed.hpp hpt.hpp hpt_impl.hpp ) diff --git a/src/mlpack/core/hpt/deduce_hp_types.hpp b/src/mlpack/core/hpt/deduce_hp_types.hpp index 1c8e32d6ca0..034bc36a963 100644 --- a/src/mlpack/core/hpt/deduce_hp_types.hpp +++ b/src/mlpack/core/hpt/deduce_hp_types.hpp @@ -13,7 +13,7 @@ #ifndef MLPACK_CORE_HPT_DEDUCE_HP_TYPES_HPP #define MLPACK_CORE_HPT_DEDUCE_HP_TYPES_HPP -#include +#include namespace mlpack { namespace hpt { @@ -60,10 +60,10 @@ struct DeduceHyperParameterTypes /** * Defining DeduceHyperParameterTypes for the case when not all argument types * have been processed, and the next one is the type of an argument that should - * be bound. + * be fixed. */ template -struct DeduceHyperParameterTypes, Args...> +struct DeduceHyperParameterTypes, Args...> { template struct ResultHolder diff --git a/src/mlpack/core/hpt/bind.hpp b/src/mlpack/core/hpt/fixed.hpp similarity index 62% rename from src/mlpack/core/hpt/bind.hpp rename to src/mlpack/core/hpt/fixed.hpp index 00803b6f713..122df0eef36 100644 --- a/src/mlpack/core/hpt/bind.hpp +++ b/src/mlpack/core/hpt/fixed.hpp @@ -1,16 +1,16 @@ /** - * @file bind.hpp + * @file fixed.hpp * @author Kirill Mishchenko * - * Facilities for supporting bound arguments. + * Facilities for supporting fixed arguments. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_HPT_BIND_HPP -#define MLPACK_CORE_HPT_BIND_HPP +#ifndef MLPACK_CORE_HPT_FIXED_HPP +#define MLPACK_CORE_HPT_FIXED_HPP #include @@ -20,55 +20,55 @@ namespace mlpack { namespace hpt { template -struct PreBoundArg; +struct PreFixedArg; /** - * Mark the given argument as one that should be bound. It can be applied to + * Mark the given argument as one that should be fixed. It can be applied to * arguments that are passed to the Optimize method of HyperParameterTuner. * * The implementation avoids data copying. If the passed argument is an l-value * reference, we store it as a const l-value rerefence inside the returned - * PreBoundArg object. If the passed argument is an r-value reference, + * PreFixedArg object. If the passed argument is an r-value reference, * ligth-weight coping (by taking possesion of the r-value) will be made during - * the initialization of the returned PreBoundArg object. + * the initialization of the returned PreFixedArg object. */ template -PreBoundArg Bind(T&& value) +PreFixedArg Fixed(T&& value) { - return PreBoundArg{std::forward(value)}; + return PreFixedArg{std::forward(value)}; } /** - * A struct for storing information about a bound argument. Objects of this type + * A struct for storing information about a fixed argument. Objects of this type * are supposed to be passed into the CVFunction constructor. * * This struct is not meant to be used directly by users. Rather use the - * mlpack::hpt::Bind function. + * mlpack::hpt::Fixed function. * - * @tparam T The type of the bound argument. - * @tparam I The index of the bound argument. + * @tparam T The type of the fixed argument. + * @tparam I The index of the fixed argument. */ template -struct BoundArg +struct FixedArg { - //! The index of the bound argument. + //! The index of the fixed argument. static const size_t index = I; - //! The value of the bound argument. + //! The value of the fixed argument. const T& value; }; /** - * A struct for marking arguments as ones that should be bound (it can be useful + * A struct for marking arguments as ones that should be fixed (it can be useful * for the Optimize method of HyperParameterTuner). Arguments of this type are - * supposed to be converted into structs of the type BoundArg by adding + * supposed to be converted into structs of the type FixedArg by adding * information about argument positions. * * This struct is not meant to be used directly by users. Rather use the - * mlpack::hpt::Bind function. + * mlpack::hpt::Fixed function. */ template -struct PreBoundArg +struct PreFixedArg { using Type = T; @@ -79,10 +79,10 @@ struct PreBoundArg * The specialization of the template for references. * * This struct is not meant to be used directly by users. Rather use the - * mlpack::hpt::Bind function. + * mlpack::hpt::Fixed function. */ template -struct PreBoundArg +struct PreFixedArg { using Type = T; @@ -90,16 +90,16 @@ struct PreBoundArg }; /** - * A type function for checking whether the given type is PreBoundArg. + * A type function for checking whether the given type is PreFixedArg. */ template -class IsPreBoundArg +class IsPreFixedArg { template struct Implementation : std::false_type {}; template - struct Implementation> : std::true_type {}; + struct Implementation> : std::true_type {}; public: static const bool value = Implementation::type>::value; diff --git a/src/mlpack/core/hpt/hpt.hpp b/src/mlpack/core/hpt/hpt.hpp index 92215cee3b9..b502d8281ea 100644 --- a/src/mlpack/core/hpt/hpt.hpp +++ b/src/mlpack/core/hpt/hpt.hpp @@ -50,7 +50,7 @@ namespace hpt { * @endcode * * When some hyper-parameters should not be optimized, you can specify values - * for them with the Bind function as in the following example of finding good + * for them with the Fixed function as in the following example of finding good * lambda1 and lambda2 values for LARS. * * @code @@ -63,8 +63,8 @@ namespace hpt { * arma::vec lambda2Set{0.0, 0.002, 0.02, 0.2, 2.0}; * * double bestLambda1, bestLambda2; - * std::tie(bestLambda1, bestLambda2) = hpt2.Optimize(Bind(transposeData), - * Bind(useCholesky), lambda1Set, lambda2Set); + * std::tie(bestLambda1, bestLambda2) = hpt2.Optimize(Fixed(transposeData), + * Fixed(useCholesky), lambda1Set, lambda2Set); * @endcode * * @tparam MLAlgorithm A machine learning algorithm. @@ -112,7 +112,7 @@ class HyperParameterTuner * The set of values should be an STL-compatible container (it should * provide begin() and end() methods returning iterators). * 2. A starting value (when using any other optimizer than GridSearch). - * 3. A value bound by using the function mlpack::hpt::Bind. In this case the + * 3. A value fixed by using the function mlpack::hpt::Fixed. In this case the * hyper-parameter will not be optimized. * * All arguments should be passed in the same order as if the corresponding @@ -123,7 +123,7 @@ class HyperParameterTuner * should be provided. * * The method returns a tuple of values for hyper-parameters that haven't been - * bound. + * fixed. * * @param args Arguments corresponding to hyper-parameters (see the method * description for more information). @@ -176,7 +176,7 @@ class HyperParameterTuner /** * The set of methods to initialize the GridSearch optimizer. We basically * need to go through all arguments passed to the Optimize method, gather all - * non-bound arguments (collections) and pass them into the GridSearch + * non-fixed arguments (collections) and pass them into the GridSearch * constructor. */ template::value>, - typename = std::enable_if_t::type>::value>> inline void InitGridSearch(const ArgsTuple& args, Collections... collections); @@ -199,7 +199,7 @@ class HyperParameterTuner typename ArgsTuple, typename... Collections, typename = std::enable_if_t::value>, - typename = std::enable_if_t::type>::value>, typename = void> inline void InitGridSearch(const ArgsTuple& args, @@ -209,37 +209,37 @@ class HyperParameterTuner * The set of methods to initialize a cost function (CVFunction) object and * run optimization to find the best hyper-parameters. To initialize a * CVFunction object we go through all arguments passed to the Optimize - * method, gather all bound values and pass them into the CVFunction + * method, gather all fixed values and pass them into the CVFunction * constructor. */ template::value>> inline void InitCVFunctionAndOptimize(const ArgsTuple& args, arma::mat& bestParams, - BoundArgs... boundArgs); + FixedArgs... fixedArgs); template::value>, - typename = std::enable_if_t::type>::value>> inline void InitCVFunctionAndOptimize(const ArgsTuple& args, arma::mat& bestParams, - BoundArgs... boundArgs); + FixedArgs... fixedArgs); template::value>, - typename = std::enable_if_t::type>::value>, typename = void> inline void InitCVFunctionAndOptimize(const ArgsTuple& args, arma::mat& bestParams, - BoundArgs... boundArgs); + FixedArgs... fixedArgs); /** * The set of methods to convert the given arma::vec vector to the tuple of diff --git a/src/mlpack/core/hpt/hpt_impl.hpp b/src/mlpack/core/hpt/hpt_impl.hpp index 77869a275a0..6bbc8ad242c 100644 --- a/src/mlpack/core/hpt/hpt_impl.hpp +++ b/src/mlpack/core/hpt/hpt_impl.hpp @@ -130,7 +130,7 @@ template -template +template void HyperParameterTuner::InitCVFunctionAndOptimize( const ArgsTuple& /* args */, arma::mat& bestParams, - BoundArgs... boundArgs) + FixedArgs... fixedArgs) { static const size_t totalArgs = std::tuple_size::value; - CVFunction - cvFunction(cv, boundArgs...); + CVFunction + cvFunction(cv, fixedArgs...); bestObjective = Metric::NeedsMinimization? optimizer.Optimize(cvFunction, bestParams) : -optimizer.Optimize(cvFunction, bestParams); @@ -159,7 +159,7 @@ template -template +template void HyperParameterTuner::InitCVFunctionAndOptimize( const ArgsTuple& args, arma::mat& bestParams, - BoundArgs... boundArgs) + FixedArgs... fixedArgs) { - using PreBoundArgT = typename std::remove_reference< + using PreFixedArgT = typename std::remove_reference< typename std::tuple_element::type>::type; - using BoundArgT = BoundArg; + using FixedArgT = FixedArg; - InitCVFunctionAndOptimize(args, bestParams, boundArgs..., - BoundArgT{std::get(args).value}); + InitCVFunctionAndOptimize(args, bestParams, fixedArgs..., + FixedArgT{std::get(args).value}); } template -template +template void HyperParameterTuner::InitCVFunctionAndOptimize( const ArgsTuple& args, arma::mat& bestParams, - BoundArgs... boundArgs) + FixedArgs... fixedArgs) { - InitCVFunctionAndOptimize(args, bestParams, boundArgs...); + InitCVFunctionAndOptimize(args, bestParams, fixedArgs...); } template #include #include -#include #include +#include #include #include #include @@ -29,7 +29,7 @@ using namespace mlpack::regression; BOOST_AUTO_TEST_SUITE(HPTTest); /** - * Test CVFunction runs cross-validation in according with specified bound + * Test CVFunction runs cross-validation in according with specified fixed * arguments and passed parameters. */ BOOST_AUTO_TEST_CASE(CVFunctionTest) @@ -45,10 +45,10 @@ BOOST_AUTO_TEST_CASE(CVFunctionTest) double lambda1 = 1.0; double lambda2 = 2.0; - BoundArg boundUseCholesky{useCholesky}; - BoundArg boundLambda1{lambda2}; - CVFunction, BoundArg> - cvFun(cv, boundUseCholesky, boundLambda1); + FixedArg fixedUseCholesky{useCholesky}; + FixedArg fixedLambda1{lambda2}; + CVFunction, FixedArg> + cvFun(cv, fixedUseCholesky, fixedLambda1); double expected = cv.Evaluate(transposeData, useCholesky, lambda1, lambda2); double actual = cvFun.Evaluate(arma::vec{double(transposeData), lambda1}); @@ -139,7 +139,7 @@ BOOST_AUTO_TEST_CASE(GridSearchTest) SimpleCV cv(validationSize, xs, ys); GridSearch optimizer(lambda1Set, lambda2Set); - CVFunction, BoundArg> + CVFunction, FixedArg> cvFun(cv, {transposeData}, {useCholesky}); arma::mat actualParameters; double actualObjective = optimizer.Optimize(cvFun, actualParameters); @@ -177,8 +177,8 @@ BOOST_AUTO_TEST_CASE(HPTTest) double actualLambda1, actualLambda2; HyperParameterTuner hpt(validationSize, xs, ys); - std::tie(actualLambda1, actualLambda2) = hpt.Optimize(Bind(transposeData), - Bind(useCholesky), lambda1Set, lambda2Set); + std::tie(actualLambda1, actualLambda2) = hpt.Optimize(Fixed(transposeData), + Fixed(useCholesky), lambda1Set, lambda2Set); BOOST_REQUIRE_CLOSE(expectedObjective, hpt.BestObjective(), 1e-5); BOOST_REQUIRE_CLOSE(expectedLambda1, actualLambda1, 1e-5); From 85060c8c78ffed836bf96d941f93c1158e271ce3 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Fri, 11 Aug 2017 10:46:00 +0500 Subject: [PATCH 17/27] Revert "Make optimal lambdas in the "middle" of the space" This reverts commit c6f897f05dd22cbe2b649b7a11a37c7f5b7d9693. --- src/mlpack/tests/hpt_test.cpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp index a55f3d4cf9d..d16ca6314b9 100644 --- a/src/mlpack/tests/hpt_test.cpp +++ b/src/mlpack/tests/hpt_test.cpp @@ -60,9 +60,6 @@ void InitProneToOverfittingData(arma::mat& xs, arma::rowvec& ys, double& validationSize) { - // Making the function generate the same data for all callers. - arma::arma_rng::set_seed(11); - // Total number of data points. size_t N = 10; // Total number of features (all except the first one are redundant). @@ -132,10 +129,6 @@ BOOST_AUTO_TEST_CASE(GridSearchTest) lambda1Set, lambda2Set, expectedLambda1, expectedLambda2, expectedObjective); - // We should get these values (it has been found empirically). - BOOST_REQUIRE_CLOSE(expectedLambda1, 0.01, 1e-5); - BOOST_REQUIRE_CLOSE(expectedLambda2, 0.05, 1e-5); - SimpleCV cv(validationSize, xs, ys); GridSearch optimizer(lambda1Set, lambda2Set); @@ -170,10 +163,6 @@ BOOST_AUTO_TEST_CASE(HPTTest) lambda1Set, lambda2Set, expectedLambda1, expectedLambda2, expectedObjective); - // We should get these values (it has been found empirically). - BOOST_REQUIRE_CLOSE(expectedLambda1, 0.01, 1e-5); - BOOST_REQUIRE_CLOSE(expectedLambda2, 0.05, 1e-5); - double actualLambda1, actualLambda2; HyperParameterTuner hpt(validationSize, xs, ys); From 6744351dc8b0591d267f249925efb760207d64ac Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Mon, 14 Aug 2017 11:21:20 +0500 Subject: [PATCH 18/27] Refactor GridSearch to use DatasetMapper --- .../optimizers/grid_search/grid_search.hpp | 58 ++++++------------- .../grid_search/grid_search_impl.hpp | 53 ++++++++++------- src/mlpack/tests/hpt_test.cpp | 15 ++++- 3 files changed, 63 insertions(+), 63 deletions(-) diff --git a/src/mlpack/core/optimizers/grid_search/grid_search.hpp b/src/mlpack/core/optimizers/grid_search/grid_search.hpp index cda39b8e601..a584237f628 100644 --- a/src/mlpack/core/optimizers/grid_search/grid_search.hpp +++ b/src/mlpack/core/optimizers/grid_search/grid_search.hpp @@ -29,27 +29,24 @@ namespace optimization { class GridSearch { public: - /** - * Initialize a GridSearch object. - * - * @param collections Collections of values (one for each parameter). Each - * collection should be an STL-compatible container (it should provide - * begin() and end() methods returning iterators). - */ - template - GridSearch(const Collections&... collections); - /** * Optimize (minimize) the given function by iterating through the all - * possible combinations of values for the parameters. + * possible combinations of values for the parameters specified in + * datasetInfo. + * + * @param function Function to optimize. + * @param bestParameters Variable for storing results. + * @param datasetInfo Type information for each dimension of the dataset. It + * should store possible values for each parameter. + * @return Objective value of the final point. */ template - double Optimize(FunctionType& function, arma::mat& bestParameters); + double Optimize( + FunctionType& function, + arma::mat& bestParameters, + data::DatasetMapper& datasetInfo); private: - //! Collections of parameter values (one for each parameter). - std::vector> parameterValueCollections; - /** * Iterate through the last (parameterValueCollections.size() - i) dimensions * of the grid and change the arguments bestObjective and bestParameters if @@ -58,30 +55,13 @@ class GridSearch * argument. */ template - void Optimize(FunctionType& function, - double& bestObjective, - arma::mat& bestParameters, - arma::vec& currentParameters, - size_t i); - - /** - * Convert each specified collection into a std::vector container and - * put results into parameterValueCollections. - */ - template - void InitParameterValueCollections(const Collection& collection, - const Collections&... collections) - { - parameterValueCollections.push_back( - std::vector(collection.begin(), collection.end())); - - InitParameterValueCollections(collections...); - } - - /** - * Finish initialization. - */ - void InitParameterValueCollections() {} + void Optimize( + FunctionType& function, + double& bestObjective, + arma::mat& bestParameters, + arma::vec& currentParameters, + data::DatasetMapper& datasetInfo, + size_t i); }; } // namespace optimization diff --git a/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp b/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp index e6e704ad3fd..e0fa2433306 100644 --- a/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp +++ b/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp @@ -17,44 +17,55 @@ namespace mlpack { namespace optimization { -template -GridSearch::GridSearch(const Collections&... collections) -{ - InitParameterValueCollections(collections...); -} - template -double GridSearch::Optimize(FunctionType& function, arma::mat& bestParameters) +double GridSearch::Optimize( + FunctionType& function, + arma::mat& bestParameters, + data::DatasetMapper& datasetInfo) { + for (size_t i = 0; i < datasetInfo.Dimensionality(); ++i) + { + if (datasetInfo.Type(i) != data::Datatype::categorical) + { + std::ostringstream oss; + oss << "GridSearch::Optimize(): the dimension " << i + << "is not categorical" << std::endl; + throw std::invalid_argument(oss.str()); + } + } + double bestObjective = std::numeric_limits::max(); - bestParameters = arma::mat(parameterValueCollections.size(), 1); - arma::vec currentParameters = arma::vec(parameterValueCollections.size()); + bestParameters = arma::mat(datasetInfo.Dimensionality(), 1); + arma::vec currentParameters = arma::vec(datasetInfo.Dimensionality()); /* Initialize best parameters for the case (very unlikely though) when no set * of parameters gives an objective value better than * std::numeric_limits::max() */ - for (size_t i = 0; i < parameterValueCollections.size(); ++i) - bestParameters(i, 0) = parameterValueCollections[i][0]; + for (size_t i = 0; i < datasetInfo.Dimensionality(); ++i) + bestParameters(i, 0) = datasetInfo.UnmapString(0, i); - Optimize(function, bestObjective, bestParameters, currentParameters, 0); + Optimize(function, bestObjective, bestParameters, currentParameters, + datasetInfo, 0); return bestObjective; } template -void GridSearch::Optimize(FunctionType& function, - double& bestObjective, - arma::mat& bestParameters, - arma::vec& currentParameters, - size_t i) +void GridSearch::Optimize( + FunctionType& function, + double& bestObjective, + arma::mat& bestParameters, + arma::vec& currentParameters, + data::DatasetMapper& datasetInfo, + size_t i) { - if (i < parameterValueCollections.size()) + if (i < datasetInfo.Dimensionality()) { - for (double value : parameterValueCollections[i]) + for (size_t j = 0; j < datasetInfo.NumMappings(i); ++j) { - currentParameters(i) = value; + currentParameters(i) = datasetInfo.UnmapString(j, i); Optimize(function, bestObjective, bestParameters, currentParameters, - i + 1); + datasetInfo, i + 1); } } else diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp index d16ca6314b9..9423ead3426 100644 --- a/src/mlpack/tests/hpt_test.cpp +++ b/src/mlpack/tests/hpt_test.cpp @@ -22,6 +22,7 @@ #include using namespace mlpack::cv; +using namespace mlpack::data; using namespace mlpack::hpt; using namespace mlpack::optimization; using namespace mlpack::regression; @@ -130,12 +131,20 @@ BOOST_AUTO_TEST_CASE(GridSearchTest) expectedObjective); SimpleCV cv(validationSize, xs, ys); - - GridSearch optimizer(lambda1Set, lambda2Set); CVFunction, FixedArg> cvFun(cv, {transposeData}, {useCholesky}); + + IncrementPolicy policy(true); + DatasetMapper datasetInfo(policy, 2); + for (double lambda1 : lambda1Set) + datasetInfo.MapString(lambda1, 0); + for (double lambda2 : lambda2Set) + datasetInfo.MapString(lambda2, 1); + + GridSearch optimizer; arma::mat actualParameters; - double actualObjective = optimizer.Optimize(cvFun, actualParameters); + double actualObjective = + optimizer.Optimize(cvFun, actualParameters, datasetInfo); BOOST_REQUIRE_CLOSE(expectedObjective, actualObjective, 1e-5); BOOST_REQUIRE_CLOSE(expectedLambda1, actualParameters(0, 0), 1e-5); From 9486986b8b54e8acf7569dcc8b1b6853bb19bf7d Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Mon, 14 Aug 2017 12:03:43 +0500 Subject: [PATCH 19/27] Use the new GridSearch interface in the HPT module --- src/mlpack/core/hpt/hpt.hpp | 86 +++++++++++++---------------- src/mlpack/core/hpt/hpt_impl.hpp | 94 ++++++++------------------------ 2 files changed, 60 insertions(+), 120 deletions(-) diff --git a/src/mlpack/core/hpt/hpt.hpp b/src/mlpack/core/hpt/hpt.hpp index b502d8281ea..13c60c8f9d9 100644 --- a/src/mlpack/core/hpt/hpt.hpp +++ b/src/mlpack/core/hpt/hpt.hpp @@ -174,72 +174,62 @@ class HyperParameterTuner MLAlgorithm bestModel; /** - * The set of methods to initialize the GridSearch optimizer. We basically - * need to go through all arguments passed to the Optimize method, gather all - * non-fixed arguments (collections) and pass them into the GridSearch - * constructor. + * The set of methods to initialize auxiliary objects (a CVFunction object and + * the datasetInfo parameter) and run optimization to find the best + * hyper-parameters. + * + * This template is called when we are ready to run optimization. */ - template::value>> - inline void InitGridSearch(const ArgsTuple& args, - Collections... collections); - - template::value>, - typename = std::enable_if_t::type>::value>> - inline void InitGridSearch(const ArgsTuple& args, - Collections... collections); - - template::value>, - typename = std::enable_if_t::type>::value>, - typename = void> - inline void InitGridSearch(const ArgsTuple& args, - Collections... collections); + inline void InitAndOptimize( + const ArgsTuple& args, + arma::mat& bestParams, + data::DatasetMapper& datasetInfo, + FixedArgs... fixedArgs); /** - * The set of methods to initialize a cost function (CVFunction) object and - * run optimization to find the best hyper-parameters. To initialize a - * CVFunction object we go through all arguments passed to the Optimize - * method, gather all fixed values and pass them into the CVFunction - * constructor. + * The set of methods to initialize auxiliary objects (a CVFunction object and + * the datasetInfo parameter) and run optimization to find the best + * hyper-parameters. + * + * This template is called when the next argument should be fixed (should not + * be optimized). */ - template::value>> - inline void InitCVFunctionAndOptimize(const ArgsTuple& args, - arma::mat& bestParams, - FixedArgs... fixedArgs); - - template::value>, typename = std::enable_if_t::type>::value>> - inline void InitCVFunctionAndOptimize(const ArgsTuple& args, - arma::mat& bestParams, - FixedArgs... fixedArgs); + inline void InitAndOptimize( + const ArgsTuple& args, + arma::mat& bestParams, + data::DatasetMapper& datasetInfo, + FixedArgs... fixedArgs); - template::value>, typename = std::enable_if_t::type>::value>, typename = void> - inline void InitCVFunctionAndOptimize(const ArgsTuple& args, - arma::mat& bestParams, - FixedArgs... fixedArgs); + inline void InitAndOptimize( + const ArgsTuple& args, + arma::mat& bestParams, + data::DatasetMapper& datasetInfo, + FixedArgs... fixedArgs); /** * The set of methods to convert the given arma::vec vector to the tuple of diff --git a/src/mlpack/core/hpt/hpt_impl.hpp b/src/mlpack/core/hpt/hpt_impl.hpp index 6bbc8ad242c..f558985844e 100644 --- a/src/mlpack/core/hpt/hpt_impl.hpp +++ b/src/mlpack/core/hpt/hpt_impl.hpp @@ -51,78 +51,20 @@ TupleOfHyperParameters HyperParameterTuner::Optimize( const Args&... args) { + static const size_t numberOfParametersToOptimize = + std::tuple_size>::value; + data::IncrementPolicy policy(true); + data::DatasetMapper datasetInfo(policy, + numberOfParametersToOptimize); + arma::mat bestParameters; const auto argsTuple = std::tie(args...); - InitGridSearch<0>(argsTuple); - InitCVFunctionAndOptimize<0>(argsTuple, bestParameters); + InitAndOptimize<0>(argsTuple, bestParameters, datasetInfo); return VectorToTuple, 0>(bestParameters); } -template class CV, - typename Optimizer, - typename MatType, - typename PredictionsType, - typename WeightsType> -template -void HyperParameterTuner::InitGridSearch( - const ArgsTuple& /* args */, - Collections... collections) -{ - optimizer = Optimizer(collections...); -} - -template class CV, - typename Optimizer, - typename MatType, - typename PredictionsType, - typename WeightsType> -template -void HyperParameterTuner::InitGridSearch( - const ArgsTuple& args, - Collections... collections) -{ - InitGridSearch(args, collections...); -} - -template class CV, - typename Optimizer, - typename MatType, - typename PredictionsType, - typename WeightsType> -template -void HyperParameterTuner::InitGridSearch( - const ArgsTuple& args, - Collections... collections) -{ - InitGridSearch(args, collections..., std::get(args)); -} - template class CV, @@ -137,9 +79,10 @@ void HyperParameterTuner::InitCVFunctionAndOptimize( + WeightsType>::InitAndOptimize( const ArgsTuple& /* args */, arma::mat& bestParams, + data::DatasetMapper& datasetInfo, FixedArgs... fixedArgs) { static const size_t totalArgs = std::tuple_size::value; @@ -147,8 +90,8 @@ void HyperParameterTuner cvFunction(cv, fixedArgs...); bestObjective = Metric::NeedsMinimization? - optimizer.Optimize(cvFunction, bestParams) : - -optimizer.Optimize(cvFunction, bestParams); + optimizer.Optimize(cvFunction, bestParams, datasetInfo) : + -optimizer.Optimize(cvFunction, bestParams, datasetInfo); bestModel = std::move(cvFunction.BestModel()); } @@ -166,16 +109,17 @@ void HyperParameterTuner::InitCVFunctionAndOptimize( + WeightsType>::InitAndOptimize( const ArgsTuple& args, arma::mat& bestParams, + data::DatasetMapper& datasetInfo, FixedArgs... fixedArgs) { using PreFixedArgT = typename std::remove_reference< typename std::tuple_element::type>::type; using FixedArgT = FixedArg; - InitCVFunctionAndOptimize(args, bestParams, fixedArgs..., + InitAndOptimize(args, bestParams, datasetInfo, fixedArgs..., FixedArgT{std::get(args).value}); } @@ -193,12 +137,18 @@ void HyperParameterTuner::InitCVFunctionAndOptimize( + WeightsType>::InitAndOptimize( const ArgsTuple& args, arma::mat& bestParams, + data::DatasetMapper& datasetInfo, FixedArgs... fixedArgs) { - InitCVFunctionAndOptimize(args, bestParams, fixedArgs...); + static const size_t dimension = + I - std::tuple_size>::value; + for (auto value : std::get(args)) + datasetInfo.MapString(value, dimension); + + InitAndOptimize(args, bestParams, datasetInfo, fixedArgs...); } template Date: Mon, 14 Aug 2017 12:13:00 +0500 Subject: [PATCH 20/27] Add separate comments --- src/mlpack/core/hpt/hpt.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/hpt/hpt.hpp b/src/mlpack/core/hpt/hpt.hpp index 13c60c8f9d9..68580dc8e49 100644 --- a/src/mlpack/core/hpt/hpt.hpp +++ b/src/mlpack/core/hpt/hpt.hpp @@ -232,8 +232,8 @@ class HyperParameterTuner FixedArgs... fixedArgs); /** - * The set of methods to convert the given arma::vec vector to the tuple of - * the target type by gathering all elements in an argument list. + * Gather all elements of vector in an argument list and use them to create a + * tuple. */ template::value>> inline TupleType VectorToTuple(const arma::vec& vector, const Args&... args); + /** + * Create a tuple from args. + */ template Date: Thu, 17 Aug 2017 09:09:24 +0500 Subject: [PATCH 21/27] Add gradient evaluation to CVFunction --- src/mlpack/core/hpt/cv_function.hpp | 30 ++++++++++- src/mlpack/core/hpt/cv_function_impl.hpp | 29 +++++++++- src/mlpack/core/hpt/hpt.hpp | 12 +++++ src/mlpack/core/hpt/hpt_impl.hpp | 4 +- src/mlpack/tests/hpt_test.cpp | 67 +++++++++++++++++++++++- 5 files changed, 135 insertions(+), 7 deletions(-) diff --git a/src/mlpack/core/hpt/cv_function.hpp b/src/mlpack/core/hpt/cv_function.hpp index cfabcb28f2e..3430e2ea063 100644 --- a/src/mlpack/core/hpt/cv_function.hpp +++ b/src/mlpack/core/hpt/cv_function.hpp @@ -43,11 +43,23 @@ class CVFunction * Initialize a CVFunction object. * * @param cv A cross-validation object. + * @param relativeDelta Relative increase of arguments for calculation of + * partial derivatives (by the definition). The exact increase for some + * particular argument is equal to the absolute value of the argument + * multiplied by the relative increase (see also the documentation for the + * minDelta parameter). + * @param minDelta Minimum increase of arguments for calculation of partial + * derivatives (by the definition). This value is going to be used when it + * is greater than the increase calculated with the rules described in the + * documentation for the relativeDelta parameter. * @param BoundArgs Arguments that should be passed into the Evaluate method * of the CVType object but are not going to be passed into the Evaluate * method of this object. */ - CVFunction(CVType& cv, const BoundArgs&... args); + CVFunction(CVType& cv, + const double relativeDelta, + const double minDelta, + const BoundArgs&... args); /** * Run cross-validation with the bound and passed parameters. @@ -57,6 +69,16 @@ class CVFunction */ double Evaluate(const arma::mat& parameters); + /** + * Evaluate numerically the gradient of the CVFunction with the given + * parameters. + * + * @param parameters Arguments (rather than the bound arguments) that should + * be passed into the Evaluate method of the CVType object. + * @param gradient Vector to output the gradient into. + */ + void Gradient(const arma::mat& parameters, arma::mat& gradient); + //! Access and modify the best model so far. MLAlgorithm& BestModel() { return bestModel; } @@ -90,6 +112,12 @@ class CVFunction //! The best model so far. MLAlgorithm bestModel; + //! Relative increase of arguments for calculation of gradient. + double relativeDelta; + + //! Minimum absolute increase of arguments for calculation of gradient. + double minDelta; + /** * Collect all arguments and run cross-validation. */ diff --git a/src/mlpack/core/hpt/cv_function_impl.hpp b/src/mlpack/core/hpt/cv_function_impl.hpp index 5f854a10820..c8bb36b6ddd 100644 --- a/src/mlpack/core/hpt/cv_function_impl.hpp +++ b/src/mlpack/core/hpt/cv_function_impl.hpp @@ -45,10 +45,15 @@ template CVFunction::CVFunction( - CVType& cv, const BoundArgs&... args) : + CVType& cv, + const double relativeDelta, + const double minDelta, + const BoundArgs&... args) : cv(cv), boundArgs(args...), - bestObjective(std::numeric_limits::max()) + bestObjective(std::numeric_limits::max()), + relativeDelta(relativeDelta), + minDelta(minDelta) { /* Nothing left to do. */ } template::Evaluate( return Evaluate<0, 0>(parameters); } +template +void CVFunction::Gradient( + const arma::mat& parameters, + arma::mat& gradient) +{ + gradient = arma::mat(arma::size(parameters)); + arma::mat increasedParameters = parameters; + for (size_t i = 0; i < parameters.n_rows; ++i) + { + double delta = std::max(std::abs(parameters(i)) * relativeDelta, minDelta); + increasedParameters(i) += delta; + gradient(i) = (Evaluate(increasedParameters) - Evaluate(parameters)) / + delta; + increasedParameters(i) = parameters(i); + } +} + template::HyperParameterTuner(const CVArgs&... args) : - cv(args...) {} + cv(args...), relativeDelta(0.01), minDelta(1e-10) {} template::value; CVFunction - cvFunction(cv, fixedArgs...); + cvFunction(cv, relativeDelta, minDelta, fixedArgs...); bestObjective = Metric::NeedsMinimization? optimizer.Optimize(cvFunction, bestParams, datasetInfo) : -optimizer.Optimize(cvFunction, bestParams, datasetInfo); diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp index 9423ead3426..f7882272523 100644 --- a/src/mlpack/tests/hpt_test.cpp +++ b/src/mlpack/tests/hpt_test.cpp @@ -49,7 +49,7 @@ BOOST_AUTO_TEST_CASE(CVFunctionTest) FixedArg fixedUseCholesky{useCholesky}; FixedArg fixedLambda1{lambda2}; CVFunction, FixedArg> - cvFun(cv, fixedUseCholesky, fixedLambda1); + cvFun(cv, 0.0, 0.0, fixedUseCholesky, fixedLambda1); double expected = cv.Evaluate(transposeData, useCholesky, lambda1, lambda2); double actual = cvFun.Evaluate(arma::vec{double(transposeData), lambda1}); @@ -57,6 +57,69 @@ BOOST_AUTO_TEST_CASE(CVFunctionTest) BOOST_REQUIRE_CLOSE(expected, actual, 1e-5); } +/** + * This class provides the interface of CV classes, but really implements a + * simple quadratic function of three variables. + */ +template +class QuadraticFunction +{ + public: + QuadraticFunction(double a, double b, double c, double d) : + a(a), b(b), c(c), d(d) {} + + double Evaluate(double x, double y, double z) + { + return a * x * x + b * y * y + c * z * z + d; + } + + // Declaring and defining it just in order to provide the same interface as + // other CV classes. + MLAlgorithm Model() + { + return MLAlgorithm(); + } + + private: + double a, b, c, d; +}; + +/** + * Test CVFunction approximates gradient in the expected way. + */ +BOOST_AUTO_TEST_CASE(CVFunctionGradientTest) +{ + double a = 1.0; + double b = -1.5; + double c = 2.5; + double d = 3.0; + QuadraticFunction lf(a, b, c, d); + + double relativeDelta = 0.01; + double minDelta = 0.001; + CVFunction cvFun(lf, relativeDelta, minDelta); + + double x = 0.0; + double y = -1.0; + double z = 2.0; + arma::mat gradient; + cvFun.Gradient(arma::vec{x, y, z}, gradient); + + double xDelta = minDelta; + double yDelta = relativeDelta * abs(y); + double zDelta = relativeDelta * abs(z); + + double aproximateXPartialDerivative = a * (2 * x + xDelta); + double aproximateYPartialDerivative = b * (2 * y + yDelta); + double aproximateZPartialDerivative = c * (2 * z + zDelta); + + BOOST_REQUIRE_EQUAL(gradient.n_elem, 3); + BOOST_REQUIRE_CLOSE(gradient(0), aproximateXPartialDerivative, 1e-5); + BOOST_REQUIRE_CLOSE(gradient(1), aproximateYPartialDerivative, 1e-5); + BOOST_REQUIRE_CLOSE(gradient(2), aproximateZPartialDerivative, 1e-5); +} + + void InitProneToOverfittingData(arma::mat& xs, arma::rowvec& ys, double& validationSize) @@ -132,7 +195,7 @@ BOOST_AUTO_TEST_CASE(GridSearchTest) SimpleCV cv(validationSize, xs, ys); CVFunction, FixedArg> - cvFun(cv, {transposeData}, {useCholesky}); + cvFun(cv, 0.0, 0.0, {transposeData}, {useCholesky}); IncrementPolicy policy(true); DatasetMapper datasetInfo(policy, 2); From 5a6df5578c1b9570970c6831f851a87582761c00 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Fri, 18 Aug 2017 15:31:53 +0500 Subject: [PATCH 22/27] Add support for GradientDescent in the HPT module --- src/mlpack/core/hpt/deduce_hp_types.hpp | 28 ++++++-- src/mlpack/core/hpt/hpt.hpp | 65 ++++++++++++++++--- src/mlpack/core/hpt/hpt_impl.hpp | 31 ++++++++- .../gradient_descent/gradient_descent.hpp | 23 ++++++- .../gradient_descent_impl.hpp | 29 +++++++++ src/mlpack/tests/hpt_test.cpp | 64 ++++++++++++++++-- 6 files changed, 221 insertions(+), 19 deletions(-) diff --git a/src/mlpack/core/hpt/deduce_hp_types.hpp b/src/mlpack/core/hpt/deduce_hp_types.hpp index 034bc36a963..03817b5d19b 100644 --- a/src/mlpack/core/hpt/deduce_hp_types.hpp +++ b/src/mlpack/core/hpt/deduce_hp_types.hpp @@ -42,16 +42,36 @@ struct DeduceHyperParameterTypes /** * Defining DeduceHyperParameterTypes for the case when not all argument types - * have been processed, and the next one is a collection type. + * have been processed, and the next one (T) is a collection type or an + * arithmetic type. */ -template -struct DeduceHyperParameterTypes +template +struct DeduceHyperParameterTypes { + /** + * A type function to deduce the result hyper-parameter type for ArgumentType. + */ + template::value> + struct ResultHPType; + + template + struct ResultHPType + { + using Type = ArithmeticType; + }; + + template + struct ResultHPType + { + using Type = typename CollectionType::value_type; + }; + template struct ResultHolder { using TupleType = typename DeduceHyperParameterTypes::template - ResultHolder::TupleType; + ResultHolder::Type>::TupleType; }; using TupleType = typename ResultHolder<>::TupleType; diff --git a/src/mlpack/core/hpt/hpt.hpp b/src/mlpack/core/hpt/hpt.hpp index a0995c2ee66..f1601d548ec 100644 --- a/src/mlpack/core/hpt/hpt.hpp +++ b/src/mlpack/core/hpt/hpt.hpp @@ -71,7 +71,8 @@ namespace hpt { * @tparam Metric A metric to assess the quality of a trained model. * @tparam CV A cross-validation strategy used to assess a set of * hyper-parameters. - * @tparam Optimizer An optimization strategy. + * @tparam OptimizerType An optimization strategy (GridSearch and + * GradientDescent are supported). * @tparam MatType The type of data. * @tparam PredictionsType The type of predictions (should be passed when the * predictions type is a template parameter in Train methods of the given @@ -84,7 +85,7 @@ namespace hpt { template class CV, - typename Optimizer = mlpack::optimization::GridSearch, + typename OptimizerType = mlpack::optimization::GridSearch, typename MatType = arma::mat, typename PredictionsType = typename cv::MetaInfoExtractor HyperParameterTuner(const CVArgs& ...args); + //! Access and modify the optimizer. + OptimizerType& Optimizer() { return optimizer; } + + /** + * Access and modify relative increase of arguments for calculation of partial + * derivatives (by the definition) in gradient-based optimization. The + * exact increase for some particular argument is equal to the absolute + * value of the argument multiplied by the relative increase (see also the + * documentation for MinDelta()). + */ + double& RelativeDelta() { return relativeDelta; } + + /** + * Access and modify minimum increase of arguments for calculation of partial + * derivatives (by the definition) in gradient-based optimization. This value + * is going to be used when it is greater than the increase calculated with + * the rules described in the documentation for RelativeDelta(). + */ + double& MinDelta() { return minDelta; } + /** * Find the best hyper-parameters by using the given Optimizer. For each * hyper-parameter one of the following should be passed as an argument. @@ -138,10 +159,6 @@ class HyperParameterTuner MLAlgorithm& BestModel() { return bestModel; } private: - static_assert( - std::is_same::value, - "HyperParameterTuner now supports only the GridSearch optimizer"); - /** * A decorator that returns negated values of the original metric. */ @@ -165,7 +182,7 @@ class HyperParameterTuner CVType cv; //! The optimizer. - Optimizer optimizer; + OptimizerType optimizer; //! The best objective from the last run. double bestObjective; @@ -185,6 +202,14 @@ class HyperParameterTuner */ double minDelta; + /** + * A type function to check whether the element I of the tuple type is an + * arithmetic type. + */ + template + using IsArithmetic = std::is_arithmetic::type>::type>; + /** * The set of methods to initialize auxiliary objects (a CVFunction object and * the datasetInfo parameter) and run optimization to find the best @@ -222,6 +247,28 @@ class HyperParameterTuner data::DatasetMapper& datasetInfo, FixedArgs... fixedArgs); + /** + * The set of methods to initialize auxiliary objects (a CVFunction object and + * the datasetInfo parameter) and run optimization to find the best + * hyper-parameters. + * + * This template is called when the next argument is of an arithmetic type and + * should be used as an initial value for the hyper-parameter. + */ + template::value>, + typename = std::enable_if_t::type>::value && + IsArithmetic::value>, + typename = void> + inline void InitAndOptimize( + const ArgsTuple& args, + arma::mat& bestParams, + data::DatasetMapper& datasetInfo, + FixedArgs... fixedArgs); + /** * The set of methods to initialize auxiliary objects (a CVFunction object and * the datasetInfo parameter) and run optimization to find the best @@ -235,7 +282,9 @@ class HyperParameterTuner typename... FixedArgs, typename = std::enable_if_t::value>, typename = std::enable_if_t::type>::value>, + typename std::tuple_element::type>::value && + !IsArithmetic::value>, + typename = void, typename = void> inline void InitAndOptimize( const ArgsTuple& args, diff --git a/src/mlpack/core/hpt/hpt_impl.hpp b/src/mlpack/core/hpt/hpt_impl.hpp index 868e37203e1..d07831e8f2d 100644 --- a/src/mlpack/core/hpt/hpt_impl.hpp +++ b/src/mlpack/core/hpt/hpt_impl.hpp @@ -57,7 +57,7 @@ TupleOfHyperParameters HyperParameterTuner datasetInfo(policy, numberOfParametersToOptimize); - arma::mat bestParameters; + arma::mat bestParameters(numberOfParametersToOptimize, 1); const auto argsTuple = std::tie(args...); InitAndOptimize<0>(argsTuple, bestParameters, datasetInfo); @@ -131,6 +131,35 @@ template template +void HyperParameterTuner::InitAndOptimize( + const ArgsTuple& args, + arma::mat& bestParams, + data::DatasetMapper& datasetInfo, + FixedArgs... fixedArgs) +{ + static const size_t dimension = + I - std::tuple_size>::value; + datasetInfo.Type(dimension) = data::Datatype::numeric; + bestParams(dimension) = std::get(args); + + InitAndOptimize(args, bestParams, datasetInfo, fixedArgs...); +} + +template class CV, + typename Optimizer, + typename MatType, + typename PredictionsType, + typename WeightsType> +template void HyperParameterTuner +#include namespace mlpack { namespace optimization { @@ -78,6 +78,27 @@ class GradientDescent template double Optimize(FunctionType& function, arma::mat& iterate); + /** + * Assert all dimensions are numeric and optimize the given function using + * gradient descent. The given starting point will be modified to store the + * finishing point of the algorithm, and the final objective value is + * returned. + * + * This overload is intended to be used primarily by the hyper-parameter + * tuning module. + * + * @tparam FunctionType Type of the function to optimize. + * @param function Function to optimize. + * @param iterate Starting point (will be modified). + * @param datasetInfo Type information for each dimension of the dataset. + * @return Objective value of the final point. + */ + template + double Optimize( + FunctionType& function, + arma::mat& iterate, + data::DatasetMapper& datasetInfo); + //! Get the step size. double StepSize() const { return stepSize; } //! Modify the step size. diff --git a/src/mlpack/core/optimizers/gradient_descent/gradient_descent_impl.hpp b/src/mlpack/core/optimizers/gradient_descent/gradient_descent_impl.hpp index 5339692b2e8..bb51bed43d7 100644 --- a/src/mlpack/core/optimizers/gradient_descent/gradient_descent_impl.hpp +++ b/src/mlpack/core/optimizers/gradient_descent/gradient_descent_impl.hpp @@ -67,6 +67,35 @@ double GradientDescent::Optimize( return overallObjective; } +template +double GradientDescent::Optimize( + FunctionType& function, + arma::mat& iterate, + data::DatasetMapper& datasetInfo) +{ + if (datasetInfo.Dimensionality() != iterate.n_rows) + { + std::ostringstream oss; + oss << "GradientDescent::Optimize(): expected information about " + << iterate.n_rows << " dimensions in datasetInfo, but found about " + << datasetInfo.Dimensionality() << std::endl; + throw std::invalid_argument(oss.str()); + } + + for (size_t i = 0; i < datasetInfo.Dimensionality(); ++i) + { + if (datasetInfo.Type(i) != data::Datatype::numeric) + { + std::ostringstream oss; + oss << "GradientDescent::Optimize(): the dimension " << i + << "is not numeric" << std::endl; + throw std::invalid_argument(oss.str()); + } + } + + return Optimize(function, iterate); +} + } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp index f7882272523..3783997f970 100644 --- a/src/mlpack/tests/hpt_test.cpp +++ b/src/mlpack/tests/hpt_test.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -61,16 +62,27 @@ BOOST_AUTO_TEST_CASE(CVFunctionTest) * This class provides the interface of CV classes, but really implements a * simple quadratic function of three variables. */ -template +template class QuadraticFunction { public: - QuadraticFunction(double a, double b, double c, double d) : - a(a), b(b), c(c), d(d) {} + QuadraticFunction(double a, + double b, + double c, + double d, + double xMin = 0.0, + double yMin = 0.0, + double zMin = 0.0) : + a(a), b(b), c(c), d(d), xMin(xMin), yMin(yMin), zMin(zMin) {} double Evaluate(double x, double y, double z) { - return a * x * x + b * y * y + c * z * z + d; + return a * pow(x - xMin, 2) + b * pow(y - yMin, 2) + c * pow(z - zMin, 2) + + d; } // Declaring and defining it just in order to provide the same interface as @@ -81,7 +93,7 @@ class QuadraticFunction } private: - double a, b, c, d; + double a, b, c, d, xMin, yMin, zMin; }; /** @@ -288,4 +300,46 @@ BOOST_AUTO_TEST_CASE(HPTMaximizationTest) BOOST_REQUIRE_CLOSE(actualLambda, 0.0, 1e-5); } +/** + * Test HyperParameterTuner works with GradientDescent. + */ +BOOST_AUTO_TEST_CASE(HPTGradientDescentTest) +{ + // Constructor arguments for the fake CV function (QuadraticFunction). + double a = 1.0; + double b = -1.5; + double c = 2.5; + double d = 3.0; + + // Optimal values for three "hyper-parameters". + double xMin = 1.5; + double yMin = 0.0; + double zMin = -2.0; + + // We pass LARS just because some ML algorithm should be passed. We pass MSE + // to tell HyperParameterTuner that the objective function (QuadraticFunction) + // should be minimized. + HyperParameterTuner + hpt(a, b, c, d, xMin, yMin, zMin); + + // Setting GradientDescent to find more close solution to the optimal one. + hpt.Optimizer().StepSize() = 0.1; + hpt.Optimizer().Tolerance() = 1e-15; + + // Always using the same small increase of arguments in calculation of partial + // derivatives. + hpt.RelativeDelta() = 0.0; + hpt.MinDelta() = 1e-10; + + // We will try to find optimal values only for two "hyper-parameters". + double x0 = 3.0; + double y = yMin; + double z0 = -3.0; + + double xOptimized, zOptimized; + std::tie(xOptimized, zOptimized) = hpt.Optimize(x0, Fixed(y), z0); + BOOST_REQUIRE_CLOSE(xOptimized, xMin, 1e-4); + BOOST_REQUIRE_CLOSE(zOptimized, zMin, 1e-4); +} + BOOST_AUTO_TEST_SUITE_END(); From fd172d9283b6e38b0fbb2ab68706dec5c5d2fd14 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Fri, 18 Aug 2017 15:56:22 +0500 Subject: [PATCH 23/27] Refactor template conditions for InitAndOptimize --- src/mlpack/core/hpt/hpt.hpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/mlpack/core/hpt/hpt.hpp b/src/mlpack/core/hpt/hpt.hpp index f1601d548ec..7ffd779ecd0 100644 --- a/src/mlpack/core/hpt/hpt.hpp +++ b/src/mlpack/core/hpt/hpt.hpp @@ -202,6 +202,13 @@ class HyperParameterTuner */ double minDelta; + /** + * A type function to check whether the element I of the tuple type is a + * PreFixedArg. + */ + template + using IsPreFixed = IsPreFixedArg::type>; + /** * A type function to check whether the element I of the tuple type is an * arithmetic type. @@ -239,8 +246,7 @@ class HyperParameterTuner typename ArgsTuple, typename... FixedArgs, typename = std::enable_if_t::value>, - typename = std::enable_if_t::type>::value>> + typename = std::enable_if_t::value>> inline void InitAndOptimize( const ArgsTuple& args, arma::mat& bestParams, @@ -259,8 +265,7 @@ class HyperParameterTuner typename ArgsTuple, typename... FixedArgs, typename = std::enable_if_t::value>, - typename = std::enable_if_t::type>::value && + typename = std::enable_if_t::value && IsArithmetic::value>, typename = void> inline void InitAndOptimize( @@ -281,8 +286,7 @@ class HyperParameterTuner typename ArgsTuple, typename... FixedArgs, typename = std::enable_if_t::value>, - typename = std::enable_if_t::type>::value && + typename = std::enable_if_t::value && !IsArithmetic::value>, typename = void, typename = void> From fd95da51db2ae99af0d7e820f980dfdb1801f8b1 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Fri, 18 Aug 2017 17:04:48 +0500 Subject: [PATCH 24/27] Add assertion for argument types of Optimize The added assertion causes a compilation error if one of aguments passed to the Optimize method of HyperParameterTuner is neither of an arithmetic type, nor of a collection type, nor fixed with the Fixed function. --- src/mlpack/core/hpt/deduce_hp_types.hpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/mlpack/core/hpt/deduce_hp_types.hpp b/src/mlpack/core/hpt/deduce_hp_types.hpp index 03817b5d19b..a1e45a7adca 100644 --- a/src/mlpack/core/hpt/deduce_hp_types.hpp +++ b/src/mlpack/core/hpt/deduce_hp_types.hpp @@ -61,9 +61,32 @@ struct DeduceHyperParameterTypes using Type = ArithmeticType; }; + /** + * A type function to check whether Type is a collection type (for that it + * should define value_type). + */ + template + struct IsCollectionType + { + using Yes = char[1]; + using No = char[2]; + + template + static Yes& Check(typename TypeToCheck::value_type*); + template + static No& Check(...); + + static const bool value = + sizeof(decltype(Check(0))) == sizeof(Yes); + }; + template struct ResultHPType { + static_assert(IsCollectionType::value, + "One of the passed arguments is neither of an arithmetic type, nor of " + "a collection type, nor fixed with the Fixed function."); + using Type = typename CollectionType::value_type; }; From 7de610e423559c68a2f67f7ff1c0016a3532e6c9 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Fri, 18 Aug 2017 17:23:57 +0500 Subject: [PATCH 25/27] Add assertion that input collections aren't empty --- src/mlpack/core/hpt/hpt_impl.hpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/mlpack/core/hpt/hpt_impl.hpp b/src/mlpack/core/hpt/hpt_impl.hpp index d07831e8f2d..36d4962795d 100644 --- a/src/mlpack/core/hpt/hpt_impl.hpp +++ b/src/mlpack/core/hpt/hpt_impl.hpp @@ -177,6 +177,14 @@ void HyperParameterTuner(args)) datasetInfo.MapString(value, dimension); + if (datasetInfo.NumMappings(dimension) == 0) + { + std::ostringstream oss; + oss << "HyperParameterTuner::Optimize(): the collection passed as the " + << "argument " << I + 1 << " is empty" << std::endl; + throw std::invalid_argument(oss.str()); + } + InitAndOptimize(args, bestParams, datasetInfo, fixedArgs...); } From ab46e58dd1f85c6dabdcf0f190570b76ba9f551a Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Fri, 25 Aug 2017 09:47:21 +0500 Subject: [PATCH 26/27] Add const getters in HyperParameterTuner --- src/mlpack/core/hpt/hpt.hpp | 42 +++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/src/mlpack/core/hpt/hpt.hpp b/src/mlpack/core/hpt/hpt.hpp index 7ffd779ecd0..fb5dc306773 100644 --- a/src/mlpack/core/hpt/hpt.hpp +++ b/src/mlpack/core/hpt/hpt.hpp @@ -110,19 +110,36 @@ class HyperParameterTuner OptimizerType& Optimizer() { return optimizer; } /** - * Access and modify relative increase of arguments for calculation of partial - * derivatives (by the definition) in gradient-based optimization. The - * exact increase for some particular argument is equal to the absolute - * value of the argument multiplied by the relative increase (see also the - * documentation for MinDelta()). + * Get relative increase of arguments for calculation of partial + * derivatives (by the definition) in gradient-based optimization. The exact + * increase for some particular argument is equal to the absolute value of the + * argument multiplied by the relative increase (see also the documentation + * for MinDelta()). + */ + double RelativeDelta() const { return relativeDelta; } + + /** + * Modify relative increase of arguments for calculation of partial + * derivatives (by the definition) in gradient-based optimization. The exact + * increase for some particular argument is equal to the absolute value of the + * argument multiplied by the relative increase (see also the documentation + * for MinDelta()). */ double& RelativeDelta() { return relativeDelta; } /** - * Access and modify minimum increase of arguments for calculation of partial - * derivatives (by the definition) in gradient-based optimization. This value - * is going to be used when it is greater than the increase calculated with - * the rules described in the documentation for RelativeDelta(). + * Get minimum increase of arguments for calculation of partial derivatives + * (by the definition) in gradient-based optimization. This value is going to + * be used when it is greater than the increase calculated with the rules + * described in the documentation for RelativeDelta(). + */ + double MinDelta() const { return minDelta; } + + /** + * Modify minimum increase of arguments for calculation of partial derivatives + * (by the definition) in gradient-based optimization. This value is going to + * be used when it is greater than the increase calculated with the rules + * described in the documentation for RelativeDelta(). */ double& MinDelta() { return minDelta; } @@ -152,10 +169,13 @@ class HyperParameterTuner template TupleOfHyperParameters Optimize(const Args&... args); - //! Access the performance measurement of the best model from the last run. + //! Get the performance measurement of the best model from the last run. double BestObjective() const { return bestObjective; } - //! Access and modify the best model from the last run. + //! Get the best model from the last run. + const MLAlgorithm& BestModel() const { return bestModel; } + + //! Modify the best model from the last run. MLAlgorithm& BestModel() { return bestModel; } private: From 5ac45dd7c208a762e3d0877e7863faddc5253f52 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Fri, 25 Aug 2017 10:07:58 +0500 Subject: [PATCH 27/27] Make CVFunction cache computations for gradient --- src/mlpack/core/hpt/cv_function_impl.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/hpt/cv_function_impl.hpp b/src/mlpack/core/hpt/cv_function_impl.hpp index c8bb36b6ddd..d6e46784751 100644 --- a/src/mlpack/core/hpt/cv_function_impl.hpp +++ b/src/mlpack/core/hpt/cv_function_impl.hpp @@ -76,12 +76,13 @@ void CVFunction::Gradient( { gradient = arma::mat(arma::size(parameters)); arma::mat increasedParameters = parameters; + double originalParametersEvaluation = Evaluate(parameters); for (size_t i = 0; i < parameters.n_rows; ++i) { double delta = std::max(std::abs(parameters(i)) * relativeDelta, minDelta); increasedParameters(i) += delta; - gradient(i) = (Evaluate(increasedParameters) - Evaluate(parameters)) / - delta; + gradient(i) = + (Evaluate(increasedParameters) - originalParametersEvaluation) / delta; increasedParameters(i) = parameters(i); } }