Add a hyper-parameter tuning module

src/mlpack/core/CMakeLists.txt

@@ -5,6 +5,7 @@ set(DIRS
   cv
   data
   dists
+  hpt
   kernels
   math
   metrics

src/mlpack/core/hpt/CMakeLists.txt

@@ -0,0 +1,15 @@
+set(SOURCES
+  cv_function.hpp
+  cv_function_impl.hpp
+  deduce_hp_types.hpp
+  fixed.hpp
+  hpt.hpp
+  hpt_impl.hpp
+)
+
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)

src/mlpack/core/hpt/cv_function.hpp

@@ -0,0 +1,172 @@
+/**
+ * @file cv_function.hpp
+ * @author Kirill Mishchenko
+ *
+ * A cross-validation wrapper for optimizers.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_HPT_CV_FUNCTION_HPP
+#define MLPACK_CORE_HPT_CV_FUNCTION_HPP
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace hpt {
+
+/**
+ * This wrapper serves for adapting the interface of the cross-validation
+ * classes to the one that can be utilized by the mlpack optimizers.
+ *
+ * This class is not supposed to be used directly by users. To tune
+ * hyper-parameters see HyperParameterTuner.
+ *
+ * @tparam CVType A cross-validation strategy.
+ * @tparam MLAlgorithm The machine learning algorithm used in cross-validation.
+ * @tparam TotalArgs The total number of arguments that are supposed to be
+ *     passed to the Evaluate method of a CVType object.
+ * @tparam BoundArgs Types of arguments (wrapped into the BoundArg struct) that
+ *     should be passed into the Evaluate method of a CVType object but are not
+ *     going to be passed into the Evaluate method of a CVFunction object.
+ */
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+class CVFunction
+{
+ public:
+  /**
+   * Initialize a CVFunction object.
+   *
+   * @param cv A cross-validation object.
+   * @param relativeDelta Relative increase of arguments for calculation of
+   *     partial derivatives (by the definition). The exact increase for some
+   *     particular argument is equal to the absolute value of the argument
+   *     multiplied by the relative increase (see also the documentation for the
+   *     minDelta parameter).
+   * @param minDelta Minimum increase of arguments for calculation of partial
+   *     derivatives (by the definition). This value is going to be used when it
+   *     is greater than the increase calculated with the rules described in the
+   *     documentation for the relativeDelta parameter.
+   * @param BoundArgs Arguments that should be passed into the Evaluate method
+   *     of the CVType object but are not going to be passed into the Evaluate
+   *     method of this object.
+   */
+  CVFunction(CVType& cv,
+             const double relativeDelta,
+             const double minDelta,
+             const BoundArgs&... args);
+
+  /**
+   * Run cross-validation with the bound and passed parameters.
+   *
+   * @param parameters Arguments (rather than the bound arguments) that should
+   *     be passed into the Evaluate method of the CVType object.
+   */
+  double Evaluate(const arma::mat& parameters);
+
+  /**
+   * Evaluate numerically the gradient of the CVFunction with the given
+   * parameters.
+   *
+   * @param parameters Arguments (rather than the bound arguments) that should
+   *     be passed into the Evaluate method of the CVType object.
+   * @param gradient Vector to output the gradient into.
+   */
+  void Gradient(const arma::mat& parameters, arma::mat& gradient);
+
+  //! Access and modify the best model so far.
+  MLAlgorithm& BestModel() { return bestModel; }
+
+ private:
+  //! The type of tuples of BoundArgs.
+  using BoundArgsTupleType = std::tuple<BoundArgs...>;
+
+  //! The amount of bound arguments.
+  static const size_t BoundArgsAmount =
+      std::tuple_size<BoundArgsTupleType>::value;
+
+  /**
+   * A struct that finds out whether the next argument for the Evaluate method
+   * of a CVType object should be a bound argument at the position BoundArgIndex
+   * rather than an element of parameters at the position ParamIndex.
+   */
+  template<size_t BoundArgIndex,
+           size_t ParamIndex,
+           bool BoundArgsIndexInRange = BoundArgIndex < BoundArgsAmount>
+  struct UseBoundArg;
+
+  //! A reference to the cross-validation object.
+  CVType& cv;
+
+  //! The bound arguments.
+  BoundArgsTupleType boundArgs;
+
+  //! The best objective so far.
+  double bestObjective;
+
+  //! The best model so far.
+  MLAlgorithm bestModel;
+
+  //! Relative increase of arguments for calculation of gradient.
+  double relativeDelta;
+
+  //! Minimum absolute increase of arguments for calculation of gradient.
+  double minDelta;
+
+  /**
+   * Collect all arguments and run cross-validation.
+   */
+  template<size_t BoundArgIndex,
+           size_t ParamIndex,
+           typename... Args,
+           typename = typename
+               std::enable_if<BoundArgIndex + ParamIndex < TotalArgs>::type>
+  inline double Evaluate(const arma::mat& parameters, const Args&... args);
+
+  /**
+   * Run cross-validation with the collected arguments.
+   */
+  template<size_t BoundArgIndex,
+           size_t ParamIndex,
+           typename... Args,
+           typename = typename
+               std::enable_if<BoundArgIndex + ParamIndex == TotalArgs>::type,
+           typename = void>
+  inline double Evaluate(const arma::mat& parameters, const Args&... args);
+
+  /**
+   * Put the bound argument (at the BoundArgIndex position) as the next one.
+   */
+  template<size_t BoundArgIndex,
+           size_t ParamIndex,
+           typename... Args,
+           typename = typename std::enable_if<
+               UseBoundArg<BoundArgIndex, ParamIndex>::value>::type>
+  inline double PutNextArg(const arma::mat& parameters, const Args&... args);
+
+  /**
+   * Put the element (at the ParamIndex position) of the parameters as the next
+   * one.
+   */
+  template<size_t BoundArgIndex,
+           size_t ParamIndex,
+           typename... Args,
+           typename = typename std::enable_if<
+               !UseBoundArg<BoundArgIndex, ParamIndex>::value>::type,
+           typename = void>
+  inline double PutNextArg(const arma::mat& parameters, const Args&... args);
+};
+
+
+} // namespace hpt
+} // namespace mlpack
+
+// Include implementation
+#include "cv_function_impl.hpp"
+
+#endif

src/mlpack/core/hpt/cv_function_impl.hpp

@@ -0,0 +1,168 @@
+/**
+ * @file cv_function_impl.hpp
+ * @author Kirill Mishchenko
+ *
+ * The implementation of the class CVFunction.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_HPT_CV_FUNCTION_IMPL_HPP
+#define MLPACK_CORE_HPT_CV_FUNCTION_IMPL_HPP
+
+namespace mlpack {
+namespace hpt {
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+template<size_t BoundArgIndex, size_t ParamIndex>
+struct CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::UseBoundArg<
+    BoundArgIndex, ParamIndex, true>
+{
+  using BoundArgType =
+      typename std::tuple_element<BoundArgIndex, BoundArgsTupleType>::type;
+
+  static const bool value = BoundArgType::index == BoundArgIndex + ParamIndex;
+};
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+template<size_t BoundArgIndex, size_t ParamIndex>
+struct CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::UseBoundArg<
+    BoundArgIndex, ParamIndex, false>
+{
+  static const bool value = false;
+};
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::CVFunction(
+    CVType& cv,
+    const double relativeDelta,
+    const double minDelta,
+    const BoundArgs&... args) :
+    cv(cv),
+    boundArgs(args...),
+    bestObjective(std::numeric_limits<double>::max()),
+    relativeDelta(relativeDelta),
+    minDelta(minDelta)
+{ /* Nothing left to do. */ }
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+double CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::Evaluate(
+    const arma::mat& parameters)
+{
+  return Evaluate<0, 0>(parameters);
+}
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+void CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::Gradient(
+    const arma::mat& parameters,
+    arma::mat& gradient)
+{
+  gradient = arma::mat(arma::size(parameters));
+  arma::mat increasedParameters = parameters;
+  double originalParametersEvaluation = Evaluate(parameters);
+  for (size_t i = 0; i < parameters.n_rows; ++i)
+  {
+    double delta = std::max(std::abs(parameters(i)) * relativeDelta, minDelta);
+    increasedParameters(i) += delta;
+    gradient(i) =
+        (Evaluate(increasedParameters) - originalParametersEvaluation) / delta;
+    increasedParameters(i) = parameters(i);
+  }
+}
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+template<size_t BoundArgIndex,
+         size_t ParamIndex,
+         typename... Args,
+         typename>
+double CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::Evaluate(
+    const arma::mat& parameters,
+    const Args&... args)
+{
+  return PutNextArg<BoundArgIndex, ParamIndex>(parameters, args...);
+}
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+template<size_t BoundArgIndex,
+         size_t ParamIndex,
+         typename... Args,
+         typename,
+         typename>
+double CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::Evaluate(
+    const arma::mat& /* parameters */,
+    const Args&... args)
+{
+  double objective = cv.Evaluate(args...);
+
+  // Change the best model if we have got a better score, or if we probably
+  // have not assigned any valid (trained) model yet.
+  if (bestObjective > objective ||
+      bestObjective == std::numeric_limits<double>::max())
+  {
+    bestObjective = objective;
+    bestModel = std::move(cv.Model());
+  }
+
+  return objective;
+}
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+template<size_t BoundArgIndex,
+         size_t ParamIndex,
+         typename... Args,
+         typename>
+double CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::PutNextArg(
+    const arma::mat& parameters,
+    const Args&... args)
+{
+  return Evaluate<BoundArgIndex + 1, ParamIndex>(
+      parameters, args..., std::get<BoundArgIndex>(boundArgs).value);
+}
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+template<size_t BoundArgIndex,
+         size_t ParamIndex,
+         typename... Args,
+         typename,
+         typename>
+double CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::PutNextArg(
+    const arma::mat& parameters,
+    const Args&... args)
+{
+  return Evaluate<BoundArgIndex, ParamIndex + 1>(
+      parameters, args..., parameters(ParamIndex, 0));
+}
+
+} // namespace hpt
+} // namespace mlpack
+
+#endif