diff --git a/src/mlpack/core/CMakeLists.txt b/src/mlpack/core/CMakeLists.txt
index 6d9194cc2f7..bdc15a79b68 100644
--- a/src/mlpack/core/CMakeLists.txt
+++ b/src/mlpack/core/CMakeLists.txt
@@ -5,6 +5,7 @@ set(DIRS
   cv
   data
   dists
+  hpt
   kernels
   math
   metrics
diff --git a/src/mlpack/core/hpt/CMakeLists.txt b/src/mlpack/core/hpt/CMakeLists.txt
new file mode 100644
index 00000000000..c39f9b674e3
--- /dev/null
+++ b/src/mlpack/core/hpt/CMakeLists.txt
@@ -0,0 +1,15 @@
+set(SOURCES
+  cv_function.hpp
+  cv_function_impl.hpp
+  deduce_hp_types.hpp
+  fixed.hpp
+  hpt.hpp
+  hpt_impl.hpp
+)
+
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
diff --git a/src/mlpack/core/hpt/cv_function.hpp b/src/mlpack/core/hpt/cv_function.hpp
new file mode 100644
index 00000000000..3430e2ea063
--- /dev/null
+++ b/src/mlpack/core/hpt/cv_function.hpp
@@ -0,0 +1,172 @@
+/**
+ * @file cv_function.hpp
+ * @author Kirill Mishchenko
+ *
+ * A cross-validation wrapper for optimizers.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_HPT_CV_FUNCTION_HPP
+#define MLPACK_CORE_HPT_CV_FUNCTION_HPP
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace hpt {
+
+/**
+ * This wrapper serves for adapting the interface of the cross-validation
+ * classes to the one that can be utilized by the mlpack optimizers.
+ *
+ * This class is not supposed to be used directly by users. To tune
+ * hyper-parameters see HyperParameterTuner.
+ *
+ * @tparam CVType A cross-validation strategy.
+ * @tparam MLAlgorithm The machine learning algorithm used in cross-validation.
+ * @tparam TotalArgs The total number of arguments that are supposed to be
+ *     passed to the Evaluate method of a CVType object.
+ * @tparam BoundArgs Types of arguments (wrapped into the BoundArg struct) that
+ *     should be passed into the Evaluate method of a CVType object but are not
+ *     going to be passed into the Evaluate method of a CVFunction object.
+ */
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+class CVFunction
+{
+ public:
+  /**
+   * Initialize a CVFunction object.
+   *
+   * @param cv A cross-validation object.
+   * @param relativeDelta Relative increase of arguments for calculation of
+   *     partial derivatives (by the definition). The exact increase for some
+   *     particular argument is equal to the absolute value of the argument
+   *     multiplied by the relative increase (see also the documentation for the
+   *     minDelta parameter).
+   * @param minDelta Minimum increase of arguments for calculation of partial
+   *     derivatives (by the definition). This value is going to be used when it
+   *     is greater than the increase calculated with the rules described in the
+   *     documentation for the relativeDelta parameter.
+   * @param BoundArgs Arguments that should be passed into the Evaluate method
+   *     of the CVType object but are not going to be passed into the Evaluate
+   *     method of this object.
+   */
+  CVFunction(CVType& cv,
+             const double relativeDelta,
+             const double minDelta,
+             const BoundArgs&... args);
+
+  /**
+   * Run cross-validation with the bound and passed parameters.
+   *
+   * @param parameters Arguments (rather than the bound arguments) that should
+   *     be passed into the Evaluate method of the CVType object.
+   */
+  double Evaluate(const arma::mat& parameters);
+
+  /**
+   * Evaluate numerically the gradient of the CVFunction with the given
+   * parameters.
+   *
+   * @param parameters Arguments (rather than the bound arguments) that should
+   *     be passed into the Evaluate method of the CVType object.
+   * @param gradient Vector to output the gradient into.
+   */
+  void Gradient(const arma::mat& parameters, arma::mat& gradient);
+
+  //! Access and modify the best model so far.
+  MLAlgorithm& BestModel() { return bestModel; }
+
+ private:
+  //! The type of tuples of BoundArgs.
+  using BoundArgsTupleType = std::tuple<BoundArgs...>;
+
+  //! The amount of bound arguments.
+  static const size_t BoundArgsAmount =
+      std::tuple_size<BoundArgsTupleType>::value;
+
+  /**
+   * A struct that finds out whether the next argument for the Evaluate method
+   * of a CVType object should be a bound argument at the position BoundArgIndex
+   * rather than an element of parameters at the position ParamIndex.
+   */
+  template<size_t BoundArgIndex,
+           size_t ParamIndex,
+           bool BoundArgsIndexInRange = BoundArgIndex < BoundArgsAmount>
+  struct UseBoundArg;
+
+  //! A reference to the cross-validation object.
+  CVType& cv;
+
+  //! The bound arguments.
+  BoundArgsTupleType boundArgs;
+
+  //! The best objective so far.
+  double bestObjective;
+
+  //! The best model so far.
+  MLAlgorithm bestModel;
+
+  //! Relative increase of arguments for calculation of gradient.
+  double relativeDelta;
+
+  //! Minimum absolute increase of arguments for calculation of gradient.
+  double minDelta;
+
+  /**
+   * Collect all arguments and run cross-validation.
+   */
+  template<size_t BoundArgIndex,
+           size_t ParamIndex,
+           typename... Args,
+           typename = typename
+               std::enable_if<BoundArgIndex + ParamIndex < TotalArgs>::type>
+  inline double Evaluate(const arma::mat& parameters, const Args&... args);
+
+  /**
+   * Run cross-validation with the collected arguments.
+   */
+  template<size_t BoundArgIndex,
+           size_t ParamIndex,
+           typename... Args,
+           typename = typename
+               std::enable_if<BoundArgIndex + ParamIndex == TotalArgs>::type,
+           typename = void>
+  inline double Evaluate(const arma::mat& parameters, const Args&... args);
+
+  /**
+   * Put the bound argument (at the BoundArgIndex position) as the next one.
+   */
+  template<size_t BoundArgIndex,
+           size_t ParamIndex,
+           typename... Args,
+           typename = typename std::enable_if<
+               UseBoundArg<BoundArgIndex, ParamIndex>::value>::type>
+  inline double PutNextArg(const arma::mat& parameters, const Args&... args);
+
+  /**
+   * Put the element (at the ParamIndex position) of the parameters as the next
+   * one.
+   */
+  template<size_t BoundArgIndex,
+           size_t ParamIndex,
+           typename... Args,
+           typename = typename std::enable_if<
+               !UseBoundArg<BoundArgIndex, ParamIndex>::value>::type,
+           typename = void>
+  inline double PutNextArg(const arma::mat& parameters, const Args&... args);
+};
+
+
+} // namespace hpt
+} // namespace mlpack
+
+// Include implementation
+#include "cv_function_impl.hpp"
+
+#endif
diff --git a/src/mlpack/core/hpt/cv_function_impl.hpp b/src/mlpack/core/hpt/cv_function_impl.hpp
new file mode 100644
index 00000000000..d6e46784751
--- /dev/null
+++ b/src/mlpack/core/hpt/cv_function_impl.hpp
@@ -0,0 +1,168 @@
+/**
+ * @file cv_function_impl.hpp
+ * @author Kirill Mishchenko
+ *
+ * The implementation of the class CVFunction.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_HPT_CV_FUNCTION_IMPL_HPP
+#define MLPACK_CORE_HPT_CV_FUNCTION_IMPL_HPP
+
+namespace mlpack {
+namespace hpt {
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+template<size_t BoundArgIndex, size_t ParamIndex>
+struct CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::UseBoundArg<
+    BoundArgIndex, ParamIndex, true>
+{
+  using BoundArgType =
+      typename std::tuple_element<BoundArgIndex, BoundArgsTupleType>::type;
+
+  static const bool value = BoundArgType::index == BoundArgIndex + ParamIndex;
+};
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+template<size_t BoundArgIndex, size_t ParamIndex>
+struct CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::UseBoundArg<
+    BoundArgIndex, ParamIndex, false>
+{
+  static const bool value = false;
+};
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::CVFunction(
+    CVType& cv,
+    const double relativeDelta,
+    const double minDelta,
+    const BoundArgs&... args) :
+    cv(cv),
+    boundArgs(args...),
+    bestObjective(std::numeric_limits<double>::max()),
+    relativeDelta(relativeDelta),
+    minDelta(minDelta)
+{ /* Nothing left to do. */ }
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+double CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::Evaluate(
+    const arma::mat& parameters)
+{
+  return Evaluate<0, 0>(parameters);
+}
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+void CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::Gradient(
+    const arma::mat& parameters,
+    arma::mat& gradient)
+{
+  gradient = arma::mat(arma::size(parameters));
+  arma::mat increasedParameters = parameters;
+  double originalParametersEvaluation = Evaluate(parameters);
+  for (size_t i = 0; i < parameters.n_rows; ++i)
+  {
+    double delta = std::max(std::abs(parameters(i)) * relativeDelta, minDelta);
+    increasedParameters(i) += delta;
+    gradient(i) =
+        (Evaluate(increasedParameters) - originalParametersEvaluation) / delta;
+    increasedParameters(i) = parameters(i);
+  }
+}
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+template<size_t BoundArgIndex,
+         size_t ParamIndex,
+         typename... Args,
+         typename>
+double CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::Evaluate(
+    const arma::mat& parameters,
+    const Args&... args)
+{
+  return PutNextArg<BoundArgIndex, ParamIndex>(parameters, args...);
+}
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+template<size_t BoundArgIndex,
+         size_t ParamIndex,
+         typename... Args,
+         typename,
+         typename>
+double CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::Evaluate(
+    const arma::mat& /* parameters */,
+    const Args&... args)
+{
+  double objective = cv.Evaluate(args...);
+
+  // Change the best model if we have got a better score, or if we probably
+  // have not assigned any valid (trained) model yet.
+  if (bestObjective > objective ||
+      bestObjective == std::numeric_limits<double>::max())
+  {
+    bestObjective = objective;
+    bestModel = std::move(cv.Model());
+  }
+
+  return objective;
+}
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+template<size_t BoundArgIndex,
+         size_t ParamIndex,
+         typename... Args,
+         typename>
+double CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::PutNextArg(
+    const arma::mat& parameters,
+    const Args&... args)
+{
+  return Evaluate<BoundArgIndex + 1, ParamIndex>(
+      parameters, args..., std::get<BoundArgIndex>(boundArgs).value);
+}
+
+template<typename CVType,
+         typename MLAlgorithm,
+         size_t TotalArgs,
+         typename... BoundArgs>
+template<size_t BoundArgIndex,
+         size_t ParamIndex,
+         typename... Args,
+         typename,
+         typename>
+double CVFunction<CVType, MLAlgorithm, TotalArgs, BoundArgs...>::PutNextArg(
+    const arma::mat& parameters,
+    const Args&... args)
+{
+  return Evaluate<BoundArgIndex, ParamIndex + 1>(
+      parameters, args..., parameters(ParamIndex, 0));
+}
+
+} // namespace hpt
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/core/hpt/deduce_hp_types.hpp b/src/mlpack/core/hpt/deduce_hp_types.hpp
new file mode 100644
index 00000000000..a1e45a7adca
--- /dev/null
+++ b/src/mlpack/core/hpt/deduce_hp_types.hpp
@@ -0,0 +1,132 @@
+/**
+ * @file deduce_hp_types.hpp
+ * @author Kirill Mishchenko
+ *
+ * Tools to deduce types of hyper-parameters from types of arguments in the
+ * Optimize method in HyperParameterTuner.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_HPT_DEDUCE_HP_TYPES_HPP
+#define MLPACK_CORE_HPT_DEDUCE_HP_TYPES_HPP
+
+#include <mlpack/core/hpt/fixed.hpp>
+
+namespace mlpack {
+namespace hpt {
+
+/**
+ * A type function for deducing types of hyper-parameters from types of
+ * arguments in the Optimize method in HyperParameterTuner.
+ *
+ * We start by putting all types of the arguments into Args, and then process
+ * each of them one by one and put results into the internal struct
+ * ResultHolder. By the end Args become empty, while ResultHolder holds the
+ * tuple type of hyper-parameters.
+ *
+ * Here we declare and define DeduceHyperParameterTypes for the end phase when
+ * Args are empty (all argument types have been processed).
+ */
+template<typename... Args>
+struct DeduceHyperParameterTypes
+{
+  template<typename... HPTypes>
+  struct ResultHolder
+  {
+    using TupleType = std::tuple<HPTypes...>;
+  };
+};
+
+/**
+ * Defining DeduceHyperParameterTypes for the case when not all argument types
+ * have been processed, and the next one (T) is a collection type or an
+ * arithmetic type.
+ */
+template<typename T, typename... Args>
+struct DeduceHyperParameterTypes<T, Args...>
+{
+  /**
+   * A type function to deduce the result hyper-parameter type for ArgumentType.
+   */
+  template<typename ArgumentType,
+           bool IsArithmetic = std::is_arithmetic<ArgumentType>::value>
+  struct ResultHPType;
+
+  template<typename ArithmeticType>
+  struct ResultHPType<ArithmeticType, true>
+  {
+    using Type = ArithmeticType;
+  };
+
+  /**
+   * A type function to check whether Type is a collection type (for that it
+   * should define value_type).
+   */
+  template<typename Type>
+  struct IsCollectionType
+  {
+    using Yes = char[1];
+    using No = char[2];
+
+    template<typename TypeToCheck>
+    static Yes& Check(typename TypeToCheck::value_type*);
+    template<typename>
+    static No& Check(...);
+
+    static const bool value  =
+      sizeof(decltype(Check<Type>(0))) == sizeof(Yes);
+  };
+
+  template<typename CollectionType>
+  struct ResultHPType<CollectionType, false>
+  {
+    static_assert(IsCollectionType<CollectionType>::value,
+        "One of the passed arguments is neither of an arithmetic type, nor of "
+        "a collection type, nor fixed with the Fixed function.");
+
+    using Type = typename CollectionType::value_type;
+  };
+
+  template<typename... HPTypes>
+  struct ResultHolder
+  {
+    using TupleType = typename DeduceHyperParameterTypes<Args...>::template
+        ResultHolder<HPTypes..., typename ResultHPType<T>::Type>::TupleType;
+  };
+
+  using TupleType = typename ResultHolder<>::TupleType;
+};
+
+/**
+ * Defining DeduceHyperParameterTypes for the case when not all argument types
+ * have been processed, and the next one is the type of an argument that should
+ * be fixed.
+ */
+template<typename T, typename... Args>
+struct DeduceHyperParameterTypes<PreFixedArg<T>, Args...>
+{
+  template<typename... HPTypes>
+  struct ResultHolder
+  {
+    using TupleType = typename DeduceHyperParameterTypes<Args...>::template
+        ResultHolder<HPTypes...>::TupleType;
+  };
+
+  using TupleType = typename ResultHolder<>::TupleType;
+};
+
+/**
+ * A short alias for deducing types of hyper-parameters from types of arguments
+ * in the Optimize method in HyperParameterTuner.
+ */
+template<typename... Args>
+using TupleOfHyperParameters =
+    typename DeduceHyperParameterTypes<Args...>::TupleType;
+
+} // namespace hpt
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/core/hpt/fixed.hpp b/src/mlpack/core/hpt/fixed.hpp
new file mode 100644
index 00000000000..122df0eef36
--- /dev/null
+++ b/src/mlpack/core/hpt/fixed.hpp
@@ -0,0 +1,111 @@
+/**
+ * @file fixed.hpp
+ * @author Kirill Mishchenko
+ *
+ * Facilities for supporting fixed arguments.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_HPT_FIXED_HPP
+#define MLPACK_CORE_HPT_FIXED_HPP
+
+#include <type_traits>
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace hpt {
+
+template<typename>
+struct PreFixedArg;
+
+/**
+ * Mark the given argument as one that should be fixed. It can be applied to
+ * arguments that are passed to the Optimize method of HyperParameterTuner.
+ *
+ * The implementation avoids data copying. If the passed argument is an l-value
+ * reference, we store it as a const l-value rerefence inside the returned
+ * PreFixedArg object. If the passed argument is an r-value reference,
+ * ligth-weight coping (by taking possesion of the r-value) will be made during
+ * the initialization of the returned PreFixedArg object.
+ */
+template<typename T>
+PreFixedArg<T> Fixed(T&& value)
+{
+  return PreFixedArg<T>{std::forward<T>(value)};
+}
+
+/**
+ * A struct for storing information about a fixed argument. Objects of this type
+ * are supposed to be passed into the CVFunction constructor.
+ *
+ * This struct is not meant to be used directly by users. Rather use the
+ * mlpack::hpt::Fixed function.
+ *
+ * @tparam T The type of the fixed argument.
+ * @tparam I The index of the fixed argument.
+ */
+template<typename T, size_t I>
+struct FixedArg
+{
+  //! The index of the fixed argument.
+  static const size_t index = I;
+
+  //! The value of the fixed argument.
+  const T& value;
+};
+
+/**
+ * A struct for marking arguments as ones that should be fixed (it can be useful
+ * for the Optimize method of HyperParameterTuner). Arguments of this type are
+ * supposed to be converted into structs of the type FixedArg by adding
+ * information about argument positions.
+ *
+ * This struct is not meant to be used directly by users. Rather use the
+ * mlpack::hpt::Fixed function.
+ */
+template<typename T>
+struct PreFixedArg
+{
+  using Type = T;
+
+  const T value;
+};
+
+/**
+ * The specialization of the template for references.
+ *
+ * This struct is not meant to be used directly by users. Rather use the
+ * mlpack::hpt::Fixed function.
+ */
+template<typename T>
+struct PreFixedArg<T&>
+{
+  using Type = T;
+
+  const T& value;
+};
+
+/**
+ * A type function for checking whether the given type is PreFixedArg.
+ */
+template<typename T>
+class IsPreFixedArg
+{
+  template<typename>
+  struct Implementation : std::false_type {};
+
+  template<typename Type>
+  struct Implementation<PreFixedArg<Type>> : std::true_type {};
+
+ public:
+  static const bool value = Implementation<typename std::decay<T>::type>::value;
+};
+
+} // namespace hpt
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/core/hpt/hpt.hpp b/src/mlpack/core/hpt/hpt.hpp
new file mode 100644
index 00000000000..fb5dc306773
--- /dev/null
+++ b/src/mlpack/core/hpt/hpt.hpp
@@ -0,0 +1,348 @@
+/**
+ * @file hpt.hpp
+ * @author Kirill Mishchenko
+ *
+ * Hyper-parameter tuning.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_HPT_HPT_HPP
+#define MLPACK_CORE_HPT_HPT_HPP
+
+#include <mlpack/core/cv/meta_info_extractor.hpp>
+#include <mlpack/core/hpt/deduce_hp_types.hpp>
+#include <mlpack/core/optimizers/grid_search/grid_search.hpp>
+
+namespace mlpack {
+namespace hpt {
+
+/**
+ * The class HyperParameterTuner for the given MLAlgorithm utilizes the provided
+ * Optimizer to find the values of hyper-parameters that optimize the value of
+ * the given Metric. The value of the Metric is calculated by performing
+ * cross-validation with the provided cross-validation strategy.
+ *
+ * To construct a HyperParameterTuner object you need to pass the same arguments
+ * as for construction of an object of the given CV class. For example, we can
+ * use the following code to try to find a good lambda value for
+ * LinearRegression.
+ *
+ * @code
+ * // 100-point 5-dimensional random dataset.
+ * arma::mat data = arma::randu<arma::mat>(5, 100);
+ * // Noisy responses retrieved by a random linear transformation of data.
+ * arma::rowvec responses = arma::randu<arma::rowvec>(5) * data +
+ *     0.1 * arma::randn<arma::rowvec>(100);
+ *
+ * // Using 80% of data for training and remaining 20% for assessing MSE.
+ * double validationSize = 0.2;
+ * HyperParameterTuner<LinearRegression, MSE, SimpleCV> hpt(validationSize,
+ *     data, responses);
+ *
+ * // Finding the best value for lambda from the values 0.0, 0.001, 0.01, 0.1,
+ * // and 1.0.
+ * arma::vec lambdas{0.0, 0.001, 0.01, 0.1, 1.0};
+ * double bestLambda;
+ * std::tie(bestLambda) = hpt.Optimize(lambdas);
+ * @endcode
+ *
+ * When some hyper-parameters should not be optimized, you can specify values
+ * for them with the Fixed function as in the following example of finding good
+ * lambda1 and lambda2 values for LARS.
+ *
+ * @code
+ * HyperParameterTuner<LARS, MSE, SimpleCV> hpt2(validationSize, data,
+ *     responses);
+ *
+ * bool transposeData = true;
+ * bool useCholesky = false;
+ * arma::vec lambda1Set{0.0, 0.001, 0.01, 0.1, 1.0};
+ * arma::vec lambda2Set{0.0, 0.002, 0.02, 0.2, 2.0};
+ *
+ * double bestLambda1, bestLambda2;
+ * std::tie(bestLambda1, bestLambda2) = hpt2.Optimize(Fixed(transposeData),
+ *     Fixed(useCholesky), lambda1Set, lambda2Set);
+ * @endcode
+ *
+ * @tparam MLAlgorithm A machine learning algorithm.
+ * @tparam Metric A metric to assess the quality of a trained model.
+ * @tparam CV A cross-validation strategy used to assess a set of
+ *     hyper-parameters.
+ * @tparam OptimizerType An optimization strategy (GridSearch and
+ *     GradientDescent are supported).
+ * @tparam MatType The type of data.
+ * @tparam PredictionsType The type of predictions (should be passed when the
+ *     predictions type is a template parameter in Train methods of the given
+ *     MLAlgorithm; arma::Row<size_t> will be used otherwise).
+ * @tparam WeightsType The type of weights (should be passed when weighted
+ *     learning is supported, and the weights type is a template parameter in
+ *     Train methods of the given MLAlgorithm; arma::vec will be used
+ *     otherwise).
+ */
+template<typename MLAlgorithm,
+         typename Metric,
+         template<typename, typename, typename, typename, typename> class CV,
+         typename OptimizerType = mlpack::optimization::GridSearch,
+         typename MatType = arma::mat,
+         typename PredictionsType =
+             typename cv::MetaInfoExtractor<MLAlgorithm,
+                 MatType>::PredictionsType,
+         typename WeightsType =
+             typename cv::MetaInfoExtractor<MLAlgorithm, MatType,
+                 PredictionsType>::WeightsType>
+class HyperParameterTuner
+{
+ public:
+  /**
+   * Create a HyperParameterTuner object by passing constructor arguments for
+   * the given cross-validation strategy (the CV class).
+   *
+   * @param args Constructor arguments for the given cross-validation
+   *     strategy (the CV class).
+   */
+  template<typename... CVArgs>
+  HyperParameterTuner(const CVArgs& ...args);
+
+  //! Access and modify the optimizer.
+  OptimizerType& Optimizer() { return optimizer; }
+
+  /**
+   * Get relative increase of arguments for calculation of partial
+   * derivatives (by the definition) in gradient-based optimization. The exact
+   * increase for some particular argument is equal to the absolute value of the
+   * argument multiplied by the relative increase (see also the documentation
+   * for MinDelta()).
+   */
+  double RelativeDelta() const { return relativeDelta; }
+
+  /**
+   * Modify relative increase of arguments for calculation of partial
+   * derivatives (by the definition) in gradient-based optimization. The exact
+   * increase for some particular argument is equal to the absolute value of the
+   * argument multiplied by the relative increase (see also the documentation
+   * for MinDelta()).
+   */
+  double& RelativeDelta() { return relativeDelta; }
+
+  /**
+   * Get minimum increase of arguments for calculation of partial derivatives
+   * (by the definition) in gradient-based optimization. This value is going to
+   * be used when it is greater than the increase calculated with the rules
+   * described in the documentation for RelativeDelta().
+   */
+  double MinDelta() const { return minDelta; }
+
+  /**
+   * Modify minimum increase of arguments for calculation of partial derivatives
+   * (by the definition) in gradient-based optimization. This value is going to
+   * be used when it is greater than the increase calculated with the rules
+   * described in the documentation for RelativeDelta().
+   */
+  double& MinDelta() { return minDelta; }
+
+  /**
+   * Find the best hyper-parameters by using the given Optimizer. For each
+   * hyper-parameter one of the following should be passed as an argument.
+   * 1. A set of values to choose from (when using GridSearch as an optimizer).
+   *   The set of values should be an STL-compatible container (it should
+   *   provide begin() and end() methods returning iterators).
+   * 2. A starting value (when using any other optimizer than GridSearch).
+   * 3. A value fixed by using the function mlpack::hpt::Fixed. In this case the
+   *   hyper-parameter will not be optimized.
+   *
+   * All arguments should be passed in the same order as if the corresponding
+   * hyper-parameters would be passed into the Evaluate method of the given CV
+   * class (in the order as they appear in the constructor(s) of the given
+   * MLAlgorithm). Also, arguments for all required hyper-parameters (ones that
+   * don't have default values in the corresponding MLAlgorithm constructor)
+   * should be provided.
+   *
+   * The method returns a tuple of values for hyper-parameters that haven't been
+   * fixed.
+   *
+   * @param args Arguments corresponding to hyper-parameters (see the method
+   *   description for more information).
+   */
+  template<typename... Args>
+  TupleOfHyperParameters<Args...> Optimize(const Args&... args);
+
+  //! Get the performance measurement of the best model from the last run.
+  double BestObjective() const { return bestObjective; }
+
+  //! Get the best model from the last run.
+  const MLAlgorithm& BestModel() const { return bestModel; }
+
+  //! Modify the best model from the last run.
+  MLAlgorithm& BestModel() { return bestModel; }
+
+ private:
+  /**
+   * A decorator that returns negated values of the original metric.
+   */
+  template<typename OriginalMetric>
+  struct Negated
+  {
+    static double Evaluate(MLAlgorithm& model,
+                           const MatType& xs,
+                           const PredictionsType& ys)
+    { return -OriginalMetric::Evaluate(model, xs, ys); }
+  };
+
+  //! A short alias for the full type of the cross-validation.
+  using CVType = typename std::conditional<Metric::NeedsMinimization,
+      CV<MLAlgorithm, Metric, MatType, PredictionsType, WeightsType>,
+      CV<MLAlgorithm, Negated<Metric>, MatType, PredictionsType,
+          WeightsType>>::type;
+
+
+  //! The cross-validation object for assessing sets of hyper-parameters.
+  CVType cv;
+
+  //! The optimizer.
+  OptimizerType optimizer;
+
+  //! The best objective from the last run.
+  double bestObjective;
+
+  //! The best model from the last run.
+  MLAlgorithm bestModel;
+
+  /**
+   * The relative increase of arguments for calculation of gradient in
+   * CVFunction.
+   */
+  double relativeDelta;
+
+  /**
+   * The minimum increase of arguments for calculation of gradient in
+   * CVFunction.
+   */
+  double minDelta;
+
+  /**
+   * A type function to check whether the element I of the tuple type is a
+   * PreFixedArg.
+   */
+  template<typename Tuple, size_t I>
+  using IsPreFixed = IsPreFixedArg<typename std::tuple_element<I, Tuple>::type>;
+
+  /**
+   * A type function to check whether the element I of the tuple type is an
+   * arithmetic type.
+   */
+  template<typename Tuple, size_t I>
+  using IsArithmetic = std::is_arithmetic<typename std::remove_reference<
+      typename std::tuple_element<I, Tuple>::type>::type>;
+
+  /**
+   * The set of methods to initialize auxiliary objects (a CVFunction object and
+   * the datasetInfo parameter) and run optimization to find the best
+   * hyper-parameters.
+   *
+   * This template is called when we are ready to run optimization.
+   */
+  template<size_t I /* Index of the next argument to handle. */,
+           typename ArgsTuple,
+           typename... FixedArgs,
+           typename = std::enable_if_t<I == std::tuple_size<ArgsTuple>::value>>
+  inline void InitAndOptimize(
+      const ArgsTuple& args,
+      arma::mat& bestParams,
+      data::DatasetMapper<data::IncrementPolicy, double>& datasetInfo,
+      FixedArgs... fixedArgs);
+
+  /**
+   * The set of methods to initialize auxiliary objects (a CVFunction object and
+   * the datasetInfo parameter) and run optimization to find the best
+   * hyper-parameters.
+   *
+   * This template is called when the next argument should be fixed (should not
+   * be optimized).
+   */
+  template<size_t I /* Index of the next argument to handle. */,
+           typename ArgsTuple,
+           typename... FixedArgs,
+           typename = std::enable_if_t<I < std::tuple_size<ArgsTuple>::value>,
+           typename = std::enable_if_t<IsPreFixed<ArgsTuple, I>::value>>
+  inline void InitAndOptimize(
+      const ArgsTuple& args,
+      arma::mat& bestParams,
+      data::DatasetMapper<data::IncrementPolicy, double>& datasetInfo,
+      FixedArgs... fixedArgs);
+
+  /**
+   * The set of methods to initialize auxiliary objects (a CVFunction object and
+   * the datasetInfo parameter) and run optimization to find the best
+   * hyper-parameters.
+   *
+   * This template is called when the next argument is of an arithmetic type and
+   * should be used as an initial value for the hyper-parameter.
+   */
+  template<size_t I /* Index of the next argument to handle. */,
+           typename ArgsTuple,
+           typename... FixedArgs,
+           typename = std::enable_if_t<I < std::tuple_size<ArgsTuple>::value>,
+           typename = std::enable_if_t<!IsPreFixed<ArgsTuple, I>::value &&
+                   IsArithmetic<ArgsTuple, I>::value>,
+           typename = void>
+  inline void InitAndOptimize(
+      const ArgsTuple& args,
+      arma::mat& bestParams,
+      data::DatasetMapper<data::IncrementPolicy, double>& datasetInfo,
+      FixedArgs... fixedArgs);
+
+  /**
+   * The set of methods to initialize auxiliary objects (a CVFunction object and
+   * the datasetInfo parameter) and run optimization to find the best
+   * hyper-parameters.
+   *
+   * This template is called when the next argument should be used to specify
+   * possible values for the hyper-parameter in datasetInfo.
+   */
+  template<size_t I /* Index of the next argument to handle. */,
+           typename ArgsTuple,
+           typename... FixedArgs,
+           typename = std::enable_if_t<I < std::tuple_size<ArgsTuple>::value>,
+           typename = std::enable_if_t<!IsPreFixed<ArgsTuple, I>::value &&
+                   !IsArithmetic<ArgsTuple, I>::value>,
+           typename = void,
+           typename = void>
+  inline void InitAndOptimize(
+      const ArgsTuple& args,
+      arma::mat& bestParams,
+      data::DatasetMapper<data::IncrementPolicy, double>& datasetInfo,
+      FixedArgs... fixedArgs);
+
+  /**
+   * Gather all elements of vector in an argument list and use them to create a
+   * tuple.
+   */
+  template<typename TupleType,
+           size_t I,
+           typename... Args,
+           typename = typename
+               std::enable_if_t<I < std::tuple_size<TupleType>::value>>
+  inline TupleType VectorToTuple(const arma::vec& vector, const Args&... args);
+
+  /**
+   * Create a tuple from args.
+   */
+  template<typename TupleType,
+           size_t I,
+           typename... Args,
+           typename = typename
+               std::enable_if_t<I == std::tuple_size<TupleType>::value>,
+           typename = void>
+  inline TupleType VectorToTuple(const arma::vec& vector, const Args&... args);
+};
+
+} // namespace hpt
+} // namespace mlpack
+
+// Include implementation
+#include "hpt_impl.hpp"
+
+#endif
diff --git a/src/mlpack/core/hpt/hpt_impl.hpp b/src/mlpack/core/hpt/hpt_impl.hpp
new file mode 100644
index 00000000000..36d4962795d
--- /dev/null
+++ b/src/mlpack/core/hpt/hpt_impl.hpp
@@ -0,0 +1,234 @@
+/**
+ * @file hpt_impl.hpp
+ * @author Kirill Mishchenko
+ *
+ * Implementation of hyper-parameter tuning.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_HPT_HPT_IMPL_HPP
+#define MLPACK_CORE_HPT_HPT_IMPL_HPP
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace hpt {
+
+template<typename MLAlgorithm,
+         typename Metric,
+         template<typename, typename, typename, typename, typename> class CV,
+         typename Optimizer,
+         typename MatType,
+         typename PredictionsType,
+         typename WeightsType>
+template<typename... CVArgs>
+HyperParameterTuner<MLAlgorithm,
+                    Metric,
+                    CV,
+                    Optimizer,
+                    MatType,
+                    PredictionsType,
+                    WeightsType>::HyperParameterTuner(const CVArgs&... args) :
+    cv(args...), relativeDelta(0.01), minDelta(1e-10) {}
+
+template<typename MLAlgorithm,
+         typename Metric,
+         template<typename, typename, typename, typename, typename> class CV,
+         typename Optimizer,
+         typename MatType,
+         typename PredictionsType,
+         typename WeightsType>
+template<typename... Args>
+TupleOfHyperParameters<Args...> HyperParameterTuner<MLAlgorithm,
+                                                    Metric,
+                                                    CV,
+                                                    Optimizer,
+                                                    MatType,
+                                                    PredictionsType,
+                                                    WeightsType>::Optimize(
+    const Args&... args)
+{
+  static const size_t numberOfParametersToOptimize =
+      std::tuple_size<TupleOfHyperParameters<Args...>>::value;
+  data::IncrementPolicy policy(true);
+  data::DatasetMapper<data::IncrementPolicy, double> datasetInfo(policy,
+      numberOfParametersToOptimize);
+
+  arma::mat bestParameters(numberOfParametersToOptimize, 1);
+  const auto argsTuple = std::tie(args...);
+
+  InitAndOptimize<0>(argsTuple, bestParameters, datasetInfo);
+
+  return VectorToTuple<TupleOfHyperParameters<Args...>, 0>(bestParameters);
+}
+
+template<typename MLAlgorithm,
+         typename Metric,
+         template<typename, typename, typename, typename, typename> class CV,
+         typename Optimizer,
+         typename MatType,
+         typename PredictionsType,
+         typename WeightsType>
+template<size_t I, typename ArgsTuple, typename... FixedArgs, typename>
+void HyperParameterTuner<MLAlgorithm,
+                         Metric,
+                         CV,
+                         Optimizer,
+                         MatType,
+                         PredictionsType,
+                         WeightsType>::InitAndOptimize(
+    const ArgsTuple& /* args */,
+    arma::mat& bestParams,
+    data::DatasetMapper<data::IncrementPolicy, double>& datasetInfo,
+    FixedArgs... fixedArgs)
+{
+  static const size_t totalArgs = std::tuple_size<ArgsTuple>::value;
+
+  CVFunction<CVType, MLAlgorithm, totalArgs, FixedArgs...>
+      cvFunction(cv, relativeDelta, minDelta, fixedArgs...);
+  bestObjective = Metric::NeedsMinimization?
+      optimizer.Optimize(cvFunction, bestParams, datasetInfo) :
+      -optimizer.Optimize(cvFunction, bestParams, datasetInfo);
+  bestModel = std::move(cvFunction.BestModel());
+}
+
+template<typename MLAlgorithm,
+         typename Metric,
+         template<typename, typename, typename, typename, typename> class CV,
+         typename Optimizer,
+         typename MatType,
+         typename PredictionsType,
+         typename WeightsType>
+template<size_t I, class ArgsTuple, class... FixedArgs, class, class>
+void HyperParameterTuner<MLAlgorithm,
+                         Metric,
+                         CV,
+                         Optimizer,
+                         MatType,
+                         PredictionsType,
+                         WeightsType>::InitAndOptimize(
+    const ArgsTuple& args,
+    arma::mat& bestParams,
+    data::DatasetMapper<data::IncrementPolicy, double>& datasetInfo,
+    FixedArgs... fixedArgs)
+{
+  using PreFixedArgT = typename std::remove_reference<
+      typename std::tuple_element<I, ArgsTuple>::type>::type;
+  using FixedArgT = FixedArg<typename PreFixedArgT::Type, I>;
+
+  InitAndOptimize<I + 1>(args, bestParams, datasetInfo, fixedArgs...,
+       FixedArgT{std::get<I>(args).value});
+}
+
+template<typename MLAlgorithm,
+         typename Metric,
+         template<typename, typename, typename, typename, typename> class CV,
+         typename Optimizer,
+         typename MatType,
+         typename PredictionsType,
+         typename WeightsType>
+template<size_t I, class ArgsTuple, class... FixedArgs, class, class, class>
+void HyperParameterTuner<MLAlgorithm,
+                         Metric,
+                         CV,
+                         Optimizer,
+                         MatType,
+                         PredictionsType,
+                         WeightsType>::InitAndOptimize(
+    const ArgsTuple& args,
+    arma::mat& bestParams,
+    data::DatasetMapper<data::IncrementPolicy, double>& datasetInfo,
+    FixedArgs... fixedArgs)
+{
+  static const size_t dimension =
+      I - std::tuple_size<std::tuple<FixedArgs...>>::value;
+  datasetInfo.Type(dimension) = data::Datatype::numeric;
+  bestParams(dimension) = std::get<I>(args);
+
+  InitAndOptimize<I + 1>(args, bestParams, datasetInfo, fixedArgs...);
+}
+
+template<typename MLAlgorithm,
+         typename Metric,
+         template<typename, typename, typename, typename, typename> class CV,
+         typename Optimizer,
+         typename MatType,
+         typename PredictionsType,
+         typename WeightsType>
+template<size_t I, class ArgsTuple, class... FixedArgs, class, class, class,
+    class>
+void HyperParameterTuner<MLAlgorithm,
+                         Metric,
+                         CV,
+                         Optimizer,
+                         MatType,
+                         PredictionsType,
+                         WeightsType>::InitAndOptimize(
+    const ArgsTuple& args,
+    arma::mat& bestParams,
+    data::DatasetMapper<data::IncrementPolicy, double>& datasetInfo,
+    FixedArgs... fixedArgs)
+{
+  static const size_t dimension =
+      I - std::tuple_size<std::tuple<FixedArgs...>>::value;
+  for (auto value : std::get<I>(args))
+    datasetInfo.MapString<size_t>(value, dimension);
+
+  if (datasetInfo.NumMappings(dimension) == 0)
+  {
+      std::ostringstream oss;
+      oss << "HyperParameterTuner::Optimize(): the collection passed as the "
+          << "argument " << I + 1 << " is empty" << std::endl;
+      throw std::invalid_argument(oss.str());
+  }
+
+  InitAndOptimize<I + 1>(args, bestParams, datasetInfo, fixedArgs...);
+}
+
+template<typename MLAlgorithm,
+         typename Metric,
+         template<typename, typename, typename, typename, typename> class CV,
+         typename Optimizer,
+         typename MatType,
+         typename PredictionsType,
+         typename WeightsType>
+template<typename TupleType, size_t I, typename... Args, typename>
+TupleType HyperParameterTuner<MLAlgorithm,
+                           Metric,
+                           CV,
+                           Optimizer,
+                           MatType,
+                           PredictionsType,
+                           WeightsType>::VectorToTuple(
+    const arma::vec& vector, const Args&... args)
+{
+  return VectorToTuple<TupleType, I + 1>(vector, args..., vector(I));
+}
+
+template<typename MLAlgorithm,
+         typename Metric,
+         template<typename, typename, typename, typename, typename> class CV,
+         typename Optimizer,
+         typename MatType,
+         typename PredictionsType,
+         typename WeightsType>
+template<typename TupleType, size_t, typename... Args, typename, typename>
+TupleType HyperParameterTuner<MLAlgorithm,
+                           Metric,
+                           CV,
+                           Optimizer,
+                           MatType,
+                           PredictionsType,
+                           WeightsType>::VectorToTuple(
+    const arma::vec& /* vector */, const Args&... args)
+{
+  return TupleType(args...);
+}
+
+} // namespace hpt
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/core/optimizers/CMakeLists.txt b/src/mlpack/core/optimizers/CMakeLists.txt
index 072fd27b32e..ce218252827 100644
--- a/src/mlpack/core/optimizers/CMakeLists.txt
+++ b/src/mlpack/core/optimizers/CMakeLists.txt
@@ -4,6 +4,7 @@ set(DIRS
   adam
   aug_lagrangian
   gradient_descent
+  grid_search
   lbfgs
   minibatch_sgd
   rmsprop
diff --git a/src/mlpack/core/optimizers/gradient_descent/gradient_descent.hpp b/src/mlpack/core/optimizers/gradient_descent/gradient_descent.hpp
index 9947d994749..fb74117a058 100644
--- a/src/mlpack/core/optimizers/gradient_descent/gradient_descent.hpp
+++ b/src/mlpack/core/optimizers/gradient_descent/gradient_descent.hpp
@@ -12,7 +12,7 @@
 #ifndef MLPACK_CORE_OPTIMIZERS_GRADIENT_DESCENT_GRADIENT_DESCENT_HPP
 #define MLPACK_CORE_OPTIMIZERS_GRADIENT_DESCENT_GRADIENT_DESCENT_HPP
 
-#include <mlpack/prereqs.hpp>
+#include <mlpack/core.hpp>
 
 namespace mlpack {
 namespace optimization {
@@ -78,6 +78,27 @@ class GradientDescent
   template<typename FunctionType>
   double Optimize(FunctionType& function, arma::mat& iterate);
 
+  /**
+   * Assert all dimensions are numeric and optimize the given function using
+   * gradient descent. The given starting point will be modified to store the
+   * finishing point of the algorithm, and the final objective value is
+   * returned.
+   *
+   * This overload is intended to be used primarily by the hyper-parameter
+   * tuning module.
+   *
+   * @tparam FunctionType Type of the function to optimize.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @param datasetInfo Type information for each dimension of the dataset.
+   * @return Objective value of the final point.
+   */
+  template<typename FunctionType>
+  double Optimize(
+      FunctionType& function,
+      arma::mat& iterate,
+      data::DatasetMapper<data::IncrementPolicy, double>& datasetInfo);
+
   //! Get the step size.
   double StepSize() const { return stepSize; }
   //! Modify the step size.
diff --git a/src/mlpack/core/optimizers/gradient_descent/gradient_descent_impl.hpp b/src/mlpack/core/optimizers/gradient_descent/gradient_descent_impl.hpp
index 5339692b2e8..bb51bed43d7 100644
--- a/src/mlpack/core/optimizers/gradient_descent/gradient_descent_impl.hpp
+++ b/src/mlpack/core/optimizers/gradient_descent/gradient_descent_impl.hpp
@@ -67,6 +67,35 @@ double GradientDescent::Optimize(
   return overallObjective;
 }
 
+template<typename FunctionType>
+double GradientDescent::Optimize(
+    FunctionType& function,
+    arma::mat& iterate,
+    data::DatasetMapper<data::IncrementPolicy, double>& datasetInfo)
+{
+  if (datasetInfo.Dimensionality() != iterate.n_rows)
+  {
+      std::ostringstream oss;
+      oss << "GradientDescent::Optimize(): expected information about "
+          << iterate.n_rows << " dimensions in datasetInfo, but found about "
+          << datasetInfo.Dimensionality() << std::endl;
+      throw std::invalid_argument(oss.str());
+  }
+
+  for (size_t i = 0; i < datasetInfo.Dimensionality(); ++i)
+  {
+    if (datasetInfo.Type(i) != data::Datatype::numeric)
+    {
+      std::ostringstream oss;
+      oss << "GradientDescent::Optimize(): the dimension " << i
+          << "is not numeric" << std::endl;
+      throw std::invalid_argument(oss.str());
+    }
+  }
+
+  return Optimize(function, iterate);
+}
+
 } // namespace optimization
 } // namespace mlpack
 
diff --git a/src/mlpack/core/optimizers/grid_search/CMakeLists.txt b/src/mlpack/core/optimizers/grid_search/CMakeLists.txt
new file mode 100644
index 00000000000..ecfd1aff8de
--- /dev/null
+++ b/src/mlpack/core/optimizers/grid_search/CMakeLists.txt
@@ -0,0 +1,11 @@
+set(SOURCES
+  grid_search.hpp
+  grid_search_impl.hpp
+)
+
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
diff --git a/src/mlpack/core/optimizers/grid_search/grid_search.hpp b/src/mlpack/core/optimizers/grid_search/grid_search.hpp
new file mode 100644
index 00000000000..a584237f628
--- /dev/null
+++ b/src/mlpack/core/optimizers/grid_search/grid_search.hpp
@@ -0,0 +1,73 @@
+/**
+ * @file grid_search.hpp
+ * @author Kirill Mishchenko
+ *
+ * Grid-search optimization.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_GRID_SEARCH_GRID_SEARCH_HPP
+#define MLPACK_CORE_OPTIMIZERS_GRID_SEARCH_GRID_SEARCH_HPP
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * An optimizer that finds the minimum of a given function by iterating through
+ * points on a multidimensional grid.
+ *
+ * For GridSearch to work, a FunctionType template parameter is required. This
+ * class must implement the following function:
+ *
+ *   double Evaluate(const arma::mat& coordinates);
+ */
+class GridSearch
+{
+ public:
+  /**
+   * Optimize (minimize) the given function by iterating through the all
+   * possible combinations of values for the parameters specified in
+   * datasetInfo.
+   *
+   * @param function Function to optimize.
+   * @param bestParameters Variable for storing results.
+   * @param datasetInfo Type information for each dimension of the dataset. It
+   *     should store possible values for each parameter.
+   * @return Objective value of the final point.
+   */
+  template<typename FunctionType>
+  double Optimize(
+      FunctionType& function,
+      arma::mat& bestParameters,
+      data::DatasetMapper<data::IncrementPolicy, double>& datasetInfo);
+
+ private:
+  /**
+   * Iterate through the last (parameterValueCollections.size() - i) dimensions
+   * of the grid and change the arguments bestObjective and bestParameters if
+   * there is something better. The values for the first i dimensions
+   * (parameters) are specified in the first i rows of the currentParameters
+   * argument.
+   */
+  template<typename FunctionType>
+  void Optimize(
+      FunctionType& function,
+      double& bestObjective,
+      arma::mat& bestParameters,
+      arma::vec& currentParameters,
+      data::DatasetMapper<data::IncrementPolicy, double>& datasetInfo,
+      size_t i);
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+// Include implementation
+#include "grid_search_impl.hpp"
+
+#endif
diff --git a/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp b/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp
new file mode 100644
index 00000000000..e0fa2433306
--- /dev/null
+++ b/src/mlpack/core/optimizers/grid_search/grid_search_impl.hpp
@@ -0,0 +1,85 @@
+/**
+ * @file grid_search_impl.hpp
+ * @author Kirill Mishchenko
+ *
+ * Implementation of the grid-search optimization.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_GRID_SEARCH_GRID_SEARCH_IMPL_HPP
+#define MLPACK_CORE_OPTIMIZERS_GRID_SEARCH_GRID_SEARCH_IMPL_HPP
+
+#include <limits>
+
+namespace mlpack {
+namespace optimization {
+
+template<typename FunctionType>
+double GridSearch::Optimize(
+    FunctionType& function,
+    arma::mat& bestParameters,
+    data::DatasetMapper<data::IncrementPolicy, double>& datasetInfo)
+{
+  for (size_t i = 0; i < datasetInfo.Dimensionality(); ++i)
+  {
+    if (datasetInfo.Type(i) != data::Datatype::categorical)
+    {
+      std::ostringstream oss;
+      oss << "GridSearch::Optimize(): the dimension " << i
+          << "is not categorical" << std::endl;
+      throw std::invalid_argument(oss.str());
+    }
+  }
+
+  double bestObjective = std::numeric_limits<double>::max();
+  bestParameters = arma::mat(datasetInfo.Dimensionality(), 1);
+  arma::vec currentParameters = arma::vec(datasetInfo.Dimensionality());
+
+  /* Initialize best parameters for the case (very unlikely though) when no set
+   * of parameters gives an objective value better than
+   * std::numeric_limits<double>::max() */
+  for (size_t i = 0; i < datasetInfo.Dimensionality(); ++i)
+    bestParameters(i, 0) = datasetInfo.UnmapString(0, i);
+
+  Optimize(function, bestObjective, bestParameters, currentParameters,
+      datasetInfo, 0);
+
+  return bestObjective;
+}
+
+template<typename FunctionType>
+void GridSearch::Optimize(
+    FunctionType& function,
+    double& bestObjective,
+    arma::mat& bestParameters,
+    arma::vec& currentParameters,
+    data::DatasetMapper<data::IncrementPolicy, double>& datasetInfo,
+    size_t i)
+{
+  if (i < datasetInfo.Dimensionality())
+  {
+    for (size_t j = 0; j < datasetInfo.NumMappings(i); ++j)
+    {
+      currentParameters(i) = datasetInfo.UnmapString(j, i);
+      Optimize(function, bestObjective, bestParameters, currentParameters,
+          datasetInfo, i + 1);
+    }
+  }
+  else
+  {
+    double objective = function.Evaluate(currentParameters);
+    if (objective < bestObjective)
+    {
+      bestObjective = objective;
+      bestParameters = currentParameters;
+    }
+  }
+}
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt
index d31aeed2259..3acefc542bf 100644
--- a/src/mlpack/tests/CMakeLists.txt
+++ b/src/mlpack/tests/CMakeLists.txt
@@ -32,6 +32,7 @@ add_executable(mlpack_test
   gradient_descent_test.cpp
   hmm_test.cpp
   hoeffding_tree_test.cpp
+  hpt_test.cpp
   hyperplane_test.cpp
   imputation_test.cpp
   init_rules_test.cpp
diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp
new file mode 100644
index 00000000000..3783997f970
--- /dev/null
+++ b/src/mlpack/tests/hpt_test.cpp
@@ -0,0 +1,345 @@
+/**
+ * @file hpt_test.cpp
+ *
+ * Tests for the hyper-parameter tuning module.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#include <mlpack/core/cv/metrics/mse.hpp>
+#include <mlpack/core/cv/metrics/accuracy.hpp>
+#include <mlpack/core/cv/simple_cv.hpp>
+#include <mlpack/core/hpt/cv_function.hpp>
+#include <mlpack/core/hpt/fixed.hpp>
+#include <mlpack/core/hpt/hpt.hpp>
+#include <mlpack/core/optimizers/grid_search/grid_search.hpp>
+#include <mlpack/core/optimizers/gradient_descent/gradient_descent.cpp>
+#include <mlpack/methods/lars/lars.hpp>
+#include <mlpack/methods/logistic_regression/logistic_regression.hpp>
+
+#include <boost/test/unit_test.hpp>
+
+using namespace mlpack::cv;
+using namespace mlpack::data;
+using namespace mlpack::hpt;
+using namespace mlpack::optimization;
+using namespace mlpack::regression;
+
+BOOST_AUTO_TEST_SUITE(HPTTest);
+
+/**
+ * Test CVFunction runs cross-validation in according with specified fixed
+ * arguments and passed parameters.
+ */
+BOOST_AUTO_TEST_CASE(CVFunctionTest)
+{
+  arma::mat xs = arma::randn(5, 100);
+  arma::vec beta = arma::randn(5, 1);
+  arma::rowvec ys = beta.t() * xs + 0.1 * arma::randn(1, 100);
+
+  SimpleCV<LARS, MSE> cv(0.2, xs, ys);
+
+  bool transposeData = true;
+  bool useCholesky = false;
+  double lambda1 = 1.0;
+  double lambda2 = 2.0;
+
+  FixedArg<bool, 1> fixedUseCholesky{useCholesky};
+  FixedArg<double, 3> fixedLambda1{lambda2};
+  CVFunction<decltype(cv), LARS, 4, FixedArg<bool, 1>, FixedArg<double, 3>>
+      cvFun(cv, 0.0, 0.0, fixedUseCholesky, fixedLambda1);
+
+  double expected = cv.Evaluate(transposeData, useCholesky, lambda1, lambda2);
+  double actual = cvFun.Evaluate(arma::vec{double(transposeData), lambda1});
+
+  BOOST_REQUIRE_CLOSE(expected, actual, 1e-5);
+}
+
+/**
+ * This class provides the interface of CV classes, but really implements a
+ * simple quadratic function of three variables.
+ */
+template<typename MLAlgorithm,
+         typename Metric = void,
+         typename MatType = void,
+         typename PredictionsType = void,
+         typename WeightsType = void>
+class QuadraticFunction
+{
+ public:
+   QuadraticFunction(double a,
+                     double b,
+                     double c,
+                     double d,
+                     double xMin = 0.0,
+                     double yMin = 0.0,
+                     double zMin = 0.0) :
+     a(a), b(b), c(c), d(d), xMin(xMin), yMin(yMin), zMin(zMin) {}
+
+   double Evaluate(double x, double y, double z)
+   {
+     return a * pow(x - xMin, 2)  + b * pow(y - yMin, 2) + c * pow(z - zMin, 2)
+         + d;
+   }
+
+   // Declaring and defining it just in order to provide the same interface as
+   // other CV classes.
+   MLAlgorithm Model()
+   {
+     return MLAlgorithm();
+   }
+
+ private:
+   double a, b, c, d, xMin, yMin, zMin;
+};
+
+/**
+ * Test CVFunction approximates gradient in the expected way.
+ */
+BOOST_AUTO_TEST_CASE(CVFunctionGradientTest)
+{
+  double a = 1.0;
+  double b = -1.5;
+  double c = 2.5;
+  double d = 3.0;
+  QuadraticFunction<LARS> lf(a, b, c, d);
+
+  double relativeDelta = 0.01;
+  double minDelta = 0.001;
+  CVFunction<decltype(lf), LARS, 3> cvFun(lf, relativeDelta, minDelta);
+
+  double x = 0.0;
+  double y = -1.0;
+  double z = 2.0;
+  arma::mat gradient;
+  cvFun.Gradient(arma::vec{x, y, z}, gradient);
+
+  double xDelta = minDelta;
+  double yDelta = relativeDelta * abs(y);
+  double zDelta = relativeDelta * abs(z);
+
+  double aproximateXPartialDerivative = a * (2 * x + xDelta);
+  double aproximateYPartialDerivative = b * (2 * y + yDelta);
+  double aproximateZPartialDerivative = c * (2 * z + zDelta);
+
+  BOOST_REQUIRE_EQUAL(gradient.n_elem, 3);
+  BOOST_REQUIRE_CLOSE(gradient(0), aproximateXPartialDerivative, 1e-5);
+  BOOST_REQUIRE_CLOSE(gradient(1), aproximateYPartialDerivative, 1e-5);
+  BOOST_REQUIRE_CLOSE(gradient(2), aproximateZPartialDerivative, 1e-5);
+}
+
+
+void InitProneToOverfittingData(arma::mat& xs,
+                                arma::rowvec& ys,
+                                double& validationSize)
+{
+  // Total number of data points.
+  size_t N = 10;
+  // Total number of features (all except the first one are redundant).
+  size_t M = 5;
+
+  arma::rowvec data = arma::linspace<arma::rowvec>(0.0, 10.0, N);
+  xs = data;
+  for (size_t i = 2; i <= M; ++i)
+    xs = arma::join_cols(xs, arma::pow(data, i));
+
+  // Responses that approximately follow the function y = 2 * x. Adding noise to
+  // avoid having a polynomial of degree 1 that exactly fits the points.
+  ys = 2 * data + 0.05 * arma::randn(1, N);
+
+  validationSize = 0.3;
+}
+
+template<typename T1, typename T2>
+void FindLARSBestLambdas(arma::mat& xs,
+                         arma::rowvec& ys,
+                         double& validationSize,
+                         bool transposeData,
+                         bool useCholesky,
+                         const T1& lambda1Set,
+                         const T2& lambda2Set,
+                         double& bestLambda1,
+                         double& bestLambda2,
+                         double& bestObjective)
+{
+  SimpleCV<LARS, MSE> cv(validationSize, xs, ys);
+
+  bestObjective = std::numeric_limits<double>::max();
+
+  for (double lambda1 : lambda1Set)
+    for (double lambda2 : lambda2Set)
+    {
+      double objective =
+          cv.Evaluate(transposeData, useCholesky, lambda1, lambda2);
+      if (objective < bestObjective)
+      {
+        bestObjective = objective;
+        bestLambda1 = lambda1;
+        bestLambda2 = lambda2;
+      }
+    }
+}
+
+ /**
+ * Test grid-search optimization leads to the best parameters from the specified
+ * ones.
+ */
+BOOST_AUTO_TEST_CASE(GridSearchTest)
+{
+  arma::mat xs;
+  arma::rowvec ys;
+  double validationSize;
+  InitProneToOverfittingData(xs, ys, validationSize);
+
+  bool transposeData = true;
+  bool useCholesky = false;
+  arma::vec lambda1Set =
+      arma::join_cols(arma::vec{0}, arma::logspace<arma::vec>(-3, 2, 6));
+  std::array<double, 4> lambda2Set{{0.0, 0.05, 0.5, 5.0}};
+
+  double expectedLambda1, expectedLambda2, expectedObjective;
+  FindLARSBestLambdas(xs, ys, validationSize, transposeData, useCholesky,
+      lambda1Set, lambda2Set, expectedLambda1, expectedLambda2,
+      expectedObjective);
+
+  SimpleCV<LARS, MSE> cv(validationSize, xs, ys);
+  CVFunction<decltype(cv), LARS, 4, FixedArg<bool, 0>, FixedArg<bool, 1>>
+      cvFun(cv, 0.0, 0.0, {transposeData}, {useCholesky});
+
+  IncrementPolicy policy(true);
+  DatasetMapper<IncrementPolicy, double> datasetInfo(policy, 2);
+  for (double lambda1 : lambda1Set)
+    datasetInfo.MapString<size_t>(lambda1, 0);
+  for (double lambda2 : lambda2Set)
+    datasetInfo.MapString<size_t>(lambda2, 1);
+
+  GridSearch optimizer;
+  arma::mat actualParameters;
+  double actualObjective =
+      optimizer.Optimize(cvFun, actualParameters, datasetInfo);
+
+  BOOST_REQUIRE_CLOSE(expectedObjective, actualObjective, 1e-5);
+  BOOST_REQUIRE_CLOSE(expectedLambda1, actualParameters(0, 0), 1e-5);
+  BOOST_REQUIRE_CLOSE(expectedLambda2, actualParameters(1, 0), 1e-5);
+}
+
+/**
+ * Test HyperParameterTuner.
+ */
+BOOST_AUTO_TEST_CASE(HPTTest)
+{
+  arma::mat xs;
+  arma::rowvec ys;
+  double validationSize;
+  InitProneToOverfittingData(xs, ys, validationSize);
+
+  bool transposeData = true;
+  bool useCholesky = false;
+  arma::vec lambda1Set =
+      arma::join_cols(arma::vec{0}, arma::logspace<arma::vec>(-3, 2, 6));
+  arma::vec lambda2Set{0.0, 0.05, 0.5, 5.0};
+
+  double expectedLambda1, expectedLambda2, expectedObjective;
+  FindLARSBestLambdas(xs, ys, validationSize, transposeData, useCholesky,
+      lambda1Set, lambda2Set, expectedLambda1, expectedLambda2,
+      expectedObjective);
+
+  double actualLambda1, actualLambda2;
+  HyperParameterTuner<LARS, MSE, SimpleCV, GridSearch>
+      hpt(validationSize, xs, ys);
+  std::tie(actualLambda1, actualLambda2) = hpt.Optimize(Fixed(transposeData),
+      Fixed(useCholesky), lambda1Set, lambda2Set);
+
+  BOOST_REQUIRE_CLOSE(expectedObjective, hpt.BestObjective(), 1e-5);
+  BOOST_REQUIRE_CLOSE(expectedLambda1, actualLambda1, 1e-5);
+  BOOST_REQUIRE_CLOSE(expectedLambda2, actualLambda2, 1e-5);
+
+  /* Checking that the model provided by the hyper-parameter tuner shows the
+   * same performance. */
+  size_t validationFirstColumn = round(xs.n_cols * (1.0 - validationSize));
+  arma::mat validationXs = xs.cols(validationFirstColumn, xs.n_cols - 1);
+  arma::rowvec validationYs = ys.cols(validationFirstColumn, ys.n_cols - 1);
+  double objective = MSE::Evaluate(hpt.BestModel(), validationXs, validationYs);
+  BOOST_REQUIRE_CLOSE(expectedObjective, objective, 1e-5);
+}
+
+/**
+ * Test HyperParamterTuner maximizes Accuracy rather than minimizes it.
+ */
+BOOST_AUTO_TEST_CASE(HPTMaximizationTest)
+{
+  // Initializing a linearly separable dataset.
+  arma::mat xs = arma::linspace<arma::rowvec>(0.0, 10.0, 50);
+  arma::Row<size_t> ys = arma::join_rows(arma::zeros<arma::Row<size_t>>(25),
+      arma::ones<arma::Row<size_t>>(25));
+
+  // We will train and validate on the same dataset.
+  double validationSize = 0.5;
+  arma::mat doubledXs = arma::join_rows(xs, xs);
+  arma::Row<size_t> doubledYs = arma::join_rows(ys, ys);
+
+  // Defining lambdas to choose from. Zero should be preferred since big lambdas
+  // are likely to restrict capabilities of logistic regression.
+  arma::vec lambdas{0.0, 1e12};
+
+  // Making sure that the assumption above is true.
+  SimpleCV<LogisticRegression<>, Accuracy>
+      cv(validationSize, doubledXs, doubledYs);
+  BOOST_REQUIRE_GT(cv.Evaluate(0.0), cv.Evaluate(1e12));
+
+  HyperParameterTuner<LogisticRegression<>, Accuracy, SimpleCV>
+      hpt(validationSize, doubledXs, doubledYs);
+
+  double actualLambda;
+  std::tie(actualLambda) = hpt.Optimize(lambdas);
+
+  BOOST_REQUIRE_CLOSE(hpt.BestObjective(), 1.0, 1e-5);
+  BOOST_REQUIRE_CLOSE(actualLambda, 0.0, 1e-5);
+}
+
+/**
+ * Test HyperParameterTuner works with GradientDescent.
+ */
+BOOST_AUTO_TEST_CASE(HPTGradientDescentTest)
+{
+  // Constructor arguments for the fake CV function (QuadraticFunction).
+  double a = 1.0;
+  double b = -1.5;
+  double c = 2.5;
+  double d = 3.0;
+
+  // Optimal values for three "hyper-parameters".
+  double xMin = 1.5;
+  double yMin = 0.0;
+  double zMin = -2.0;
+
+  // We pass LARS just because some ML algorithm should be passed. We pass MSE
+  // to tell HyperParameterTuner that the objective function (QuadraticFunction)
+  // should be minimized.
+  HyperParameterTuner<LARS, MSE, QuadraticFunction, GradientDescent>
+      hpt(a, b, c, d, xMin, yMin, zMin);
+
+  // Setting GradientDescent to find more close solution to the optimal one.
+  hpt.Optimizer().StepSize() = 0.1;
+  hpt.Optimizer().Tolerance() = 1e-15;
+
+  // Always using the same small increase of arguments in calculation of partial
+  // derivatives.
+  hpt.RelativeDelta() = 0.0;
+  hpt.MinDelta() = 1e-10;
+
+  // We will try to find optimal values only for two "hyper-parameters".
+  double x0 = 3.0;
+  double y = yMin;
+  double z0 = -3.0;
+
+  double xOptimized, zOptimized;
+  std::tie(xOptimized, zOptimized) = hpt.Optimize(x0, Fixed(y), z0);
+  BOOST_REQUIRE_CLOSE(xOptimized, xMin, 1e-4);
+  BOOST_REQUIRE_CLOSE(zOptimized, zMin, 1e-4);
+}
+
+BOOST_AUTO_TEST_SUITE_END();