Remove gdc::Index, make the vector a template parameter (04710f1f) · Commits · Jakub Klinkovský / TNL gradient descent

README.md

+3 −2

Original line number	Diff line number	Diff line
		@@ -54,7 +54,7 @@ struct Ackley
		int main()
		{
		// Define the vector type
		using Vector = TNL::Containers::Vector<double, TNL::Devices::Host, gdc::Index>;
		using Vector = TNL::Containers::Vector<double, TNL::Devices::Host, int>;

		// Create optimizer object with Ackley functor as objective.
		//
		@@ -65,7 +65,8 @@ int main()
		// You can additionally specify a FiniteDifferences functor as template
		// parameter. There are Forward-, Backward- and CentralDifferences
		// available. (Default is CentralDifferences)
		gdc::GradientDescent<double, Ackley, gdc::WolfeBacktracking<double>> optimizer;
		using StepSize = gdc::WolfeBacktracking<Vector>;
		gdc::GradientDescent<Vector, Ackley, StepSize> optimizer;

		// Set number of iterations as stop criterion.
		// Set it to 0 or negative for infinite iterations (default is 0).

examples/ackley.cpp

+3 −2

Original line number	Diff line number	Diff line
		@@ -25,7 +25,7 @@ struct Ackley
		int main()
		{
		// Define the vector type
		using Vector = TNL::Containers::Vector<double, TNL::Devices::Host, gdc::Index>;
		using Vector = TNL::Containers::Vector<double, TNL::Devices::Host, int>;

		// Create optimizer object with Ackley functor as objective.
		//
		@@ -36,7 +36,8 @@ int main()
		// You can additionally specify a FiniteDifferences functor as template
		// parameter. There are Forward-, Backward- and CentralDifferences
		// available. (Default is CentralDifferences)
		gdc::GradientDescent<double, Ackley, gdc::WolfeBacktracking<double>> optimizer;
		using StepSize = gdc::WolfeBacktracking<Vector>;
		gdc::GradientDescent<Vector, Ackley, StepSize> optimizer;

		// Set number of iterations as stop criterion.
		// Set it to 0 or negative for infinite iterations (default is 0).

examples/paraboloid.cpp

+4 −3

Original line number	Diff line number	Diff line
		@@ -20,7 +20,7 @@ struct Paraboloid
		int main()
		{
		// Define the vector type
		using Vector = TNL::Containers::Vector<double, TNL::Devices::Host, gdc::Index>;
		using Vector = TNL::Containers::Vector<double, TNL::Devices::Host, int>;

		// Create optimizer object with Paraboloid functor as objective.
		//
		@@ -31,7 +31,8 @@ int main()
		// You can additionally specify a FiniteDifferences functor as template
		// parameter. There are Forward-, Backward- and CentralDifferences
		// available. (Default is CentralDifferences)
		gdc::GradientDescent<double, Paraboloid, gdc::ConstantStepSize<double>> optimizer;
		using StepSize = gdc::ConstantStepSize<Vector>;
		gdc::GradientDescent<Vector, Paraboloid, StepSize> optimizer;

		// Set number of iterations as stop criterion.
		// Set it to 0 or negative for infinite iterations (default is 0).
		@@ -48,7 +49,7 @@ int main()
		optimizer.setMinStepLength(1e-6);

		// Set the the parametrized StepSize functor used for the step calculation.
		optimizer.setStepSize(gdc::ConstantStepSize<double>(0.8));
		optimizer.setStepSize(StepSize(0.8));

		// Set the momentum rate used for the step calculation (default is 0.0).
		// Defines how much momentum is kept from previous iterations.

include/gdcpp.h

+31 −24

Original line number	Diff line number	Diff line
		@@ -17,8 +17,6 @@

		namespace gdc
		{
		using Index = long int;

		/** Functor to compute forward differences.
		* Computes the gradient of the objective f(x) as follows:
		*
		@@ -26,11 +24,12 @@ namespace gdc
		*
		* The computation requires len(x) evaluations of the objective.
		*/
		template<typename Scalar>
		template<typename Vector>
		class ForwardDifferences
		{
		public:
		using Vector = TNL::Containers::Vector<Scalar, TNL::Devices::Host, Index>;
		using Scalar = typename Vector::RealType;
		using Index = typename Vector::IndexType;
		using Objective = std::function<Scalar(const Vector &)>;
		private:
		Scalar eps_;
		@@ -87,11 +86,12 @@ namespace gdc
		*
		* The computation requires len(x) evaluations of the objective.
		*/
		template<typename Scalar>
		template<typename Vector>
		class BackwardDifferences
		{
		public:
		using Vector = TNL::Containers::Vector<Scalar, TNL::Devices::Host, Index>;
		using Scalar = typename Vector::RealType;
		using Index = typename Vector::IndexType;
		using Objective = std::function<Scalar(const Vector &)>;
		private:
		Scalar eps_;
		@@ -147,11 +147,12 @@ namespace gdc
		*
		* The computation requires 2 * len(x) evaluations of the objective.
		*/
		template<typename Scalar>
		template<typename Vector>
		struct CentralDifferences
		{
		public:
		using Vector = TNL::Containers::Vector<Scalar, TNL::Devices::Host, Index>;
		using Scalar = typename Vector::RealType;
		using Index = typename Vector::IndexType;
		using Objective = std::function<Scalar(const Vector &)>;
		private:
		Scalar eps_;
		@@ -209,11 +210,12 @@ namespace gdc
		};

		/** Step size functor, which returns a constant step size. */
		template<typename Scalar>
		template<typename Vector>
		class ConstantStepSize
		{
		public:
		using Vector = TNL::Containers::Vector<Scalar, TNL::Devices::Host, Index>;
		using Scalar = typename Vector::RealType;
		using Index = typename Vector::IndexType;
		using Objective = std::function<Scalar(const Vector &, Vector &)>;
		using FiniteDifferences = std::function<void(const Vector &, const Scalar, Vector &)>;
		private:
		@@ -259,11 +261,12 @@ namespace gdc
		* Inverse: stepSize = (y_k^T * s_k) / (y_k^T * y_k)
		*
		* The very first step is computed as a constant. */
		template<typename Scalar>
		template<typename Vector>
		class BarzilaiBorwein
		{
		public:
		using Vector = TNL::Containers::Vector<Scalar, TNL::Devices::Host, Index>;
		using Scalar = typename Vector::RealType;
		using Index = typename Vector::IndexType;
		using Objective = std::function<Scalar(const Vector &, Vector &)>;
		using FiniteDifferences = std::function<void(const Vector &, const Scalar, Vector &)>;

		@@ -374,11 +377,12 @@ namespace gdc
		* If either condition does not hold the step size is decreased:
		*
		* stepSize = decrease * stepSize */
		template<typename Scalar>
		template<typename Vector>
		class ArmijoBacktracking
		{
		public:
		using Vector = TNL::Containers::Vector<Scalar, TNL::Devices::Host, Index>;
		using Scalar = typename Vector::RealType;
		using Index = typename Vector::IndexType;
		using Objective = std::function<Scalar(const Vector &, Vector &)>;
		using FiniteDifferences = std::function<void(const Vector &, const Scalar, Vector &)>;
		protected:
		@@ -520,11 +524,12 @@ namespace gdc
		* If either condition does not hold the step size is decreased:
		*
		* stepSize = decrease * stepSize */
		template<typename Scalar>
		class WolfeBacktracking : public ArmijoBacktracking<Scalar>
		template<typename Vector>
		class WolfeBacktracking : public ArmijoBacktracking<Vector>
		{
		public:
		using Vector = TNL::Containers::Vector<Scalar, TNL::Devices::Host, Index>;
		using Scalar = typename Vector::RealType;
		using Index = typename Vector::IndexType;
		using Objective = std::function<Scalar(const Vector &, Vector &)>;
		using FiniteDifferences = std::function<void(const Vector &, const Scalar, Vector &)>;
		protected:
		@@ -549,7 +554,7 @@ namespace gdc
		const Scalar minStep,
		const Scalar maxStep,
		const Index iterations)
		: ArmijoBacktracking<Scalar>(decrease, cArmijo, minStep, maxStep,
		: ArmijoBacktracking<Vector>(decrease, cArmijo, minStep, maxStep,
		iterations),cWolfe_(cWolfe)
		{
		assert(cWolfe < 1);
		@@ -581,11 +586,12 @@ namespace gdc
		*
		* This functor does not require to compute any gradients and does not use
		* finite differences. */
		template<typename Scalar>
		template<typename Vector>
		class DecreaseBacktracking
		{
		public:
		using Vector = TNL::Containers::Vector<Scalar, TNL::Devices::Host, Index>;
		using Scalar = typename Vector::RealType;
		using Index = typename Vector::IndexType;
		using Objective = std::function<Scalar(const Vector &, Vector &)>;
		using FiniteDifferences = std::function<void(const Vector &, const Scalar, Vector &)>;
		private:
		@@ -673,14 +679,15 @@ namespace gdc
		}
		};

		template<typename Scalar,
		template<typename Vector,
		typename Objective,
		typename StepSize=BarzilaiBorwein<Scalar>,
		typename FiniteDifferences=CentralDifferences<Scalar>>
		typename StepSize=BarzilaiBorwein<Vector>,
		typename FiniteDifferences=CentralDifferences<Vector>>
		class GradientDescent
		{
		public:
		using Vector = TNL::Containers::Vector<Scalar, TNL::Devices::Host, Index>;
		using Scalar = typename Vector::RealType;
		using Index = typename Vector::IndexType;
		using Callback = std::function<bool(Index iterations, const Vector & xval, Scalar fval, const Vector & gradient)>;

		struct Result

test/gdcpp.cpp

+36 −36

Original line number	Diff line number	Diff line
		@@ -3,20 +3,21 @@

		using namespace gdc;

		template<typename Scalar>
		using Scalar = float;
		using Device = TNL::Devices::Host;
		using Index = int;
		using Vector = TNL::Containers::Vector<Scalar, Device, Index>;

		struct Paraboloid
		{
		using Vector = TNL::Containers::Vector<float, TNL::Devices::Host, gdc::Index>;
		Scalar operator()(const Vector &state, Vector &)
		{
		return state(0) * state(0) + state(1) * state(1);
		}
		};

		template<typename Scalar>
		struct Rosenbrock
		{
		using Vector = TNL::Containers::Vector<float, TNL::Devices::Host, gdc::Index>;
		Scalar operator()(const Vector &state, Vector &)
		{
		Scalar delta1 = 1 - state(0);
		@@ -26,20 +27,18 @@ struct Rosenbrock
		}
		};

		using Vector = TNL::Containers::Vector<float, TNL::Devices::Host, gdc::Index>;

		TEST_CASE("gradient_descent")
		{
		const float eps = 1e-3;
		const Scalar eps = 1e-3;

		SECTION("optimize paraboloid")
		{
		SECTION("forward differences")
		{
		GradientDescent<float,
		Paraboloid<float>,
		ConstantStepSize<float>,
		ForwardDifferences<float>> optimizer;
		GradientDescent<Vector,
		Paraboloid,
		ConstantStepSize<Vector>,
		ForwardDifferences<Vector>> optimizer;
		optimizer.setMaxIterations(100);

		Vector xval = {2, 2};
		@@ -51,10 +50,10 @@ TEST_CASE("gradient_descent")

		SECTION("backward differences")
		{
		GradientDescent<float,
		Paraboloid<float>,
		ConstantStepSize<float>,
		BackwardDifferences<float>> optimizer;
		GradientDescent<Vector,
		Paraboloid,
		ConstantStepSize<Vector>,
		BackwardDifferences<Vector>> optimizer;
		optimizer.setMaxIterations(100);

		Vector xval = {2, 2};
		@@ -66,10 +65,10 @@ TEST_CASE("gradient_descent")

		SECTION("central differences")
		{
		GradientDescent<float,
		Paraboloid<float>,
		ConstantStepSize<float>,
		CentralDifferences<float>> optimizer;
		GradientDescent<Vector,
		Paraboloid,
		ConstantStepSize<Vector>,
		CentralDifferences<Vector>> optimizer;
		optimizer.setMaxIterations(100);

		Vector xval = {2, 2};
		@@ -81,9 +80,9 @@ TEST_CASE("gradient_descent")

		SECTION("constant step size")
		{
		GradientDescent<float,
		Paraboloid<float>,
		ConstantStepSize<float>> optimizer;
		GradientDescent<Vector,
		Paraboloid,
		ConstantStepSize<Vector>> optimizer;
		optimizer.setMaxIterations(100);

		Vector xval = {2, 2};
		@@ -95,9 +94,9 @@ TEST_CASE("gradient_descent")

		SECTION("Barzilai-Borwein step")
		{
		GradientDescent<float,
		Paraboloid<float>,
		BarzilaiBorwein<float>> optimizer;
		GradientDescent<Vector,
		Paraboloid,
		BarzilaiBorwein<Vector>> optimizer;
		optimizer.setMaxIterations(100);

		Vector xval = {2, 2};
		@@ -109,9 +108,9 @@ TEST_CASE("gradient_descent")

		SECTION("Wolfe linesearch")
		{
		GradientDescent<float,
		Paraboloid<float>,
		WolfeBacktracking<float>> optimizer;
		GradientDescent<Vector,
		Paraboloid,
		WolfeBacktracking<Vector>> optimizer;
		optimizer.setMaxIterations(100);

		Vector xval = {2, 2};
		@@ -123,9 +122,9 @@ TEST_CASE("gradient_descent")

		SECTION("Armijo linesearch")
		{
		GradientDescent<float,
		Paraboloid<float>,
		ArmijoBacktracking<float>> optimizer;
		GradientDescent<Vector,
		Paraboloid,
		ArmijoBacktracking<Vector>> optimizer;
		optimizer.setMaxIterations(100);

		Vector xval = {2, 2};
		@@ -137,9 +136,9 @@ TEST_CASE("gradient_descent")

		SECTION("Decrease linesearch")
		{
		GradientDescent<float,
		Paraboloid<float>,
		DecreaseBacktracking<float>> optimizer;
		GradientDescent<Vector,
		Paraboloid,
		DecreaseBacktracking<Vector>> optimizer;
		optimizer.setMaxIterations(100);

		Vector xval = {2, 2};
		@@ -152,8 +151,9 @@ TEST_CASE("gradient_descent")

		SECTION("optimize Rosenbrock")
		{
		GradientDescent<float, Rosenbrock<float>,
		WolfeBacktracking<float>> optimizer;
		GradientDescent<Vector,
		Rosenbrock,
		WolfeBacktracking<Vector>> optimizer;
		optimizer.setMaxIterations(3000);
		optimizer.setMomentum(0.9);
		Vector xval = {-0.5, 0.5};