Refactor passing the objective function to the minimize method (3efa47b1) · Commits · Jakub Klinkovský / TNL gradient descent

README.md

+4 −4

Original line number	Diff line number	Diff line
		@@ -60,7 +60,7 @@ int main()
		// Define the vector type
		using Vector = TNL::Containers::Vector<double, TNL::Devices::Host, int>;

		// Create optimizer object with Ackley functor as objective.
		// Create optimizer object for the Vector type.
		//
		// You can specify a StepSize functor as template parameter.
		// There are ConstantStepSize, BarzilaiBorwein and
		@@ -70,7 +70,7 @@ int main()
		// parameter. There are Forward-, Backward- and CentralDifferences
		// available. (Default is CentralDifferences)
		using StepSize = gdc::WolfeBacktracking<Vector>;
		gdc::GradientDescent<Vector, Ackley, StepSize> optimizer;
		gdc::GradientDescent<Vector, StepSize> optimizer;

		// Set number of iterations as stop criterion.
		// Set it to 0 or negative for infinite iterations (default is 0).
		@@ -97,8 +97,8 @@ int main()
		// Set initial guess.
		Vector initialGuess = {-2.7, 2.2};

		// Start the optimization
		auto result = optimizer.minimize(initialGuess);
		// Start the optimization with Ackley functor as objective.
		auto result = optimizer.minimize(Ackley(), initialGuess);

		std::cout << "Done! Converged: " << (result.converged ? "true" : "false")
		<< " Iterations: " << result.iterations << std::endl;

examples/ackley.cpp

+4 −4

Original line number	Diff line number	Diff line
		@@ -27,7 +27,7 @@ int main()
		// Define the vector type
		using Vector = TNL::Containers::Vector<double, TNL::Devices::Host, int>;

		// Create optimizer object with Ackley functor as objective.
		// Create optimizer object for the Vector type.
		//
		// You can specify a StepSize functor as template parameter.
		// There are ConstantStepSize, BarzilaiBorwein and
		@@ -37,7 +37,7 @@ int main()
		// parameter. There are Forward-, Backward- and CentralDifferences
		// available. (Default is CentralDifferences)
		using StepSize = gdc::WolfeBacktracking<Vector>;
		gdc::GradientDescent<Vector, Ackley, StepSize> optimizer;
		gdc::GradientDescent<Vector, StepSize> optimizer;

		// Set number of iterations as stop criterion.
		// Set it to 0 or negative for infinite iterations (default is 0).
		@@ -64,8 +64,8 @@ int main()
		// Set initial guess.
		Vector initialGuess = {-2.7, 2.2};

		// Start the optimization
		auto result = optimizer.minimize(initialGuess);
		// Start the optimization with Ackley functor as objective.
		auto result = optimizer.minimize(Ackley(), initialGuess);

		std::cout << "Done! Converged: " << (result.converged ? "true" : "false")
		<< " Iterations: " << result.iterations << std::endl;

examples/paraboloid.cpp

+4 −4

Original line number	Diff line number	Diff line
		@@ -22,7 +22,7 @@ int main()
		// Define the vector type
		using Vector = TNL::Containers::Vector<double, TNL::Devices::Host, int>;

		// Create optimizer object with Paraboloid functor as objective.
		// Create optimizer object for the Vector type.
		//
		// You can specify a StepSize functor as template parameter.
		// There are ConstantStepSize, BarzilaiBorwein and
		@@ -32,7 +32,7 @@ int main()
		// parameter. There are Forward-, Backward- and CentralDifferences
		// available. (Default is CentralDifferences)
		using StepSize = gdc::ConstantStepSize<Vector>;
		gdc::GradientDescent<Vector, Paraboloid, StepSize> optimizer;
		gdc::GradientDescent<Vector, StepSize> optimizer;

		// Set number of iterations as stop criterion.
		// Set it to 0 or negative for infinite iterations (default is 0).
		@@ -62,8 +62,8 @@ int main()
		// Set initial guess.
		Vector initialGuess = {2, 2};

		// Start the optimization
		auto result = optimizer.minimize(initialGuess);
		// Start the optimization with Paraboloid functor as objective.
		auto result = optimizer.minimize(Paraboloid(), initialGuess);

		std::cout << "Done! Converged: " << (result.converged ? "true" : "false")
		<< " Iterations: " << result.iterations << std::endl;

include/gdcpp.h

+15 −20

Original line number	Diff line number	Diff line
		@@ -62,7 +62,7 @@ namespace gdc

		void operator()(const Vector &xval,
		const Scalar fval,
		Vector &gradient)
		Vector &gradient) const
		{
		assert(objective_);

		@@ -124,7 +124,7 @@ namespace gdc

		void operator()(const Vector &xval,
		const Scalar fval,
		Vector &gradient)
		Vector &gradient) const
		{
		assert(objective_);

		@@ -185,7 +185,7 @@ namespace gdc

		void operator()(const Vector &xval,
		const Scalar,
		Vector &gradient)
		Vector &gradient) const
		{
		assert(objective_);

		@@ -394,7 +394,7 @@ namespace gdc
		Objective objective_;
		FiniteDifferences finiteDifferences_;

		Scalar evaluateObjective(const Vector &xval, Vector &gradient)
		Scalar evaluateObjective(const Vector &xval, Vector &gradient) const
		{
		gradient.resize(0);
		Scalar fval = objective_(xval, gradient);
		@@ -680,7 +680,6 @@ namespace gdc
		};

		template<typename Vector,
		typename Objective,
		typename StepSize=BarzilaiBorwein<Vector>,
		typename FiniteDifferences=CentralDifferences<Vector>>
		class GradientDescent
		@@ -703,15 +702,15 @@ namespace gdc
		Scalar minStepLen_;
		Scalar momentum_;
		Index verbosity_;
		Objective objective_;
		StepSize stepSize_;
		Callback callback_;
		FiniteDifferences finiteDifferences_;

		Scalar evaluateObjective(const Vector &xval, Vector &gradient)
		template<typename Objective>
		Scalar evaluateObjective(Objective&& objective, const Vector &xval, Vector &gradient) const
		{
		gradient.resize(0);
		Scalar fval = objective_(xval, gradient);
		Scalar fval = objective(xval, gradient);
		if(gradient.getSize() == 0)
		finiteDifferences_(xval, fval, gradient);
		return fval;
		@@ -722,7 +721,7 @@ namespace gdc
		GradientDescent()
		: maxIt_(0), minGradientLen_(static_cast<Scalar>(1e-9)),
		minStepLen_(static_cast<Scalar>(1e-9)), momentum_(0),
		verbosity_(0), objective_(), stepSize_(), callback_(),
		verbosity_(0), stepSize_(), callback_(),
		finiteDifferences_()
		{

		@@ -748,11 +747,6 @@ namespace gdc
		maxIt_ = iterations;
		}

		void setObjective(const Objective &objective)
		{
		objective_ = objective;
		}

		void setCallback(const Callback &callback)
		{
		callback_ = callback;
		@@ -783,14 +777,15 @@ namespace gdc
		verbosity_ = verbosity;
		}

		Result minimize(const Vector &initialGuess)
		template<typename Objective>
		Result minimize(Objective&& objective, const Vector &initialGuess)
		{
		finiteDifferences_.setObjective(
		[this](const Vector &xval)
		{ Vector tmp; return this->objective_(xval, tmp); });
		[&objective](const Vector &xval)
		{ Vector tmp; return objective(xval, tmp); });
		stepSize_.setObjective(
		[this](const Vector &xval, Vector &gradient)
		{ return this->objective_(xval, gradient); });
		[&objective](const Vector &xval, Vector &gradient)
		{ return objective(xval, gradient); });
		stepSize_.setFiniteDifferences(
		[this](const Vector &xval, const Scalar fval, Vector &gradient)
		{ this->finiteDifferences_(xval, fval, gradient); });
		@@ -812,7 +807,7 @@ namespace gdc
		&& callbackResult)
		{
		xval -= step;
		fval = evaluateObjective(xval, gradient);
		fval = evaluateObjective(objective, xval, gradient);
		gradientLen = TNL::l2Norm(gradient);
		// update step according to step size and momentum
		stepSize = stepSize_(xval, fval, gradient);

test/gdcpp.cpp

+9 −18

Original line number	Diff line number	Diff line
		@@ -36,7 +36,6 @@ TEST_CASE("gradient_descent")
		SECTION("forward differences")
		{
		GradientDescent<Vector,
		Paraboloid,
		ConstantStepSize<Vector>,
		ForwardDifferences<Vector>> optimizer;
		optimizer.setMaxIterations(100);
		@@ -44,14 +43,13 @@ TEST_CASE("gradient_descent")
		Vector xval = {2, 2};
		Vector xvalExp = {0, 0};

		auto result = optimizer.minimize(xval);
		auto result = optimizer.minimize(Paraboloid(), xval);
		REQUIRE_VECTOR_APPROX(xvalExp, result.xval, eps);
		}

		SECTION("backward differences")
		{
		GradientDescent<Vector,
		Paraboloid,
		ConstantStepSize<Vector>,
		BackwardDifferences<Vector>> optimizer;
		optimizer.setMaxIterations(100);
		@@ -59,14 +57,13 @@ TEST_CASE("gradient_descent")
		Vector xval = {2, 2};
		Vector xvalExp = {0, 0};

		auto result = optimizer.minimize(xval);
		auto result = optimizer.minimize(Paraboloid(), xval);
		REQUIRE_VECTOR_APPROX(xvalExp, result.xval, eps);
		}

		SECTION("central differences")
		{
		GradientDescent<Vector,
		Paraboloid,
		ConstantStepSize<Vector>,
		CentralDifferences<Vector>> optimizer;
		optimizer.setMaxIterations(100);
		@@ -74,77 +71,72 @@ TEST_CASE("gradient_descent")
		Vector xval = {2, 2};
		Vector xvalExp = {0, 0};

		auto result = optimizer.minimize(xval);
		auto result = optimizer.minimize(Paraboloid(), xval);
		REQUIRE_VECTOR_APPROX(xvalExp, result.xval, eps);
		}

		SECTION("constant step size")
		{
		GradientDescent<Vector,
		Paraboloid,
		ConstantStepSize<Vector>> optimizer;
		optimizer.setMaxIterations(100);

		Vector xval = {2, 2};
		Vector xvalExp = {0, 0};

		auto result = optimizer.minimize(xval);
		auto result = optimizer.minimize(Paraboloid(), xval);
		REQUIRE_VECTOR_APPROX(xvalExp, result.xval, eps);
		}

		SECTION("Barzilai-Borwein step")
		{
		GradientDescent<Vector,
		Paraboloid,
		BarzilaiBorwein<Vector>> optimizer;
		optimizer.setMaxIterations(100);

		Vector xval = {2, 2};
		Vector xvalExp = {0, 0};

		auto result = optimizer.minimize(xval);
		auto result = optimizer.minimize(Paraboloid(), xval);
		REQUIRE_VECTOR_APPROX(xvalExp, result.xval, eps);
		}

		SECTION("Wolfe linesearch")
		{
		GradientDescent<Vector,
		Paraboloid,
		WolfeBacktracking<Vector>> optimizer;
		optimizer.setMaxIterations(100);

		Vector xval = {2, 2};
		Vector xvalExp = {0, 0};

		auto result = optimizer.minimize(xval);
		auto result = optimizer.minimize(Paraboloid(), xval);
		REQUIRE_VECTOR_APPROX(xvalExp, result.xval, eps);
		}

		SECTION("Armijo linesearch")
		{
		GradientDescent<Vector,
		Paraboloid,
		ArmijoBacktracking<Vector>> optimizer;
		optimizer.setMaxIterations(100);

		Vector xval = {2, 2};
		Vector xvalExp = {0, 0};

		auto result = optimizer.minimize(xval);
		auto result = optimizer.minimize(Paraboloid(), xval);
		REQUIRE_VECTOR_APPROX(xvalExp, result.xval, eps);
		}

		SECTION("Decrease linesearch")
		{
		GradientDescent<Vector,
		Paraboloid,
		DecreaseBacktracking<Vector>> optimizer;
		optimizer.setMaxIterations(100);

		Vector xval = {2, 2};
		Vector xvalExp = {0, 0};

		auto result = optimizer.minimize(xval);
		auto result = optimizer.minimize(Paraboloid(), xval);
		REQUIRE_VECTOR_APPROX(xvalExp, result.xval, eps);
		}
		}
		@@ -152,14 +144,13 @@ TEST_CASE("gradient_descent")
		SECTION("optimize Rosenbrock")
		{
		GradientDescent<Vector,
		Rosenbrock,
		WolfeBacktracking<Vector>> optimizer;
		optimizer.setMaxIterations(3000);
		optimizer.setMomentum(0.9);
		Vector xval = {-0.5, 0.5};
		Vector xvalExp = {1, 1};

		auto result = optimizer.minimize(xval);
		auto result = optimizer.minimize(Rosenbrock(), xval);
		REQUIRE_VECTOR_APPROX(xvalExp, result.xval, eps);
		}
		}