From c8867fdc465cb365a54d49f45bdffd66a1399417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Oberhuber?= Date: Thu, 10 Mar 2022 17:50:45 +0100 Subject: [PATCH 01/11] Added gradient descent method. --- .../Solvers/Optimization/GradientDescent.h | 59 ++++++++++++ .../Solvers/Optimization/GradientDescent.hpp | 93 +++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 src/TNL/Solvers/Optimization/GradientDescent.h create mode 100644 src/TNL/Solvers/Optimization/GradientDescent.hpp diff --git a/src/TNL/Solvers/Optimization/GradientDescent.h b/src/TNL/Solvers/Optimization/GradientDescent.h new file mode 100644 index 000000000..cdd30b190 --- /dev/null +++ b/src/TNL/Solvers/Optimization/GradientDescent.h @@ -0,0 +1,59 @@ +/*************************************************************************** + Timer.h - description + ------------------- + begin : Mar 14, 2016 + copyright : (C) 2016 by Tomas Oberhuber + email : tomas.oberhuber@fjfi.cvut.cz + ***************************************************************************/ + +/* See Copyright Notice in tnl/Copyright */ + +#pragma once + +#include + +namespace TNL { + namespace Solvers { + namespace Optimization { + +template< typename Vector, typename SolverMonitor = IterativeSolverMonitor< typename Vector::RealType, typename Vector::IndexType > > +class GradientDescent : public IterativeSolver< typename Vector::RealType, typename Vector::IndexType, SolverMonitor > +{ + + using RealType = typename Vector::RealType; + using DeviceType = typename Vector::DeviceType; + using IndexType = typename Vector::IndexType; + using VectorType = Vector; + using VectorView = typename Vector::ViewType; + + GradientDescent() = default; + + static void + configSetup( Config::ConfigDescription& config, const String& prefix = "" ); + + bool + setup( const Config::ParameterContainer& parameters, const String& prefix = "" ); + + void + setRelaxation( const RealType& lambda ); + + const RealType& + getRelaxation() const; + + template< typename GradientGetter > + bool + solve( VectorView& w, GradientGetter&& getGradient ); + +protected: + + RealType relaxation = 1.0; + + VectorType aux; + +}; + + } //namespace Optimization + } //namespace Solvers +} //namespace TNL + +#include diff --git a/src/TNL/Solvers/Optimization/GradientDescent.hpp b/src/TNL/Solvers/Optimization/GradientDescent.hpp new file mode 100644 index 000000000..2c46f9edd --- /dev/null +++ b/src/TNL/Solvers/Optimization/GradientDescent.hpp @@ -0,0 +1,93 @@ +/*************************************************************************** + Timer.h - description + ------------------- + begin : Mar 14, 2016 + copyright : (C) 2016 by Tomas Oberhuber + email : tomas.oberhuber@fjfi.cvut.cz + ***************************************************************************/ + +/* See Copyright Notice in tnl/Copyright */ + +#pragma once + +#include + +namespace TNL { + namespace Solvers { + namespace Optimization { + + +template< typename Vector, typename SolverMonitor > +void +GradientDescent< Vector, SolverMonitor >:: +configSetup( Config::ConfigDescription& config, const String& prefix ) +{ + IterativeSolver< RealType, IndexType, SolverMonitor >::configSetup( config, prefix ); + config.addEntry< double >( prefix + "gd-relaxation", "Relaxation parameter for the gradient descent.", 1.0 ) +} + +template< typename Vector, typename SolverMonitor > +bool +GradientDescent< Vector, SolverMonitor >:: +setup( const Config::ParameterContainer& parameters, const String& prefix ) +{ + this->setRelaxation( parameters.getParameter< double >( prefix + "gd-relaxation" ) ); + return IterativeSolver< RealType, IndexType, SolverMonitor >::setup( parameters, prefix ); +} + +template< typename Vector, typename SolverMonitor > +void +GradientDescent< Vector, SolverMonitor >:: +setRelaxation( const RealType& lambda ) +{ + this->relaxation = lambda; +} + +template< typename Vector, typename SolverMonitor > +auto +GradientDescent< Vector, SolverMonitor >:: +getRelaxation() const -> RealType& +{ + return this->relaxation; +} + +template< typename Vector, typename SolverMonitor > + template< typename GradientGetter > +bool +GradientDescent< Vector, SolverMonitor >:: +solve( VectorView& w, GradientGetter&& getGradient ) +{ + this->aux.setLike( w ); + auto aux_view = aux.getView(); + auto w_view = w.getView(); + aux = 0.0; + + ///// + // Set necessary parameters + this->resetIterations(); + this->setResidue( this->getConvergenceResidue() + 1.0 ); + + ///// + // Start the main loop + while( 1 ) + { + ///// + // Compute the gradient + getGradient( w_view, aux_view ); + RealType lastResidue = this->getResidue(); + this->setResidue( addAndReduceAbs( w_view, this->relaxation * aux_view, TNL::Plus(), ( RealType ) 0.0 ) / ( this->relaxation * ( RealType ) w.getSize() ) ); + + if( ! this->nextIteration() ) + return this->checkConvergence(); + + ///// + // Check the stop condition + if( this -> getConvergenceResidue() != 0.0 && this->getResidue() < this -> getConvergenceResidue() ) + return true; + } + return false; // just to avoid warnings +} + + } //namespace Optimization + } //namespace Solvers +} //namespace TNL -- GitLab From 1e473090fca436b62e4e33b78a1d3ac6ffe27d46 Mon Sep 17 00:00:00 2001 From: Tomas Oberhuber Date: Fri, 11 Mar 2022 10:37:40 +0100 Subject: [PATCH 02/11] Fixes in gradient descent method. --- src/TNL/Solvers/Optimization/GradientDescent.h | 2 +- src/TNL/Solvers/Optimization/GradientDescent.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/TNL/Solvers/Optimization/GradientDescent.h b/src/TNL/Solvers/Optimization/GradientDescent.h index cdd30b190..3261dea4b 100644 --- a/src/TNL/Solvers/Optimization/GradientDescent.h +++ b/src/TNL/Solvers/Optimization/GradientDescent.h @@ -19,7 +19,7 @@ namespace TNL { template< typename Vector, typename SolverMonitor = IterativeSolverMonitor< typename Vector::RealType, typename Vector::IndexType > > class GradientDescent : public IterativeSolver< typename Vector::RealType, typename Vector::IndexType, SolverMonitor > { - +public: using RealType = typename Vector::RealType; using DeviceType = typename Vector::DeviceType; using IndexType = typename Vector::IndexType; diff --git a/src/TNL/Solvers/Optimization/GradientDescent.hpp b/src/TNL/Solvers/Optimization/GradientDescent.hpp index 2c46f9edd..4f2886284 100644 --- a/src/TNL/Solvers/Optimization/GradientDescent.hpp +++ b/src/TNL/Solvers/Optimization/GradientDescent.hpp @@ -23,7 +23,7 @@ GradientDescent< Vector, SolverMonitor >:: configSetup( Config::ConfigDescription& config, const String& prefix ) { IterativeSolver< RealType, IndexType, SolverMonitor >::configSetup( config, prefix ); - config.addEntry< double >( prefix + "gd-relaxation", "Relaxation parameter for the gradient descent.", 1.0 ) + config.addEntry< double >( prefix + "gd-relaxation", "Relaxation parameter for the gradient descent.", 1.0 ); } template< typename Vector, typename SolverMonitor > -- GitLab From ea91987f96b7dc86ced54b7d7ef936c5bc070d89 Mon Sep 17 00:00:00 2001 From: Tomas Oberhuber Date: Sat, 12 Mar 2022 14:14:13 +0100 Subject: [PATCH 03/11] Fix in gradient descent method. --- src/TNL/Solvers/Optimization/GradientDescent.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TNL/Solvers/Optimization/GradientDescent.hpp b/src/TNL/Solvers/Optimization/GradientDescent.hpp index 4f2886284..9e53c8a81 100644 --- a/src/TNL/Solvers/Optimization/GradientDescent.hpp +++ b/src/TNL/Solvers/Optimization/GradientDescent.hpp @@ -46,7 +46,7 @@ setRelaxation( const RealType& lambda ) template< typename Vector, typename SolverMonitor > auto GradientDescent< Vector, SolverMonitor >:: -getRelaxation() const -> RealType& +getRelaxation() const -> const RealType& { return this->relaxation; } -- GitLab From cfc7a125b8cea5f6a987abeec42d738d92bde11b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Oberhuber?= Date: Mon, 14 Mar 2022 17:54:01 +0100 Subject: [PATCH 04/11] Renaming aux to gradient in the gradient descent method. --- .../Solvers/Optimization/GradientDescent.h | 14 ++++----- .../Solvers/Optimization/GradientDescent.hpp | 30 ++++++++----------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/src/TNL/Solvers/Optimization/GradientDescent.h b/src/TNL/Solvers/Optimization/GradientDescent.h index 3261dea4b..aa71b7309 100644 --- a/src/TNL/Solvers/Optimization/GradientDescent.h +++ b/src/TNL/Solvers/Optimization/GradientDescent.h @@ -1,12 +1,8 @@ -/*************************************************************************** - Timer.h - description - ------------------- - begin : Mar 14, 2016 - copyright : (C) 2016 by Tomas Oberhuber - email : tomas.oberhuber@fjfi.cvut.cz - ***************************************************************************/ - -/* See Copyright Notice in tnl/Copyright */ +// Copyright (c) 2004-2022 Tomáš Oberhuber et al. +// +// This file is part of TNL - Template Numerical Library (https://tnl-project.org/) +// +// SPDX-License-Identifier: MIT #pragma once diff --git a/src/TNL/Solvers/Optimization/GradientDescent.hpp b/src/TNL/Solvers/Optimization/GradientDescent.hpp index 9e53c8a81..aa81e3a77 100644 --- a/src/TNL/Solvers/Optimization/GradientDescent.hpp +++ b/src/TNL/Solvers/Optimization/GradientDescent.hpp @@ -1,12 +1,8 @@ -/*************************************************************************** - Timer.h - description - ------------------- - begin : Mar 14, 2016 - copyright : (C) 2016 by Tomas Oberhuber - email : tomas.oberhuber@fjfi.cvut.cz - ***************************************************************************/ - -/* See Copyright Notice in tnl/Copyright */ +// Copyright (c) 2004-2022 Tomáš Oberhuber et al. +// +// This file is part of TNL - Template Numerical Library (https://tnl-project.org/) +// +// SPDX-License-Identifier: MIT #pragma once @@ -23,7 +19,7 @@ GradientDescent< Vector, SolverMonitor >:: configSetup( Config::ConfigDescription& config, const String& prefix ) { IterativeSolver< RealType, IndexType, SolverMonitor >::configSetup( config, prefix ); - config.addEntry< double >( prefix + "gd-relaxation", "Relaxation parameter for the gradient descent.", 1.0 ); + config.addEntry< double >( prefix + "relaxation", "Relaxation parameter for the gradient descent.", 1.0 ); } template< typename Vector, typename SolverMonitor > @@ -31,7 +27,7 @@ bool GradientDescent< Vector, SolverMonitor >:: setup( const Config::ParameterContainer& parameters, const String& prefix ) { - this->setRelaxation( parameters.getParameter< double >( prefix + "gd-relaxation" ) ); + this->setRelaxation( parameters.getParameter< double >( prefix + "relaxation" ) ); return IterativeSolver< RealType, IndexType, SolverMonitor >::setup( parameters, prefix ); } @@ -57,10 +53,10 @@ bool GradientDescent< Vector, SolverMonitor >:: solve( VectorView& w, GradientGetter&& getGradient ) { - this->aux.setLike( w ); - auto aux_view = aux.getView(); + this->gradient.setLike( w ); + auto gradient_view = gradient.getView(); auto w_view = w.getView(); - aux = 0.0; + this->gradient = 0.0; ///// // Set necessary parameters @@ -73,16 +69,16 @@ solve( VectorView& w, GradientGetter&& getGradient ) { ///// // Compute the gradient - getGradient( w_view, aux_view ); + getGradient( w_view, gradient_view ); RealType lastResidue = this->getResidue(); - this->setResidue( addAndReduceAbs( w_view, this->relaxation * aux_view, TNL::Plus(), ( RealType ) 0.0 ) / ( this->relaxation * ( RealType ) w.getSize() ) ); + this->setResidue( addAndReduceAbs( w_view, -this->relaxation * gradient_view, TNL::Plus(), ( RealType ) 0.0 ) / ( this->relaxation * ( RealType ) w.getSize() ) ); if( ! this->nextIteration() ) return this->checkConvergence(); ///// // Check the stop condition - if( this -> getConvergenceResidue() != 0.0 && this->getResidue() < this -> getConvergenceResidue() ) + if( this->getConvergenceResidue() != 0.0 && this->getResidue() < this -> getConvergenceResidue() ) return true; } return false; // just to avoid warnings -- GitLab From 4faf5e4fbfd323bb6a85c81f76f20e621ecd5b54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Oberhuber?= Date: Mon, 14 Mar 2022 17:54:32 +0100 Subject: [PATCH 05/11] Added momentum method for optimization. --- src/TNL/Solvers/Optimization/Momentum.h | 61 ++++++++++++ src/TNL/Solvers/Optimization/Momentum.hpp | 112 ++++++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 src/TNL/Solvers/Optimization/Momentum.h create mode 100644 src/TNL/Solvers/Optimization/Momentum.hpp diff --git a/src/TNL/Solvers/Optimization/Momentum.h b/src/TNL/Solvers/Optimization/Momentum.h new file mode 100644 index 000000000..36ea38e05 --- /dev/null +++ b/src/TNL/Solvers/Optimization/Momentum.h @@ -0,0 +1,61 @@ +// Copyright (c) 2004-2022 Tomáš Oberhuber et al. +// +// This file is part of TNL - Template Numerical Library (https://tnl-project.org/) +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +namespace TNL { + namespace Solvers { + namespace Optimization { + +template< typename Vector, typename SolverMonitor = IterativeSolverMonitor< typename Vector::RealType, typename Vector::IndexType > > +class Momentum : public IterativeSolver< typename Vector::RealType, typename Vector::IndexType, SolverMonitor > +{ +public: + using RealType = typename Vector::RealType; + using DeviceType = typename Vector::DeviceType; + using IndexType = typename Vector::IndexType; + using VectorType = Vector; + using VectorView = typename Vector::ViewType; + + Momentum() = default; + + static void + configSetup( Config::ConfigDescription& config, const String& prefix = "" ); + + bool + setup( const Config::ParameterContainer& parameters, const String& prefix = "" ); + + void + setRelaxation( const RealType& lambda ); + + const RealType& + getRelaxation() const; + + void + setMomentum( const RealType& beta ); + + const RealType& + getMomentum() const; + + template< typename GradientGetter > + bool + solve( VectorView& w, GradientGetter&& getGradient ); + +protected: + + RealType relaxation = 1.0, momentum = 1.0; + + VectorType gradient, v; + +}; + + } //namespace Optimization + } //namespace Solvers +} //namespace TNL + +#include diff --git a/src/TNL/Solvers/Optimization/Momentum.hpp b/src/TNL/Solvers/Optimization/Momentum.hpp new file mode 100644 index 000000000..78f4e9c4a --- /dev/null +++ b/src/TNL/Solvers/Optimization/Momentum.hpp @@ -0,0 +1,112 @@ +// Copyright (c) 2004-2022 Tomáš Oberhuber et al. +// +// This file is part of TNL - Template Numerical Library (https://tnl-project.org/) +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +namespace TNL { + namespace Solvers { + namespace Optimization { + + +template< typename Vector, typename SolverMonitor > +void +Momentum< Vector, SolverMonitor >:: +configSetup( Config::ConfigDescription& config, const String& prefix ) +{ + IterativeSolver< RealType, IndexType, SolverMonitor >::configSetup( config, prefix ); + config.addEntry< double >( prefix + "relaxation", "Relaxation parameter for the momentum method.", 1.0 ); + config.addEntry< double >( prefix + "momentum", "Momentum parameter for the momentum method.", 1.0 ); +} + +template< typename Vector, typename SolverMonitor > +bool +Momentum< Vector, SolverMonitor >:: +setup( const Config::ParameterContainer& parameters, const String& prefix ) +{ + this->setRelaxation( parameters.getParameter< double >( prefix + "relaxation" ) ); + this->setMomentum( parameters.getParameter< double >( prefix + "momentum" ) ); + return IterativeSolver< RealType, IndexType, SolverMonitor >::setup( parameters, prefix ); +} + +template< typename Vector, typename SolverMonitor > +void +Momentum< Vector, SolverMonitor >:: +setRelaxation( const RealType& lambda ) +{ + this->relaxation = lambda; +} + +template< typename Vector, typename SolverMonitor > +auto +Momentum< Vector, SolverMonitor >:: +getRelaxation() const -> const RealType& +{ + return this->relaxation; +} + +template< typename Vector, typename SolverMonitor > +void +Momentum< Vector, SolverMonitor >:: +setMomentum( const RealType& beta ) +{ + this->momentum = beta; +} + +template< typename Vector, typename SolverMonitor > +auto +Momentum< Vector, SolverMonitor >:: +getMomentum() const -> const RealType& +{ + return this->momentum; +} + +template< typename Vector, typename SolverMonitor > + template< typename GradientGetter > +bool +Momentum< Vector, SolverMonitor >:: +solve( VectorView& w, GradientGetter&& getGradient ) +{ + this->gradient.setLike( w ); + this->v.setLike( w ); + auto gradient_view = gradient.getView(); + auto w_view = w.getView(); + auto v_view = v.getView(); + this->gradient = 0.0; + this->v = 0.0; + + ///// + // Set necessary parameters + this->resetIterations(); + this->setResidue( this->getConvergenceResidue() + 1.0 ); + + ///// + // Start the main loop + while( 1 ) + { + ///// + // Compute the gradient + getGradient( w_view, gradient_view ); + v_view = this->momentum * v_view - this->relaxation * gradient_view; + + RealType lastResidue = this->getResidue(); + this->setResidue( addAndReduceAbs( w_view, v_view, TNL::Plus(), ( RealType ) 0.0 ) / ( this->relaxation * ( RealType ) w.getSize() ) ); + + if( ! this->nextIteration() ) + return this->checkConvergence(); + + ///// + // Check the stop condition + if( this->getConvergenceResidue() != 0.0 && this->getResidue() < this -> getConvergenceResidue() ) + return true; + } + return false; // just to avoid warnings +} + + } //namespace Optimization + } //namespace Solvers +} //namespace TNL -- GitLab From 98429a428dd18777b3890cc083fbf0ea70c6a361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Oberhuber?= Date: Mon, 14 Mar 2022 18:04:26 +0100 Subject: [PATCH 06/11] Renaming aux to gradient in the gradient descent method. --- src/TNL/Solvers/Optimization/GradientDescent.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TNL/Solvers/Optimization/GradientDescent.h b/src/TNL/Solvers/Optimization/GradientDescent.h index aa71b7309..6bc2076a0 100644 --- a/src/TNL/Solvers/Optimization/GradientDescent.h +++ b/src/TNL/Solvers/Optimization/GradientDescent.h @@ -44,7 +44,7 @@ protected: RealType relaxation = 1.0; - VectorType aux; + VectorType gradient; }; -- GitLab From c07162420a7155eb7c792a10faf2162995b84da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Oberhuber?= Date: Mon, 14 Mar 2022 18:41:01 +0100 Subject: [PATCH 07/11] Fixes in the memomentum optimization method. --- src/TNL/Solvers/Optimization/Momentum.h | 2 +- src/TNL/Solvers/Optimization/Momentum.hpp | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/TNL/Solvers/Optimization/Momentum.h b/src/TNL/Solvers/Optimization/Momentum.h index 36ea38e05..48b19c2b1 100644 --- a/src/TNL/Solvers/Optimization/Momentum.h +++ b/src/TNL/Solvers/Optimization/Momentum.h @@ -48,7 +48,7 @@ public: protected: - RealType relaxation = 1.0, momentum = 1.0; + RealType relaxation = 1.0, momentum = 0.9; VectorType gradient, v; diff --git a/src/TNL/Solvers/Optimization/Momentum.hpp b/src/TNL/Solvers/Optimization/Momentum.hpp index 78f4e9c4a..2a56c2f55 100644 --- a/src/TNL/Solvers/Optimization/Momentum.hpp +++ b/src/TNL/Solvers/Optimization/Momentum.hpp @@ -20,7 +20,7 @@ configSetup( Config::ConfigDescription& config, const String& prefix ) { IterativeSolver< RealType, IndexType, SolverMonitor >::configSetup( config, prefix ); config.addEntry< double >( prefix + "relaxation", "Relaxation parameter for the momentum method.", 1.0 ); - config.addEntry< double >( prefix + "momentum", "Momentum parameter for the momentum method.", 1.0 ); + config.addEntry< double >( prefix + "momentum", "Momentum parameter for the momentum method.", 0.9 ); } template< typename Vector, typename SolverMonitor > @@ -94,7 +94,14 @@ solve( VectorView& w, GradientGetter&& getGradient ) v_view = this->momentum * v_view - this->relaxation * gradient_view; RealType lastResidue = this->getResidue(); - this->setResidue( addAndReduceAbs( w_view, v_view, TNL::Plus(), ( RealType ) 0.0 ) / ( this->relaxation * ( RealType ) w.getSize() ) ); + this->setResidue( + Algorithms::reduce< DeviceType >( + ( IndexType ) 0, w_view.getSize(), + [=] __cuda_callable__ ( IndexType i ) mutable { + w_view[ i ] += v_view[ i ]; + return abs( v_view[ i ] ); + }, + TNL::Plus() ) / ( this->relaxation * ( RealType ) w.getSize() ) ); if( ! this->nextIteration() ) return this->checkConvergence(); -- GitLab From 597daa67c27bdeb8a1e9b1e18f6659b1e466d5fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Oberhuber?= Date: Thu, 17 Mar 2022 10:19:46 +0100 Subject: [PATCH 08/11] Added Nesterov momentum method. --- .../Solvers/Optimization/NesterovMomentum.h | 61 +++++++++ .../Solvers/Optimization/NesterovMomentum.hpp | 122 ++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 src/TNL/Solvers/Optimization/NesterovMomentum.h create mode 100644 src/TNL/Solvers/Optimization/NesterovMomentum.hpp diff --git a/src/TNL/Solvers/Optimization/NesterovMomentum.h b/src/TNL/Solvers/Optimization/NesterovMomentum.h new file mode 100644 index 000000000..20a3b7c8f --- /dev/null +++ b/src/TNL/Solvers/Optimization/NesterovMomentum.h @@ -0,0 +1,61 @@ +// Copyright (c) 2004-2022 Tomáš Oberhuber et al. +// +// This file is part of TNL - Template Numerical Library (https://tnl-project.org/) +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +namespace TNL { + namespace Solvers { + namespace Optimization { + +template< typename Vector, typename SolverMonitor = IterativeSolverMonitor< typename Vector::RealType, typename Vector::IndexType > > +class NesterovMomentum : public IterativeSolver< typename Vector::RealType, typename Vector::IndexType, SolverMonitor > +{ +public: + using RealType = typename Vector::RealType; + using DeviceType = typename Vector::DeviceType; + using IndexType = typename Vector::IndexType; + using VectorType = Vector; + using VectorView = typename Vector::ViewType; + + NesterovMomentum() = default; + + static void + configSetup( Config::ConfigDescription& config, const String& prefix = "" ); + + bool + setup( const Config::ParameterContainer& parameters, const String& prefix = "" ); + + void + setRelaxation( const RealType& lambda ); + + const RealType& + getRelaxation() const; + + void + setMomentum( const RealType& beta ); + + const RealType& + getMomentum() const; + + template< typename GradientGetter > + bool + solve( VectorView& w, GradientGetter&& getGradient ); + +protected: + + RealType relaxation = 1.0, momentum = 0.9; + + VectorType gradient, v, aux; + +}; + + } //namespace Optimization + } //namespace Solvers +} //namespace TNL + +#include diff --git a/src/TNL/Solvers/Optimization/NesterovMomentum.hpp b/src/TNL/Solvers/Optimization/NesterovMomentum.hpp new file mode 100644 index 000000000..9e5b91385 --- /dev/null +++ b/src/TNL/Solvers/Optimization/NesterovMomentum.hpp @@ -0,0 +1,122 @@ +// Copyright (c) 2004-2022 Tomáš Oberhuber et al. +// +// This file is part of TNL - Template Numerical Library (https://tnl-project.org/) +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +namespace TNL { + namespace Solvers { + namespace Optimization { + + +template< typename Vector, typename SolverMonitor > +void +NesterovMomentum< Vector, SolverMonitor >:: +configSetup( Config::ConfigDescription& config, const String& prefix ) +{ + IterativeSolver< RealType, IndexType, SolverMonitor >::configSetup( config, prefix ); + config.addEntry< double >( prefix + "relaxation", "Relaxation parameter for the momentum method.", 1.0 ); + config.addEntry< double >( prefix + "momentum", "Momentum parameter for the momentum method.", 0.9 ); +} + +template< typename Vector, typename SolverMonitor > +bool +NesterovMomentum< Vector, SolverMonitor >:: +setup( const Config::ParameterContainer& parameters, const String& prefix ) +{ + this->setRelaxation( parameters.getParameter< double >( prefix + "relaxation" ) ); + this->setMomentum( parameters.getParameter< double >( prefix + "momentum" ) ); + return IterativeSolver< RealType, IndexType, SolverMonitor >::setup( parameters, prefix ); +} + +template< typename Vector, typename SolverMonitor > +void +NesterovMomentum< Vector, SolverMonitor >:: +setRelaxation( const RealType& lambda ) +{ + this->relaxation = lambda; +} + +template< typename Vector, typename SolverMonitor > +auto +NesterovMomentum< Vector, SolverMonitor >:: +getRelaxation() const -> const RealType& +{ + return this->relaxation; +} + +template< typename Vector, typename SolverMonitor > +void +NesterovMomentum< Vector, SolverMonitor >:: +setMomentum( const RealType& beta ) +{ + this->momentum = beta; +} + +template< typename Vector, typename SolverMonitor > +auto +NesterovMomentum< Vector, SolverMonitor >:: +getMomentum() const -> const RealType& +{ + return this->momentum; +} + +template< typename Vector, typename SolverMonitor > + template< typename GradientGetter > +bool +NesterovMomentum< Vector, SolverMonitor >:: +solve( VectorView& w, GradientGetter&& getGradient ) +{ + this->gradient.setLike( w ); + this->v.setLike( w ); + this->aux.setLike( w ); + auto gradient_view = gradient.getView(); + auto w_view = w.getView(); + auto v_view = v.getView(); + auto aux_view = aux.getView(); + this->gradient = 0.0; + this->v = 0.0; + + ///// + // Set necessary parameters + this->resetIterations(); + this->setResidue( this->getConvergenceResidue() + 1.0 ); + + ///// + // Start the main loop + while( 1 ) + { + ///// + // Compute the gradient + aux_view = w_view + v_view; + getGradient( aux_view, gradient_view ); + v_view = this->momentum * v_view - this->relaxation * gradient_view; + + RealType lastResidue = this->getResidue(); + this->setResidue( + Algorithms::reduce< DeviceType >( + ( IndexType ) 0, w_view.getSize(), + [=] __cuda_callable__ ( IndexType i ) mutable { + w_view[ i ] += v_view[ i ]; + return abs( v_view[ i ] ); + }, + TNL::Plus() ) / ( this->relaxation * ( RealType ) w.getSize() ) ); + + if( ! this->nextIteration() ) + return this->checkConvergence(); + + ///// + // Check the stop condition + if( this->getConvergenceResidue() != 0.0 && this->getResidue() < this -> getConvergenceResidue() ) + return true; + } + return false; // just to avoid warnings +} + + } //namespace Optimization + } //namespace Solvers +} //namespace TNL -- GitLab From 376063bdeb905ad72f96929e62612ec323472a7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Oberhuber?= Date: Sun, 3 Apr 2022 20:20:40 +0200 Subject: [PATCH 09/11] Fixed header including in NesterovMomentum method. --- src/TNL/Solvers/Optimization/NesterovMomentum.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TNL/Solvers/Optimization/NesterovMomentum.h b/src/TNL/Solvers/Optimization/NesterovMomentum.h index 20a3b7c8f..3fd0fd740 100644 --- a/src/TNL/Solvers/Optimization/NesterovMomentum.h +++ b/src/TNL/Solvers/Optimization/NesterovMomentum.h @@ -58,4 +58,4 @@ protected: } //namespace Solvers } //namespace TNL -#include +#include -- GitLab From 6c4f81bc0320c303ed4c36ea1c4291809fead507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Oberhuber?= Date: Sun, 3 Apr 2022 20:21:05 +0200 Subject: [PATCH 10/11] Added AdaGrad optimizer. --- src/TNL/Solvers/Optimization/AdaGrad.h | 59 +++++++++++++++ src/TNL/Solvers/Optimization/AdaGrad.hpp | 93 ++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 src/TNL/Solvers/Optimization/AdaGrad.h create mode 100644 src/TNL/Solvers/Optimization/AdaGrad.hpp diff --git a/src/TNL/Solvers/Optimization/AdaGrad.h b/src/TNL/Solvers/Optimization/AdaGrad.h new file mode 100644 index 000000000..247457859 --- /dev/null +++ b/src/TNL/Solvers/Optimization/AdaGrad.h @@ -0,0 +1,59 @@ +// Copyright (c) 2004-2022 Tomáš Oberhuber et al. +// +// This file is part of TNL - Template Numerical Library (https://tnl-project.org/) +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +namespace TNL { + namespace Solvers { + namespace Optimization { + +/*** + * https://arxiv.org/pdf/1609.04747.pdf + * + */ +template< typename Vector, typename SolverMonitor = IterativeSolverMonitor< typename Vector::RealType, typename Vector::IndexType > > +class AdaGrad : public IterativeSolver< typename Vector::RealType, typename Vector::IndexType, SolverMonitor > +{ +public: + using RealType = typename Vector::RealType; + using DeviceType = typename Vector::DeviceType; + using IndexType = typename Vector::IndexType; + using VectorType = Vector; + using VectorView = typename Vector::ViewType; + + AdaGrad() = default; + + static void + configSetup( Config::ConfigDescription& config, const String& prefix = "" ); + + bool + setup( const Config::ParameterContainer& parameters, const String& prefix = "" ); + + void + setRelaxation( const RealType& lambda ); + + const RealType& + getRelaxation() const; + + template< typename GradientGetter > + bool + solve( VectorView& w, GradientGetter&& getGradient ); + +protected: + + RealType relaxation = 1.0, epsilon = 1.0e-8; + + VectorType gradient, a; + +}; + + } //namespace Optimization + } //namespace Solvers +} //namespace TNL + +#include diff --git a/src/TNL/Solvers/Optimization/AdaGrad.hpp b/src/TNL/Solvers/Optimization/AdaGrad.hpp new file mode 100644 index 000000000..6ac8b63fd --- /dev/null +++ b/src/TNL/Solvers/Optimization/AdaGrad.hpp @@ -0,0 +1,93 @@ +// Copyright (c) 2004-2022 Tomáš Oberhuber et al. +// +// This file is part of TNL - Template Numerical Library (https://tnl-project.org/) +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +namespace TNL { + namespace Solvers { + namespace Optimization { + + +template< typename Vector, typename SolverMonitor > +void +AdaGrad< Vector, SolverMonitor >:: +configSetup( Config::ConfigDescription& config, const String& prefix ) +{ + IterativeSolver< RealType, IndexType, SolverMonitor >::configSetup( config, prefix ); + config.addEntry< double >( prefix + "relaxation", "Relaxation parameter for the gradient descent.", 1.0 ); +} + +template< typename Vector, typename SolverMonitor > +bool +AdaGrad< Vector, SolverMonitor >:: +setup( const Config::ParameterContainer& parameters, const String& prefix ) +{ + this->setRelaxation( parameters.getParameter< double >( prefix + "relaxation" ) ); + return IterativeSolver< RealType, IndexType, SolverMonitor >::setup( parameters, prefix ); +} + +template< typename Vector, typename SolverMonitor > +void +AdaGrad< Vector, SolverMonitor >:: +setRelaxation( const RealType& lambda ) +{ + this->relaxation = lambda; +} + +template< typename Vector, typename SolverMonitor > +auto +AdaGrad< Vector, SolverMonitor >:: +getRelaxation() const -> const RealType& +{ + return this->relaxation; +} + +template< typename Vector, typename SolverMonitor > + template< typename GradientGetter > +bool +AdaGrad< Vector, SolverMonitor >:: +solve( VectorView& w, GradientGetter&& getGradient ) +{ + this->gradient.setLike( w ); + this->a.setLike( w ); + auto gradient_view = gradient.getView(); + auto w_view = w.getView(); + this->gradient = 0.0; + this->a = 0.0; + + ///// + // Set necessary parameters + this->resetIterations(); + this->setResidue( this->getConvergenceResidue() + 1.0 ); + + ///// + // Start the main loop + while( 1 ) + { + ///// + // Compute the gradient + getGradient( w_view, gradient_view ); + RealType lastResidue = this->getResidue(); + // a_i += grad_i^2 + a += gradient_view * gradient_view; + this->setResidue( addAndReduceAbs( w_view, -this->relaxation / sqrt( this->a + this->epsilon ) * gradient_view, TNL::Plus(), ( RealType ) 0.0 ) / ( this->relaxation * ( RealType ) w.getSize() ) ); + + if( ! this->nextIteration() ) + return this->checkConvergence(); + + ///// + // Check the stop condition + if( this->getConvergenceResidue() != 0.0 && this->getResidue() < this -> getConvergenceResidue() ) + return true; + } + return false; // just to avoid warnings +} + + } //namespace Optimization + } //namespace Solvers +} //namespace TNL -- GitLab From 1c600882ec6b989859187ae00c66cd12a0489de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Oberhuber?= Date: Sun, 3 Apr 2022 20:34:41 +0200 Subject: [PATCH 11/11] Added RMSProp optimizer. --- src/TNL/Solvers/Optimization/RMSProp.h | 59 +++++++++++++++ src/TNL/Solvers/Optimization/RMSProp.hpp | 95 ++++++++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 src/TNL/Solvers/Optimization/RMSProp.h create mode 100644 src/TNL/Solvers/Optimization/RMSProp.hpp diff --git a/src/TNL/Solvers/Optimization/RMSProp.h b/src/TNL/Solvers/Optimization/RMSProp.h new file mode 100644 index 000000000..06b12c71e --- /dev/null +++ b/src/TNL/Solvers/Optimization/RMSProp.h @@ -0,0 +1,59 @@ +// Copyright (c) 2004-2022 Tomáš Oberhuber et al. +// +// This file is part of TNL - Template Numerical Library (https://tnl-project.org/) +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +namespace TNL { + namespace Solvers { + namespace Optimization { + +/*** + * https://arxiv.org/pdf/1609.04747.pdf + * + */ +template< typename Vector, typename SolverMonitor = IterativeSolverMonitor< typename Vector::RealType, typename Vector::IndexType > > +class RMSProp : public IterativeSolver< typename Vector::RealType, typename Vector::IndexType, SolverMonitor > +{ +public: + using RealType = typename Vector::RealType; + using DeviceType = typename Vector::DeviceType; + using IndexType = typename Vector::IndexType; + using VectorType = Vector; + using VectorView = typename Vector::ViewType; + + RMSProp() = default; + + static void + configSetup( Config::ConfigDescription& config, const String& prefix = "" ); + + bool + setup( const Config::ParameterContainer& parameters, const String& prefix = "" ); + + void + setRelaxation( const RealType& lambda ); + + const RealType& + getRelaxation() const; + + template< typename GradientGetter > + bool + solve( VectorView& w, GradientGetter&& getGradient ); + +protected: + + RealType relaxation = 1.0, epsilon = 1.0e-8, beta = 0.9; + + VectorType gradient, a; + +}; + + } //namespace Optimization + } //namespace Solvers +} //namespace TNL + +#include diff --git a/src/TNL/Solvers/Optimization/RMSProp.hpp b/src/TNL/Solvers/Optimization/RMSProp.hpp new file mode 100644 index 000000000..cb40c3d3d --- /dev/null +++ b/src/TNL/Solvers/Optimization/RMSProp.hpp @@ -0,0 +1,95 @@ +// Copyright (c) 2004-2022 Tomáš Oberhuber et al. +// +// This file is part of TNL - Template Numerical Library (https://tnl-project.org/) +// +// SPDX-License-Identifier: MIT + +#pragma once + +#include + +namespace TNL { + namespace Solvers { + namespace Optimization { + + +template< typename Vector, typename SolverMonitor > +void +RMSProp< Vector, SolverMonitor >:: +configSetup( Config::ConfigDescription& config, const String& prefix ) +{ + IterativeSolver< RealType, IndexType, SolverMonitor >::configSetup( config, prefix ); + config.addEntry< double >( prefix + "relaxation", "Relaxation parameter for the gradient descent.", 1.0 ); + config.addEntry< double >( prefix + "beta", "Momentum parameter for computing sum of squared gradients.", 0.9 ); +} + +template< typename Vector, typename SolverMonitor > +bool +RMSProp< Vector, SolverMonitor >:: +setup( const Config::ParameterContainer& parameters, const String& prefix ) +{ + this->setRelaxation( parameters.getParameter< double >( prefix + "relaxation" ) ); + this->beta = parameters.getParameter< double >( prefix + "beta" ); + return IterativeSolver< RealType, IndexType, SolverMonitor >::setup( parameters, prefix ); +} + +template< typename Vector, typename SolverMonitor > +void +RMSProp< Vector, SolverMonitor >:: +setRelaxation( const RealType& lambda ) +{ + this->relaxation = lambda; +} + +template< typename Vector, typename SolverMonitor > +auto +RMSProp< Vector, SolverMonitor >:: +getRelaxation() const -> const RealType& +{ + return this->relaxation; +} + +template< typename Vector, typename SolverMonitor > + template< typename GradientGetter > +bool +RMSProp< Vector, SolverMonitor >:: +solve( VectorView& w, GradientGetter&& getGradient ) +{ + this->gradient.setLike( w ); + this->a.setLike( w ); + auto gradient_view = gradient.getView(); + auto w_view = w.getView(); + this->gradient = 0.0; + this->a = 0.0; + + ///// + // Set necessary parameters + this->resetIterations(); + this->setResidue( this->getConvergenceResidue() + 1.0 ); + + ///// + // Start the main loop + while( 1 ) + { + ///// + // Compute the gradient + getGradient( w_view, gradient_view ); + RealType lastResidue = this->getResidue(); + // a_i = beta * a_i + ( 1- beta ) * grad_i^2 + a = this->beta * a + ( 1.0 - this->beta ) * gradient_view * gradient_view; + this->setResidue( addAndReduceAbs( w_view, -this->relaxation / sqrt( this->a + this->epsilon ) * gradient_view, TNL::Plus(), ( RealType ) 0.0 ) / ( this->relaxation * ( RealType ) w.getSize() ) ); + + if( ! this->nextIteration() ) + return this->checkConvergence(); + + ///// + // Check the stop condition + if( this->getConvergenceResidue() != 0.0 && this->getResidue() < this -> getConvergenceResidue() ) + return true; + } + return false; // just to avoid warnings +} + + } //namespace Optimization + } //namespace Solvers +} //namespace TNL -- GitLab