From 52244d6b010960fd370e46f7476534ecc9140e44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Sat, 11 Jan 2020 09:34:31 +0100 Subject: [PATCH 1/9] Added CGS and CGSR variants of GMRES --- src/TNL/Solvers/Linear/GMRES.h | 11 ++- src/TNL/Solvers/Linear/GMRES_impl.h | 112 +++++++++++++++++++++++++++- 2 files changed, 119 insertions(+), 4 deletions(-) diff --git a/src/TNL/Solvers/Linear/GMRES.h b/src/TNL/Solvers/Linear/GMRES.h index f1a4b8732..e1c02f0ab 100644 --- a/src/TNL/Solvers/Linear/GMRES.h +++ b/src/TNL/Solvers/Linear/GMRES.h @@ -53,7 +53,16 @@ protected: using HostView = typename DeviceView::template Self< RealType, Devices::Host >; using HostVector = typename DeviceVector::template Self< RealType, Devices::Host >;; - enum class Variant { MGS, MGSR, CWY }; + enum class Variant { CGS, CGSR, MGS, MGSR, CWY }; + +// nvcc allows __cuda_callable__ lambdas only in public methods +#ifdef __NVCC__ +public: +#endif + int orthogonalize_CGS( const int m, const RealType normb, const RealType beta ); +#ifdef __NVCC__ +protected: +#endif int orthogonalize_MGS( const int m, const RealType normb, const RealType beta ); diff --git a/src/TNL/Solvers/Linear/GMRES_impl.h b/src/TNL/Solvers/Linear/GMRES_impl.h index d6cb8fdd0..8e653e28b 100644 --- a/src/TNL/Solvers/Linear/GMRES_impl.h +++ b/src/TNL/Solvers/Linear/GMRES_impl.h @@ -31,6 +31,8 @@ configSetup( Config::ConfigDescription& config, const String& prefix ) { config.addEntry< String >( prefix + "gmres-variant", "Minimal number of iterations after which the GMRES restarts.", "CWY" ); + config.addEntryEnum( "CGS" ); + config.addEntryEnum( "CGSR" ); config.addEntryEnum( "MGS" ); config.addEntryEnum( "MGSR" ); config.addEntryEnum( "CWY" ); @@ -47,7 +49,11 @@ setup( const Config::ParameterContainer& parameters, const String& prefix ) { const String var = parameters.getParameter< String >( prefix + "gmres-variant" ); - if( var == "MGS" ) + if( var == "CGS" ) + variant = Variant::CGS; + else if( var == "CGSR" ) + variant = Variant::CGSR; + else if( var == "MGS" ) variant = Variant::MGS; else if( var == "MGSR" ) variant = Variant::MGSR; @@ -133,6 +139,10 @@ solve( ConstVectorViewType b, VectorViewType x ) // orthogonalization int o_steps = 0; switch( variant ) { + case Variant::CGS: + case Variant::CGSR: + o_steps = orthogonalize_CGS( m, normb, beta ); + break; case Variant::MGS: case Variant::MGSR: o_steps = orthogonalize_MGS( m, normb, beta ); @@ -167,6 +177,102 @@ solve( ConstVectorViewType b, VectorViewType x ) return this->checkConvergence(); } +template< typename Matrix > +int +GMRES< Matrix >:: +orthogonalize_CGS( const int m, const RealType normb, const RealType beta ) +{ + // initial binding to _M_tmp sets the correct local range, global size and + // communication group for distributed views + VectorViewType v_i( _M_tmp.getView() ); + VectorViewType v_k( _M_tmp.getView() ); + + /*** + * v_0 = r / | r | = 1.0 / beta * r + */ + v_i.bind( V.getData(), size ); + v_i = (1.0 / beta) * r; + + H.setValue( 0.0 ); + s.setValue( 0.0 ); + s[ 0 ] = beta; + + /**** + * Starting m-loop + */ + for( int i = 0; i < m && this->nextIteration(); i++ ) { + v_i.bind( &V.getData()[ i * ldSize ], size ); + /**** + * Solve w from M w = A v_i + */ + preconditioned_matvec( w, v_i ); + + for( int k = 0; k <= i; k++ ) + H[ k + i * (m + 1) ] = 0.0; + const int reorthogonalize = (variant == Variant::CGSR) ? 2 : 1; + for( int l = 0; l < reorthogonalize; l++ ) { + // auxiliary array for the H coefficients of the current l-loop + RealType H_l[i + 1]; + + // CGS part 1: compute projection coefficients +// for( int k = 0; k <= i; k++ ) { +// v_k.bind( &V.getData()[ k * ldSize ], size ); +// H_l[k] = (w, v_k); +// H[ k + i * (m + 1) ] += H_l[k]; +// } + // H_l = V_i^T * w + const RealType* _V = V.getData(); + const RealType* _w = Traits::getConstLocalView( w ).getData(); + const IndexType ldSize = this->ldSize; + auto fetch = [_V, _w, ldSize] __cuda_callable__ ( IndexType idx, int k ) { return _V[ idx + k * ldSize ] * _w[ idx ]; }; + Algorithms::Multireduction< DeviceType >::reduce + ( (RealType) 0, + fetch, + std::plus<>{}, + size, + i + 1, + H_l ); + for( int k = 0; k <= i; k++ ) + H[ k + i * (m + 1) ] += H_l[k]; + + // CGS part 2: subtract the projections +// for( int k = 0; k <= i; k++ ) { +// v_k.bind( &V.getData()[ k * ldSize ], size ); +// w = w - H_l[k] * v_k; +// } + // w := w - V_i * H_l + Matrices::MatrixOperations< DeviceType >:: + gemv( size, i + 1, + -1.0, V.getData(), ldSize, H_l, + 1.0, Traits::getLocalView( w ).getData() ); + } + /*** + * H_{i+1,i} = |w| + */ + RealType normw = lpNorm( w, 2.0 ); + H[ i + 1 + i * (m + 1) ] = normw; + + /*** + * v_{i+1} = w / |w| + */ + v_i.bind( &V.getData()[ ( i + 1 ) * ldSize ], size ); + v_i = (1.0 / normw) * w; + + /**** + * Applying the Givens rotations G_0, ..., G_i + */ + apply_givens_rotations( i, m ); + + this->setResidue( std::fabs( s[ i + 1 ] ) / normb ); + if( ! this->checkNextIteration() ) + return i; + else + this->refreshSolverMonitor(); + } + + return m; +} + template< typename Matrix > int GMRES< Matrix >:: @@ -198,13 +304,13 @@ orthogonalize_MGS( const int m, const RealType normb, const RealType beta ) preconditioned_matvec( w, v_i ); for( int k = 0; k <= i; k++ ) - H[ k + i * ( m + 1 ) ] = 0.0; + H[ k + i * (m + 1) ] = 0.0; const int reorthogonalize = (variant == Variant::MGSR) ? 2 : 1; for( int l = 0; l < reorthogonalize; l++ ) for( int k = 0; k <= i; k++ ) { v_k.bind( &V.getData()[ k * ldSize ], size ); /*** - * H_{k,i} = ( w, v_k ) + * H_{k,i} = (w, v_k) */ RealType H_k_i = (w, v_k); H[ k + i * (m + 1) ] += H_k_i; -- GitLab From c380eb1300319bb4a3e1a21c9978fcfef0a9935c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Sat, 11 Jan 2020 09:50:57 +0100 Subject: [PATCH 2/9] Cosmetic changes in IterativeSolver --- src/TNL/Solvers/IterativeSolver.h | 37 ++++---- src/TNL/Solvers/IterativeSolver_impl.h | 101 +++++++++++++--------- src/TNL/Solvers/ODE/ExplicitSolver_impl.h | 2 + 3 files changed, 79 insertions(+), 61 deletions(-) diff --git a/src/TNL/Solvers/IterativeSolver.h b/src/TNL/Solvers/IterativeSolver.h index 6550e5e49..c7c903cfc 100644 --- a/src/TNL/Solvers/IterativeSolver.h +++ b/src/TNL/Solvers/IterativeSolver.h @@ -2,7 +2,7 @@ IterativeSolver.h - description ------------------- begin : Oct 19, 2012 - copyright : (C) 2012 by Tomas Oberhuber + copyright : (C) 2012 by Tomas Oberhuber et al. email : tomas.oberhuber@fjfi.cvut.cz ***************************************************************************/ @@ -10,6 +10,8 @@ #pragma once +#include + #include #include #include @@ -22,17 +24,16 @@ template< typename Real, typename SolverMonitor = IterativeSolverMonitor< Real, Index > > class IterativeSolver { - public: +public: + using SolverMonitorType = SolverMonitor; - using SolverMonitorType = SolverMonitor; - - IterativeSolver(); + IterativeSolver() = default; static void configSetup( Config::ConfigDescription& config, const String& prefix = "" ); bool setup( const Config::ParameterContainer& parameters, - const String& prefix = "" ); + const String& prefix = "" ); void setMaxIterations( const Index& maxIterations ); @@ -70,27 +71,23 @@ class IterativeSolver void refreshSolverMonitor( bool force = false ); +protected: + Index maxIterations = 1000000000; - protected: - - Index maxIterations; - - Index minIterations; + Index minIterations = 0; - Index currentIteration; + Index currentIteration = 0; - Real convergenceResidue; + Real convergenceResidue = 1e-6; - /**** - * If the current residue is over divergenceResidue the solver is stopped. - */ - Real divergenceResidue; + // If the current residue is greater than divergenceResidue, the solver is stopped. + Real divergenceResidue = std::numeric_limits< float >::max(); - Real currentResidue; + Real currentResidue = 0; - SolverMonitor* solverMonitor; + SolverMonitor* solverMonitor = nullptr; - Index refreshRate; + Index refreshRate = 1; }; } // namespace Solvers diff --git a/src/TNL/Solvers/IterativeSolver_impl.h b/src/TNL/Solvers/IterativeSolver_impl.h index 528a08ce6..b673a5c17 100644 --- a/src/TNL/Solvers/IterativeSolver_impl.h +++ b/src/TNL/Solvers/IterativeSolver_impl.h @@ -2,7 +2,7 @@ IterativeSolver_impl.h - description ------------------- begin : Oct 19, 2012 - copyright : (C) 2012 by Tomas Oberhuber + copyright : (C) 2012 by Tomas Oberhuber et al. email : tomas.oberhuber@fjfi.cvut.cz ***************************************************************************/ @@ -11,34 +11,21 @@ #pragma once #include -#include -#include #include "IterativeSolver.h" namespace TNL { -namespace Solvers { +namespace Solvers { template< typename Real, typename Index, typename SolverMonitor > -IterativeSolver< Real, Index, SolverMonitor >::IterativeSolver() -: maxIterations( 1000000000 ), - minIterations( 0 ), - currentIteration( 0 ), - convergenceResidue( 1.0e-6 ), - divergenceResidue( DBL_MAX ), - currentResidue( 0 ), - solverMonitor( 0 ), - refreshRate( 1 ) -{ -}; - -template< typename Real, typename Index, typename SolverMonitor > -void IterativeSolver< Real, Index, SolverMonitor >::configSetup( Config::ConfigDescription& config, - const String& prefix ) +void +IterativeSolver< Real, Index, SolverMonitor >:: +configSetup( Config::ConfigDescription& config, + const String& prefix ) { config.addEntry< int > ( prefix + "max-iterations", "Maximal number of iterations the solver may perform.", 1000000000 ); config.addEntry< int > ( prefix + "min-iterations", "Minimal number of iterations the solver must perform.", 0 ); - + // The default value for the convergence residue MUST be zero since not in all problems we want to stop the solver // when we reach a state near a steady state. This can be only temporary if, for example, when the boundary conditions // are time dependent (growing velocity at inlet starting from 0). @@ -49,8 +36,10 @@ void IterativeSolver< Real, Index, SolverMonitor >::configSetup( Config::ConfigD } template< typename Real, typename Index, typename SolverMonitor > -bool IterativeSolver< Real, Index, SolverMonitor >::setup( const Config::ParameterContainer& parameters, - const String& prefix ) +bool +IterativeSolver< Real, Index, SolverMonitor >:: +setup( const Config::ParameterContainer& parameters, + const String& prefix ) { this->setMaxIterations( parameters.getParameter< int >( "max-iterations" ) ); this->setMinIterations( parameters.getParameter< int >( "min-iterations" ) ); @@ -62,40 +51,51 @@ bool IterativeSolver< Real, Index, SolverMonitor >::setup( const Config::Paramet } template< typename Real, typename Index, typename SolverMonitor > -void IterativeSolver< Real, Index, SolverMonitor >::setMaxIterations( const Index& maxIterations ) +void +IterativeSolver< Real, Index, SolverMonitor >:: +setMaxIterations( const Index& maxIterations ) { this->maxIterations = maxIterations; } template< typename Real, typename Index, typename SolverMonitor > -const Index& IterativeSolver< Real, Index, SolverMonitor >::getMaxIterations() const +const Index& +IterativeSolver< Real, Index, SolverMonitor >:: +getMaxIterations() const { return this->maxIterations; } template< typename Real, typename Index, typename SolverMonitor > -void IterativeSolver< Real, Index, SolverMonitor >::setMinIterations( const Index& minIterations ) +void +IterativeSolver< Real, Index, SolverMonitor >:: +setMinIterations( const Index& minIterations ) { this->minIterations = minIterations; } template< typename Real, typename Index, typename SolverMonitor > -const Index& IterativeSolver< Real, Index, SolverMonitor >::getMinIterations() const +const Index& +IterativeSolver< Real, Index, SolverMonitor >:: +getMinIterations() const { return this->minIterations; } template< typename Real, typename Index, typename SolverMonitor > -void IterativeSolver< Real, Index, SolverMonitor >::resetIterations() +void +IterativeSolver< Real, Index, SolverMonitor >:: +resetIterations() { this->currentIteration = 0; if( this->solverMonitor ) this->solverMonitor->setIterations( 0 ); - } template< typename Real, typename Index, typename SolverMonitor > -bool IterativeSolver< Real, Index, SolverMonitor >::nextIteration() +bool +IterativeSolver< Real, Index, SolverMonitor >:: +nextIteration() { // this->checkNextIteration() must be called before the iteration counter is incremented bool result = this->checkNextIteration(); @@ -108,7 +108,9 @@ bool IterativeSolver< Real, Index, SolverMonitor >::nextIteration() } template< typename Real, typename Index, typename SolverMonitor > -bool IterativeSolver< Real, Index, SolverMonitor >::checkNextIteration() +bool +IterativeSolver< Real, Index, SolverMonitor >:: +checkNextIteration() { this->refreshSolverMonitor(); @@ -116,7 +118,7 @@ bool IterativeSolver< Real, Index, SolverMonitor >::checkNextIteration() this->getIterations() > this->getMaxIterations() || ( this->getResidue() > this->getDivergenceResidue() && this->getIterations() >= this->getMinIterations() ) || ( this->getResidue() < this->getConvergenceResidue() && this->getIterations() >= this->getMinIterations() ) ) - return false; + return false; return true; } @@ -158,32 +160,41 @@ getIterations() const } template< typename Real, typename Index, typename SolverMonitor > -void IterativeSolver< Real, Index, SolverMonitor >::setConvergenceResidue( const Real& convergenceResidue ) +void +IterativeSolver< Real, Index, SolverMonitor >:: +setConvergenceResidue( const Real& convergenceResidue ) { this->convergenceResidue = convergenceResidue; } template< typename Real, typename Index, typename SolverMonitor > -const Real& IterativeSolver< Real, Index, SolverMonitor >::getConvergenceResidue() const +const Real& +IterativeSolver< Real, Index, SolverMonitor >:: +getConvergenceResidue() const { return this->convergenceResidue; } template< typename Real, typename Index, typename SolverMonitor > -void IterativeSolver< Real, Index, SolverMonitor >::setDivergenceResidue( const Real& divergenceResidue ) +void +IterativeSolver< Real, Index, SolverMonitor >:: +setDivergenceResidue( const Real& divergenceResidue ) { this->divergenceResidue = divergenceResidue; } template< typename Real, typename Index, typename SolverMonitor > -const Real& IterativeSolver< Real, Index, SolverMonitor >::getDivergenceResidue() const +const Real& +IterativeSolver< Real, Index, SolverMonitor >:: +getDivergenceResidue() const { return this->divergenceResidue; } - template< typename Real, typename Index, typename SolverMonitor > -void IterativeSolver< Real, Index, SolverMonitor >::setResidue( const Real& residue ) +void +IterativeSolver< Real, Index, SolverMonitor >:: +setResidue( const Real& residue ) { this->currentResidue = residue; if( this->solverMonitor ) @@ -191,26 +202,34 @@ void IterativeSolver< Real, Index, SolverMonitor >::setResidue( const Real& resi } template< typename Real, typename Index, typename SolverMonitor > -const Real& IterativeSolver< Real, Index, SolverMonitor >::getResidue() const +const Real& +IterativeSolver< Real, Index, SolverMonitor >:: +getResidue() const { return this->currentResidue; } // TODO: setting refresh rate should be done in SolverStarter::setup (it's not a parameter of the IterativeSolver) template< typename Real, typename Index, typename SolverMonitor > -void IterativeSolver< Real, Index, SolverMonitor >::setRefreshRate( const Index& refreshRate ) +void +IterativeSolver< Real, Index, SolverMonitor >:: +setRefreshRate( const Index& refreshRate ) { this->refreshRate = refreshRate; } template< typename Real, typename Index, typename SolverMonitor > -void IterativeSolver< Real, Index, SolverMonitor >::setSolverMonitor( SolverMonitorType& solverMonitor ) +void +IterativeSolver< Real, Index, SolverMonitor >:: +setSolverMonitor( SolverMonitorType& solverMonitor ) { this->solverMonitor = &solverMonitor; } template< typename Real, typename Index, typename SolverMonitor > -void IterativeSolver< Real, Index, SolverMonitor >::refreshSolverMonitor( bool force ) +void +IterativeSolver< Real, Index, SolverMonitor >:: +refreshSolverMonitor( bool force ) { if( this->solverMonitor ) { diff --git a/src/TNL/Solvers/ODE/ExplicitSolver_impl.h b/src/TNL/Solvers/ODE/ExplicitSolver_impl.h index 46e69381c..eef1a30ac 100644 --- a/src/TNL/Solvers/ODE/ExplicitSolver_impl.h +++ b/src/TNL/Solvers/ODE/ExplicitSolver_impl.h @@ -10,6 +10,8 @@ #pragma once +#include + namespace TNL { namespace Solvers { namespace ODE { -- GitLab From f6c17b65c625cd0345639b9020afc7be132c48c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Sat, 11 Jan 2020 12:34:48 +0100 Subject: [PATCH 3/9] Added option to save the residual history in IterativeSolver --- src/TNL/Solvers/IterativeSolver.h | 4 ++++ src/TNL/Solvers/IterativeSolver_impl.h | 20 +++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/TNL/Solvers/IterativeSolver.h b/src/TNL/Solvers/IterativeSolver.h index c7c903cfc..75487fa01 100644 --- a/src/TNL/Solvers/IterativeSolver.h +++ b/src/TNL/Solvers/IterativeSolver.h @@ -88,6 +88,10 @@ protected: SolverMonitor* solverMonitor = nullptr; Index refreshRate = 1; + + String residualHistoryFileName = ""; + + std::ofstream residualHistoryFile; }; } // namespace Solvers diff --git a/src/TNL/Solvers/IterativeSolver_impl.h b/src/TNL/Solvers/IterativeSolver_impl.h index b673a5c17..8f28e8847 100644 --- a/src/TNL/Solvers/IterativeSolver_impl.h +++ b/src/TNL/Solvers/IterativeSolver_impl.h @@ -33,6 +33,8 @@ configSetup( Config::ConfigDescription& config, config.addEntry< double >( prefix + "divergence-residue", "Divergence occurs when the residue exceeds given limit.", std::numeric_limits< float >::max() ); // TODO: setting refresh rate should be done in SolverStarter::setup (it's not a parameter of the IterativeSolver) config.addEntry< int > ( prefix + "refresh-rate", "Number of iterations between solver monitor refreshes.", 1 ); + + config.addEntry< String >( prefix + "residual-history-file", "Path to the file where the residual history will be saved.", "" ); } template< typename Real, typename Index, typename SolverMonitor > @@ -41,12 +43,15 @@ IterativeSolver< Real, Index, SolverMonitor >:: setup( const Config::ParameterContainer& parameters, const String& prefix ) { - this->setMaxIterations( parameters.getParameter< int >( "max-iterations" ) ); - this->setMinIterations( parameters.getParameter< int >( "min-iterations" ) ); - this->setConvergenceResidue( parameters.getParameter< double >( "convergence-residue" ) ); - this->setDivergenceResidue( parameters.getParameter< double >( "divergence-residue" ) ); + this->setMaxIterations( parameters.getParameter< int >( prefix + "max-iterations" ) ); + this->setMinIterations( parameters.getParameter< int >( prefix + "min-iterations" ) ); + this->setConvergenceResidue( parameters.getParameter< double >( prefix + "convergence-residue" ) ); + this->setDivergenceResidue( parameters.getParameter< double >( prefix + "divergence-residue" ) ); // TODO: setting refresh rate should be done in SolverStarter::setup (it's not a parameter of the IterativeSolver) - this->setRefreshRate( parameters.getParameter< int >( "refresh-rate" ) ); + this->setRefreshRate( parameters.getParameter< int >( prefix + "refresh-rate" ) ); + this->residualHistoryFileName = parameters.getParameter< String >( prefix + "residual-history-file" ); + if( this->residualHistoryFileName ) + this->residualHistoryFile.open( this->residualHistoryFileName.getString() ); return true; } @@ -199,6 +204,11 @@ setResidue( const Real& residue ) this->currentResidue = residue; if( this->solverMonitor ) this->solverMonitor->setResidue( this->getResidue() ); + if( this->residualHistoryFile ) { + if( this->getIterations() == 0 ) + this->residualHistoryFile << "\n"; + this->residualHistoryFile << this->getIterations() << "\t" << std::scientific << residue << std::endl; + } } template< typename Real, typename Index, typename SolverMonitor > -- GitLab From d23a9fa51477af858b683f109906275e20c8eaae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Sat, 11 Jan 2020 22:52:37 +0100 Subject: [PATCH 4/9] Update GMRES variants in LinearSolvers benchmark --- .../tnl-benchmark-linear-solvers.h | 90 ++++++++----------- 1 file changed, 39 insertions(+), 51 deletions(-) diff --git a/src/Benchmarks/LinearSolvers/tnl-benchmark-linear-solvers.h b/src/Benchmarks/LinearSolvers/tnl-benchmark-linear-solvers.h index 0701b647a..6db341064 100644 --- a/src/Benchmarks/LinearSolvers/tnl-benchmark-linear-solvers.h +++ b/src/Benchmarks/LinearSolvers/tnl-benchmark-linear-solvers.h @@ -68,12 +68,19 @@ using CommunicatorType = Communicators::NoDistrCommunicator; static const std::set< std::string > valid_solvers = { "gmres", - "cwygmres", "tfqmr", "bicgstab", "bicgstab-ell", }; +static const std::set< std::string > valid_gmres_variants = { + "CGS", + "CGSR", + "MGS", + "MGSR", + "CWY", +}; + static const std::set< std::string > valid_preconditioners = { "jacobi", "ilu0", @@ -85,19 +92,19 @@ parse_comma_list( const Config::ParameterContainer& parameters, const char* parameter, const std::set< std::string >& options ) { - const String solvers = parameters.getParameter< String >( parameter ); + const String param = parameters.getParameter< String >( parameter ); - if( solvers == "all" ) + if( param == "all" ) return options; - std::stringstream ss( solvers.getString() ); + std::stringstream ss( param.getString() ); std::string s; std::set< std::string > set; while( std::getline( ss, s, ',' ) ) { if( ! options.count( s ) ) throw std::logic_error( std::string("Invalid value in the comma-separated list for the parameter '") - + parameter + "': '" + s + "'. The list contains: '" + solvers.getString() + "'." ); + + parameter + "': '" + s + "'. The list contains: '" + param.getString() + "'." ); set.insert( s ); @@ -138,6 +145,7 @@ benchmarkIterativeSolvers( Benchmark& benchmark, const int ell_max = 2; const std::set< std::string > solvers = parse_comma_list( parameters, "solvers", valid_solvers ); + const std::set< std::string > gmresVariants = parse_comma_list( parameters, "gmres-variants", valid_gmres_variants ); const std::set< std::string > preconditioners = parse_comma_list( parameters, "preconditioners", valid_preconditioners ); const bool with_preconditioner_update = parameters.getParameter< bool >( "with-preconditioner-update" ); @@ -151,21 +159,14 @@ benchmarkIterativeSolvers( Benchmark& benchmark, } if( solvers.count( "gmres" ) ) { - benchmark.setOperation("GMRES (Jacobi)"); - parameters.template setParameter< String >( "gmres-variant", "MGSR" ); - benchmarkSolver< GMRES, Diagonal >( benchmark, parameters, matrixPointer, x0, b ); - #ifdef HAVE_CUDA - benchmarkSolver< GMRES, Diagonal >( benchmark, parameters, cudaMatrixPointer, cuda_x0, cuda_b ); - #endif - } - - if( solvers.count( "cwygmres" ) ) { - benchmark.setOperation("CWYGMRES (Jacobi)"); - parameters.template setParameter< String >( "gmres-variant", "CWY" ); - benchmarkSolver< GMRES, Diagonal >( benchmark, parameters, matrixPointer, x0, b ); - #ifdef HAVE_CUDA - benchmarkSolver< GMRES, Diagonal >( benchmark, parameters, cudaMatrixPointer, cuda_x0, cuda_b ); - #endif + for( auto variant : gmresVariants ) { + benchmark.setOperation(variant + "-GMRES (Jacobi)"); + parameters.template setParameter< String >( "gmres-variant", variant.c_str() ); + benchmarkSolver< GMRES, Diagonal >( benchmark, parameters, matrixPointer, x0, b ); + #ifdef HAVE_CUDA + benchmarkSolver< GMRES, Diagonal >( benchmark, parameters, cudaMatrixPointer, cuda_x0, cuda_b ); + #endif + } } if( solvers.count( "tfqmr" ) ) { @@ -208,21 +209,14 @@ benchmarkIterativeSolvers( Benchmark& benchmark, } if( solvers.count( "gmres" ) ) { - benchmark.setOperation("GMRES (ILU0)"); - parameters.template setParameter< String >( "gmres-variant", "MGSR" ); - benchmarkSolver< GMRES, ILU0 >( benchmark, parameters, matrixPointer, x0, b ); - #ifdef HAVE_CUDA - benchmarkSolver< GMRES, ILU0 >( benchmark, parameters, cudaMatrixPointer, cuda_x0, cuda_b ); - #endif - } - - if( solvers.count( "cwygmres" ) ) { - benchmark.setOperation("CWYGMRES (ILU0)"); - parameters.template setParameter< String >( "gmres-variant", "CWY" ); - benchmarkSolver< GMRES, ILU0 >( benchmark, parameters, matrixPointer, x0, b ); - #ifdef HAVE_CUDA - benchmarkSolver< GMRES, ILU0 >( benchmark, parameters, cudaMatrixPointer, cuda_x0, cuda_b ); - #endif + for( auto variant : gmresVariants ) { + benchmark.setOperation(variant + "-GMRES (ILU0)"); + parameters.template setParameter< String >( "gmres-variant", variant.c_str() ); + benchmarkSolver< GMRES, ILU0 >( benchmark, parameters, matrixPointer, x0, b ); + #ifdef HAVE_CUDA + benchmarkSolver< GMRES, ILU0 >( benchmark, parameters, cudaMatrixPointer, cuda_x0, cuda_b ); + #endif + } } if( solvers.count( "tfqmr" ) ) { @@ -264,21 +258,14 @@ benchmarkIterativeSolvers( Benchmark& benchmark, } if( solvers.count( "gmres" ) ) { - benchmark.setOperation("GMRES (ILUT)"); - parameters.template setParameter< String >( "gmres-variant", "MGSR" ); - benchmarkSolver< GMRES, ILUT >( benchmark, parameters, matrixPointer, x0, b ); - #ifdef HAVE_CUDA - benchmarkSolver< GMRES, ILUT >( benchmark, parameters, cudaMatrixPointer, cuda_x0, cuda_b ); - #endif - } - - if( solvers.count( "cwygmres" ) ) { - benchmark.setOperation("CWYGMRES (ILUT)"); - parameters.template setParameter< String >( "gmres-variant", "CWY" ); - benchmarkSolver< GMRES, ILUT >( benchmark, parameters, matrixPointer, x0, b ); - #ifdef HAVE_CUDA - benchmarkSolver< GMRES, ILUT >( benchmark, parameters, cudaMatrixPointer, cuda_x0, cuda_b ); - #endif + for( auto variant : gmresVariants ) { + benchmark.setOperation(variant + "-GMRES (ILUT)"); + parameters.template setParameter< String >( "gmres-variant", variant.c_str() ); + benchmarkSolver< GMRES, ILUT >( benchmark, parameters, matrixPointer, x0, b ); + #ifdef HAVE_CUDA + benchmarkSolver< GMRES, ILUT >( benchmark, parameters, cudaMatrixPointer, cuda_x0, cuda_b ); + #endif + } } if( solvers.count( "tfqmr" ) ) { @@ -495,7 +482,8 @@ configSetup( Config::ConfigDescription& config ) config.addEntry< int >( "verbose", "Verbose mode.", 1 ); config.addEntry< bool >( "reorder-dofs", "Reorder matrix entries corresponding to the same DOF together.", false ); config.addEntry< bool >( "with-direct", "Includes the 3rd party direct solvers in the benchmark.", false ); - config.addEntry< String >( "solvers", "Comma-separated list of solvers to run benchmarks for. Options: gmres, cwygmres, tfqmr, bicgstab, bicgstab-ell.", "all" ); + config.addEntry< String >( "solvers", "Comma-separated list of solvers to run benchmarks for. Options: gmres, tfqmr, bicgstab, bicgstab-ell.", "all" ); + config.addEntry< String >( "gmres-variants", "Comma-separated list of GMRES variants to run benchmarks for. Options: CGS, CGSR, MGS, MGSR, CWY.", "all" ); config.addEntry< String >( "preconditioners", "Comma-separated list of preconditioners to run benchmarks for. Options: jacobi, ilu0, ilut.", "all" ); config.addEntry< bool >( "with-preconditioner-update", "Run benchmark for the preconditioner update.", true ); config.addEntry< String >( "devices", "Run benchmarks on these devices.", "all" ); -- GitLab From 439e69cdf02436fc0fd3a41f57bc19f57aa29a70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Tue, 14 Jan 2020 21:42:24 +0100 Subject: [PATCH 5/9] Fixed line splitting for the MTX format in MatrixReader --- src/TNL/Matrices/MatrixReader_impl.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/TNL/Matrices/MatrixReader_impl.h b/src/TNL/Matrices/MatrixReader_impl.h index dd6ddc072..0415d5f8d 100644 --- a/src/TNL/Matrices/MatrixReader_impl.h +++ b/src/TNL/Matrices/MatrixReader_impl.h @@ -165,7 +165,7 @@ template< typename Matrix > bool MatrixReader< Matrix >::checkMtxHeader( const String& header, bool& symmetric ) { - std::vector< String > parsedLine = header.split(); + std::vector< String > parsedLine = header.split( ' ', String::SplitSkip::SkipEmpty ); if( (int) parsedLine.size() < 5 ) return false; if( parsedLine[ 0 ] != "%%MatrixMarket" ) @@ -231,7 +231,7 @@ bool MatrixReader< Matrix >::readMtxHeader( std::istream& file, return false; } - parsedLine = line.split(); + parsedLine = line.split( ' ', String::SplitSkip::SkipEmpty ); if( (int) parsedLine.size() != 3 ) { std::cerr << "Wrong number of parameters in the matrix header." << std::endl; @@ -389,7 +389,7 @@ bool MatrixReader< Matrix >::parseMtxLineWithElement( const String& line, IndexType& column, RealType& value ) { - std::vector< String > parsedLine = line.split(); + std::vector< String > parsedLine = line.split( ' ', String::SplitSkip::SkipEmpty ); if( (int) parsedLine.size() != 3 ) { std::cerr << "Wrong number of parameters in the matrix row at line:" << line << std::endl; -- GitLab From c11b58057a6b26a831b4bf9fcdeee2557e3d17f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Tue, 14 Jan 2020 21:43:59 +0100 Subject: [PATCH 6/9] Extended the LinerSolvers benchmark for MTX matrices --- src/Benchmarks/LinearSolvers/benchmarks.h | 4 +- .../tnl-benchmark-linear-solvers.h | 81 +++++++++++++++++-- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/src/Benchmarks/LinearSolvers/benchmarks.h b/src/Benchmarks/LinearSolvers/benchmarks.h index 577907c78..7b22cdfc1 100644 --- a/src/Benchmarks/LinearSolvers/benchmarks.h +++ b/src/Benchmarks/LinearSolvers/benchmarks.h @@ -145,7 +145,7 @@ benchmarkSolver( Benchmark& benchmark, virtual HeaderElements getTableHeader() const override { - return HeaderElements({"time", "speedup", "converged", "iterations", "residue_precond", "residue_true"}); + return HeaderElements({"time", "stddev", "stddev/time", "speedup", "converged", "iterations", "residue_precond", "residue_true"}); } virtual RowElements getRowElements() const override @@ -160,7 +160,7 @@ benchmarkSolver( Benchmark& benchmark, r = b - r; const double residue_true = lpNorm( r, 2.0 ) / lpNorm( b, 2.0 ); - return RowElements({ time, speedup, (double) converged, (double) iterations, + return RowElements({ time, stddev, stddev/time, speedup, (double) converged, (double) iterations, residue_precond, residue_true }); } }; diff --git a/src/Benchmarks/LinearSolvers/tnl-benchmark-linear-solvers.h b/src/Benchmarks/LinearSolvers/tnl-benchmark-linear-solvers.h index 6db341064..4aabf39cd 100644 --- a/src/Benchmarks/LinearSolvers/tnl-benchmark-linear-solvers.h +++ b/src/Benchmarks/LinearSolvers/tnl-benchmark-linear-solvers.h @@ -15,6 +15,7 @@ #include #include #include +#include #ifndef NDEBUG #include @@ -30,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -115,6 +117,42 @@ parse_comma_list( const Config::ParameterContainer& parameters, return set; } +// TODO: implement this in TNL::String +bool ends_with( const std::string& value, const std::string& ending ) +{ + if (ending.size() > value.size()) + return false; + return std::equal(ending.rbegin(), ending.rend(), value.rbegin()); +} + +// initialize all vector entries with a unioformly distributed random value from the interval [a, b] +template< typename Vector > +void set_random_vector( Vector& v, typename Vector::RealType a, typename Vector::RealType b ) +{ + using RealType = typename Vector::RealType; + using IndexType = typename Vector::IndexType; + // random device will be used to obtain a seed for the random number engine + std::random_device rd; + // initialize the standard mersenne_twister_engine with rd() as the seed + std::mt19937 gen(rd()); + // create uniform distribution + std::uniform_real_distribution< RealType > dis(a, b); + + // create host vector + typename Vector::template Self< RealType, Devices::Host > host_v; + host_v.setSize( v.getSize() ); + + // initialize the host vector + auto kernel = [&] ( IndexType i ) + { + host_v[ i ] = dis(gen); + }; + Algorithms::ParallelFor< Devices::Host >::exec( (IndexType) 0, host_v.getSize(), kernel ); + + // copy the data to the device vector + v = host_v; +} + template< typename Matrix, typename Vector > void benchmarkIterativeSolvers( Benchmark& benchmark, @@ -317,11 +355,42 @@ struct LinearSolversBenchmark // const Config::ParameterContainer& parameters ) Config::ParameterContainer& parameters ) { + const String file_matrix = parameters.getParameter< String >( "input-matrix" ); + const String file_dof = parameters.getParameter< String >( "input-dof" ); + const String file_rhs = parameters.getParameter< String >( "input-rhs" ); + SharedPointer< MatrixType > matrixPointer; VectorType x0, b; - matrixPointer->load( parameters.getParameter< String >( "input-matrix" ) ); - File( parameters.getParameter< String >( "input-dof" ), std::ios_base::in ) >> x0; - File( parameters.getParameter< String >( "input-rhs" ), std::ios_base::in ) >> b; + + // load the matrix + if( ends_with( file_matrix, ".mtx" ) ) { + Matrices::MatrixReader< MatrixType > reader; + if( ! reader.readMtxFile( file_matrix, *matrixPointer ) ) + return false; + } + else { + matrixPointer->load( file_matrix ); + } + + // load the vectors + if( file_dof && file_rhs ) { + File( file_dof, std::ios_base::in ) >> x0; + File( file_rhs, std::ios_base::in ) >> b; + } + else { + // set x0 := 0 + x0.setSize( matrixPointer->getColumns() ); + x0 = 0; + + // generate random vector x + VectorType x; + x.setSize( matrixPointer->getColumns() ); + set_random_vector( x, 1e2, 1e3 ); + + // set b := A*x + b.setSize( matrixPointer->getRows() ); + matrixPointer->vectorProduct( x, b ); + } typename MatrixType::CompressedRowLengthsVector rowLengths; matrixPointer->getCompressedRowLengths( rowLengths ); @@ -475,9 +544,9 @@ configSetup( Config::ConfigDescription& config ) config.addEntryEnum( "append" ); config.addEntryEnum( "overwrite" ); config.addEntry< int >( "loops", "Number of repetitions of the benchmark.", 10 ); - config.addRequiredEntry< String >( "input-matrix", "File name of the input matrix (in binary TNL format)." ); - config.addRequiredEntry< String >( "input-dof", "File name of the input DOF vector (in binary TNL format)." ); - config.addRequiredEntry< String >( "input-rhs", "File name of the input right-hand-side vector (in binary TNL format)." ); + config.addRequiredEntry< String >( "input-matrix", "File name of the input matrix (in binary TNL format or textual MTX format)." ); + config.addEntry< String >( "input-dof", "File name of the input DOF vector (in binary TNL format).", "" ); + config.addEntry< String >( "input-rhs", "File name of the input right-hand-side vector (in binary TNL format).", "" ); config.addEntry< String >( "name", "Name of the matrix in the benchmark.", "" ); config.addEntry< int >( "verbose", "Verbose mode.", 1 ); config.addEntry< bool >( "reorder-dofs", "Reorder matrix entries corresponding to the same DOF together.", false ); -- GitLab From 64cf828fdcc5d647f59ee9bc1f4afeef6f3c6691 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Wed, 15 Jan 2020 17:47:13 +0100 Subject: [PATCH 7/9] Fixed linking to a BLAS library on Centos systems --- src/Benchmarks/BLAS/CMakeLists.txt | 24 ++++++++++++++++++------ src/Benchmarks/BLAS/blasWrappers.h | 6 ++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/Benchmarks/BLAS/CMakeLists.txt b/src/Benchmarks/BLAS/CMakeLists.txt index 3f040abc8..81d837533 100644 --- a/src/Benchmarks/BLAS/CMakeLists.txt +++ b/src/Benchmarks/BLAS/CMakeLists.txt @@ -5,16 +5,28 @@ else() add_executable( tnl-benchmark-blas tnl-benchmark-blas.cpp ) endif() -find_library( CBLAS_LIBRARY NAMES cblas - PATHS /usr/lib - /usr/lib64 - /usr/lib/x86_64-linux-gnu - /usr/local/lib - /usr/local/lib64 ) +find_library( CBLAS_LIBRARY NAMES cblas ) + +# fallback for Centos 7.5 - libcblas.so does not exist, link to libtatlas.so or libsatlas.so +# https://forums.centos.org/viewtopic.php?t=48543 +find_library( TATLAS_LIBRARY NAMES tatlas + PATH_SUFFIXES atlas ) +find_library( SATLAS_LIBRARY NAMES satlas + PATH_SUFFIXES atlas ) + if( CBLAS_LIBRARY ) target_compile_definitions( tnl-benchmark-blas PUBLIC "-DHAVE_BLAS" ) target_link_libraries( tnl-benchmark-blas ${CBLAS_LIBRARY} ) +elseif( TATLAS_LIBRARY ) + target_compile_definitions( tnl-benchmark-blas PUBLIC "-DHAVE_BLAS" ) + target_link_libraries( tnl-benchmark-blas ${TATLAS_LIBRARY} ) +elseif( SATLAS_LIBRARY ) + target_compile_definitions( tnl-benchmark-blas PUBLIC "-DHAVE_BLAS" ) + target_link_libraries( tnl-benchmark-blas ${SATLAS_LIBRARY} ) else() + # FIXME: We require the CBLAS interface, but CMake's FindBLAS cannot detect that, + # so this fails unless the BLAS implementation includes it in the same + # shared library file as the Fortran implementation (e.g. OpenBLAS does that). find_package( BLAS ) if( BLAS_FOUND ) target_compile_definitions( tnl-benchmark-blas PUBLIC "-DHAVE_BLAS" ) diff --git a/src/Benchmarks/BLAS/blasWrappers.h b/src/Benchmarks/BLAS/blasWrappers.h index d1e0edff1..ce30060bf 100644 --- a/src/Benchmarks/BLAS/blasWrappers.h +++ b/src/Benchmarks/BLAS/blasWrappers.h @@ -2,7 +2,13 @@ #ifdef HAVE_BLAS +// HOTFIX: cblas.h from the atlas-devel package (version 3.10.1-12.el7) on CentOS 7 +// does not declare the functions as `extern "C"`, which breaks name mangling. +// Note that nested `extern "C"` is valid and correct: +// https://stackoverflow.com/questions/48099828/what-happens-if-you-nest-extern-c +extern "C" { #include +} inline int blasIgamax( int n, const float *x, int incx ) { -- GitLab From faf4328fcc54aab26186a9e8908b0a8a6854d765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Wed, 15 Jan 2020 22:15:59 +0100 Subject: [PATCH 8/9] CMake: skip -march=native and -mtune=native from the pytnl target Fixes #59 --- CMakeLists.txt | 11 +++++++++-- src/Python/pytnl/tnl/CMakeLists.txt | 10 ++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ef868cc7..eb32ae2f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -81,10 +81,10 @@ set( CMAKE_CXX_STANDARD 14 ) set( CMAKE_CXX_STANDARD_REQUIRED ON ) set( CMAKE_CXX_EXTENSIONS OFF ) -# set Debug/Release options +# set default build options set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wall -Wno-unused-local-typedefs -Wno-unused-variable -Wno-unknown-pragmas" ) set( CMAKE_CXX_FLAGS_DEBUG "-g" ) -set( CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native -mtune=native -DNDEBUG" ) +set( CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG" ) # pass -rdynamic only in Debug mode set( CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "" ) set( CMAKE_SHARED_LIBRARY_LINK_C_FLAGS_DEBUG "-rdynamic" ) @@ -96,6 +96,13 @@ set( CMAKE_SHARED_LINKER_FLAGS "" ) set( CMAKE_SHARED_LINKER_FLAGS_DEBUG "-rdynamic" ) set( CMAKE_SHARED_LINKER_FLAGS_RELEASE "" ) +# set additional Debug/Release options using generator expressions +# (that way we can exclude some options for specific targets, see https://stackoverflow.com/a/59734798 for details) +add_compile_options( + # GOTCHA: CMake uses semicolons as list item separator, spaces would lead to a single argument inside double-quotes on the command line + "$<$:-march=native;-mtune=native>" +) + # disable GCC's infamous "maybe-uninitialized" warning (it produces mostly false positives) if( CMAKE_CXX_COMPILER_ID STREQUAL "GNU" ) set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-maybe-uninitialized" ) diff --git a/src/Python/pytnl/tnl/CMakeLists.txt b/src/Python/pytnl/tnl/CMakeLists.txt index b6c4182b8..de405e5e5 100644 --- a/src/Python/pytnl/tnl/CMakeLists.txt +++ b/src/Python/pytnl/tnl/CMakeLists.txt @@ -16,6 +16,16 @@ pybind11_add_module( pytnl ${sources} ) # rename the shared library to tnl.cpython-XXm-x86_64-linux-gnu.so set_target_properties( pytnl PROPERTIES LIBRARY_OUTPUT_NAME tnl ) +# Skip -march=native -mtune=native for pytnl - optimizing python bindings for +# a specific architecture is not very useful and prevents using Python tools on +# hybrid clusters. +get_target_property( pytnl_COMPILE_OPTIONS pytnl COMPILE_OPTIONS ) +if( pytnl_COMPILE_OPTIONS ) + string( REPLACE "-march=native" "" pytnl_COMPILE_OPTIONS "${pytnl_COMPILE_OPTIONS}" ) + string( REPLACE "-mtune=native" "" pytnl_COMPILE_OPTIONS "${pytnl_COMPILE_OPTIONS}" ) + set_target_properties( pytnl PROPERTIES COMPILE_OPTIONS "${pytnl_COMPILE_OPTIONS}" ) +endif() + # We have bindings for unsafe objects (e.g. Array::operator[]) where assertion # is the only safeguard, so we need to translate the TNL::AssertionError to # Python's AssertionError. -- GitLab From b36d0a5c3d2ec240ee766663c8ecabc44514c494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Thu, 16 Jan 2020 13:00:04 +0100 Subject: [PATCH 9/9] CMakeLists.txt: fixed WITH_EXAMPLES and WITH_DOC options --- CMakeLists.txt | 6 +++--- build | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eb32ae2f5..68252ba6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,11 +25,11 @@ option(WITH_GMP "Build with GMP support" OFF) option(WITH_TESTS "Build tests" ON) option(WITH_PROFILING "Enable code profiling compiler flags" OFF ) option(WITH_COVERAGE "Enable code coverage reports from unit tests" OFF) -option(WITH_EXAMPLES "Compile the 'examples' directory" ON) +option(WITH_EXAMPLES "Compile the 'src/Examples' directory" ON) option(WITH_TOOLS "Compile the 'src/Tools' directory" ON) option(WITH_BENCHMARKS "Compile the 'src/Benchmarks' directory" ON) option(WITH_PYTHON "Compile the Python bindings" ON) -option(WITH_DOC "Generate documentation" ON) +option(WITH_DOC "Build examples included in the documentation" ON) # install paths relative to the cmake's prefix set( TNL_TARGET_INCLUDE_DIRECTORY "include/TNL" ) @@ -307,7 +307,7 @@ add_subdirectory( src ) add_subdirectory( share ) # Add subdirectories for examples included in the documentation -if( ${WITH_DOC} OR ${WITH_EXAMPLES} ) +if( ${WITH_DOC} ) set( TNL_DOCUMENTATION_OUTPUT_SNIPPETS_PATH "${CMAKE_SOURCE_DIR}/Documentation/output_snippets" ) file(MAKE_DIRECTORY ${TNL_DOCUMENTATION_OUTPUT_SNIPPETS_PATH}) add_subdirectory( Documentation/Examples ) diff --git a/build b/build index 68966f0f4..76169a7c4 100755 --- a/build +++ b/build @@ -87,10 +87,11 @@ if [[ ${HELP} == "yes" ]]; then echo " --with-profiling=yes/no Enables code profiling compiler falgs. 'no' by default." echo " --with-coverage=yes/no Enables code coverage reports for unit tests. 'no' by default (lcov is required)." echo " --with-doc=yes/no Build documentation. 'yes' by default." - echo " --with-examples=yes/no Compile the 'examples' directory. 'yes' by default." + echo " --with-examples=yes/no Compile the 'src/Examples' directory. 'yes' by default." echo " --with-tools=yes/no Compile the 'src/Tools' directory. 'yes' by default." echo " --with-python=yes/no Compile the Python bindings. 'yes' by default." echo " --with-benchmarks=yes/no Compile the 'src/Benchmarks' directory. 'yes' by default." + echo " --with-doc=yes/no Generate the documentation. 'yes' by default." echo " --cmake=CMAKE Path to cmake. 'cmake' by default." echo " --verbose It enables verbose build." echo " --root-dir=PATH Path to the TNL source code root dir." -- GitLab