Commit 8ddb9e80 authored by Jakub Klinkovský's avatar Jakub Klinkovský
Browse files

Merge branch 'JK/MPI' into 'develop' 

MPI refactoring

See merge request !101
parents ae1def7a a8d7afcf

src/Benchmarks/DistSpMV/tnl-benchmark-distributed-spmv.h

+8 −8
Original line number Diff line number Diff line
@@ -108,7 +108,7 @@ benchmarkDistributedSpmv( Benchmark& benchmark, 
   // benchmark function

   auto compute = [&]() {

      matrix.vectorProduct( x, y );

      TNL::MPI::Barrier( matrix.getCommunicationGroup() );

      TNL::MPI::Barrier( matrix.getCommunicator() );

   };



   benchmark.time< typename Matrix::DeviceType >( reset, performer, compute );
@@ -223,13 +223,13 @@ struct SpmvBenchmark 
                   VectorType& vector )

   {

      // set up the distributed matrix

      const auto group = TNL::MPI::AllGroup();

      const auto localRange = Partitioner::splitRange( matrix.getRows(), group );

      DistributedMatrix distributedMatrix( localRange, matrix.getRows(), matrix.getColumns(), group );

      DistributedVector distributedVector( localRange, 0, matrix.getRows(), group );

      const auto communicator = MPI_COMM_WORLD;

      const auto localRange = Partitioner::splitRange( matrix.getRows(), communicator );

      DistributedMatrix distributedMatrix( localRange, matrix.getRows(), matrix.getColumns(), communicator );

      DistributedVector distributedVector( localRange, 0, matrix.getRows(), communicator );



      // copy the row lengths from the global matrix to the distributed matrix

      DistributedRowLengths distributedRowLengths( localRange, 0, matrix.getRows(), group );

      DistributedRowLengths distributedRowLengths( localRange, 0, matrix.getRows(), communicator );

      for( IndexType i = 0; i < distributedMatrix.getLocalMatrix().getRows(); i++ ) {

         const auto gi = distributedMatrix.getLocalRowRange().getGlobalIndex( i );

         distributedRowLengths[ gi ] = matrix.getRowCapacity( gi );
@@ -265,8 +265,8 @@ struct SpmvBenchmark 
      DistributedVector distributedY;

      distributedY.setLike( distributedVector );

      distributedMatrix.vectorProduct( distributedVector, distributedY );

      const int rank = TNL::MPI::GetRank( distributedMatrix.getCommunicationGroup() );

      const int nproc = TNL::MPI::GetSize( distributedMatrix.getCommunicationGroup() );

      const int rank = TNL::MPI::GetRank( distributedMatrix.getCommunicator() );

      const int nproc = TNL::MPI::GetSize( distributedMatrix.getCommunicator() );

      typename VectorType::ViewType subY( &y[ Partitioner::getOffset( matrix.getRows(), rank, nproc ) ],

                                          Partitioner::getSizeForRank( matrix.getRows(), rank, nproc ) );

      TNL_ASSERT_EQ( distributedY.getLocalView(), subY, "WRONG RESULT !!!" );

src/Benchmarks/HeatEquation/HeatEquationBenchmarkProblem.h

+10 −14
Original line number Diff line number Diff line
@@ -12,11 +12,9 @@ using namespace TNL::Problems; 
template< typename Mesh,

          typename BoundaryCondition,

          typename RightHandSide,

          typename DifferentialOperator,

          typename Communicator >

          typename DifferentialOperator >

class HeatEquationBenchmarkProblem:

   public PDEProblem< Mesh,

                      Communicator,

                      typename DifferentialOperator::RealType,

                      typename Mesh::DeviceType,

                      typename DifferentialOperator::IndexType >
@@ -28,13 +26,11 @@ class HeatEquationBenchmarkProblem: 
      typedef typename DifferentialOperator::IndexType IndexType;

      typedef Functions::MeshFunctionView< Mesh > MeshFunctionViewType;

      typedef Pointers::SharedPointer< MeshFunctionViewType, DeviceType > MeshFunctionViewPointer;

      typedef PDEProblem< Mesh, Communicator, RealType, DeviceType, IndexType > BaseType;

      typedef PDEProblem< Mesh, RealType, DeviceType, IndexType > BaseType;

      typedef Pointers::SharedPointer< DifferentialOperator > DifferentialOperatorPointer;

      typedef Pointers::SharedPointer< BoundaryCondition > BoundaryConditionPointer;

      typedef Pointers::SharedPointer< RightHandSide, DeviceType > RightHandSidePointer;



      typedef Communicator CommunicatorType;



      using typename BaseType::MeshType;

      using typename BaseType::MeshPointer;

      using typename BaseType::DofVectorPointer;

src/Benchmarks/HeatEquation/HeatEquationBenchmarkProblem_impl.h

+28 −41
Original line number Diff line number Diff line
@@ -19,9 +19,8 @@ 
template< typename Mesh,

          typename BoundaryCondition,

          typename RightHandSide,

          typename DifferentialOperator,

          typename Communicator >

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator, Communicator >::

          typename DifferentialOperator >

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::

HeatEquationBenchmarkProblem()

: cudaMesh( 0 ),

  cudaBoundaryConditions( 0 ),
@@ -33,10 +32,9 @@ HeatEquationBenchmarkProblem() 
template< typename Mesh,

          typename BoundaryCondition,

          typename RightHandSide,

          typename DifferentialOperator,

          typename Communicator >

          typename DifferentialOperator >

String

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator, Communicator >::

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::

getPrologHeader() const

{

   if( this->cudaKernelType == "pure-c" )
@@ -53,10 +51,9 @@ getPrologHeader() const 
template< typename Mesh,

          typename BoundaryCondition,

          typename RightHandSide,

          typename DifferentialOperator,

          typename Communicator >

          typename DifferentialOperator >

void

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator, Communicator >::

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::

writeProlog( Logger& logger, const Config::ParameterContainer& parameters ) const

{

   /****
@@ -68,10 +65,9 @@ writeProlog( Logger& logger, const Config::ParameterContainer& parameters ) cons 
template< typename Mesh,

          typename BoundaryCondition,

          typename RightHandSide,

          typename DifferentialOperator,

          typename Communicator >

          typename DifferentialOperator >

bool

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator, Communicator >::

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::

setup( const Config::ParameterContainer& parameters,

       const String& prefix )

{
@@ -95,10 +91,9 @@ setup( const Config::ParameterContainer& parameters, 
template< typename Mesh,

          typename BoundaryCondition,

          typename RightHandSide,

          typename DifferentialOperator,

          typename Communicator >

typename HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator, Communicator >::IndexType

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator, Communicator >::

          typename DifferentialOperator >

typename HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::IndexType

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::

getDofs() const

{

   /****
@@ -111,10 +106,9 @@ getDofs() const 
template< typename Mesh,

          typename BoundaryCondition,

          typename RightHandSide,

          typename DifferentialOperator,

          typename Communicator >

          typename DifferentialOperator >

void

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator, Communicator >::

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::

bindDofs( DofVectorPointer& dofsPointer )

{

   this->u->bind( this->getMesh(), *dofsPointer );
@@ -123,10 +117,9 @@ bindDofs( DofVectorPointer& dofsPointer ) 
template< typename Mesh,

          typename BoundaryCondition,

          typename RightHandSide,

          typename DifferentialOperator,

          typename Communicator >

          typename DifferentialOperator >

bool

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator, Communicator >::

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::

setInitialCondition( const Config::ParameterContainer& parameters,

                     DofVectorPointer& dofsPointer )

{
@@ -144,11 +137,10 @@ setInitialCondition( const Config::ParameterContainer& parameters, 
template< typename Mesh,

          typename BoundaryCondition,

          typename RightHandSide,

          typename DifferentialOperator,

          typename Communicator >

          typename DifferentialOperator >

   template< typename Matrix >

bool

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator, Communicator >::

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::

setupLinearSystem( Matrix& matrix )

{

   const IndexType dofs = this->getDofs();
@@ -170,10 +162,9 @@ setupLinearSystem( Matrix& matrix ) 
template< typename Mesh,

          typename BoundaryCondition,

          typename RightHandSide,

          typename DifferentialOperator,

          typename Communicator >

          typename DifferentialOperator >

bool

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator, Communicator >::

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::

makeSnapshot( const RealType& time,

              const IndexType& step,

              DofVectorPointer& dofsPointer )
@@ -383,10 +374,9 @@ heatEquationTemplatedCompact( const GridType* grid, 
template< typename Mesh,

          typename BoundaryCondition,

          typename RightHandSide,

          typename DifferentialOperator,

          typename Communicator >

          typename DifferentialOperator >

void

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator, Communicator >::

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::

getExplicitUpdate( const RealType& time,

                   const RealType& tau,

                   DofVectorPointer& uDofs,
@@ -532,7 +522,7 @@ getExplicitUpdate( const RealType& time, 
         this->u->bind( mesh, *uDofs );

         this->fu->bind( mesh, *fuDofs );

         //explicitUpdater.setGPUTransferTimer( this->gpuTransferTimer );

         this->explicitUpdater.template update< typename Mesh::Cell, CommunicatorType >( time, tau, mesh, this->u, this->fu );

         this->explicitUpdater.template update< typename Mesh::Cell >( time, tau, mesh, this->u, this->fu );

      }

      if( this->cudaKernelType == "tunning" )

      {
@@ -636,10 +626,9 @@ getExplicitUpdate( const RealType& time, 
template< typename Mesh,

          typename BoundaryCondition,

          typename RightHandSide,

          typename DifferentialOperator,

          typename Communicator >

          typename DifferentialOperator >

void

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator, Communicator >::

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::

applyBoundaryConditions( const RealType& time,

                            DofVectorPointer& uDofs )

{
@@ -719,11 +708,10 @@ applyBoundaryConditions( const RealType& time, 
template< typename Mesh,

          typename BoundaryCondition,

          typename RightHandSide,

          typename DifferentialOperator,

          typename Communicator >

          typename DifferentialOperator >

   template< typename MatrixPointer >

void

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator, Communicator >::

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::

assemblyLinearSystem( const RealType& time,

                      const RealType& tau,

                      DofVectorPointer& _u,
@@ -750,9 +738,8 @@ assemblyLinearSystem( const RealType& time, 
template< typename Mesh,

          typename BoundaryCondition,

          typename RightHandSide,

          typename DifferentialOperator,

          typename Communicator >

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator, Communicator >::

          typename DifferentialOperator >

HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::

~HeatEquationBenchmarkProblem()

{

   if( this->cudaMesh ) Cuda::freeFromDevice( this->cudaMesh );

src/Benchmarks/HeatEquation/tnl-benchmark-heat-equation.h

+5 −6
Original line number Diff line number Diff line
@@ -49,8 +49,7 @@ template< typename Real, 
          typename Index,

          typename MeshType,

          typename ConfigTag,

          typename SolverStarter,

          typename CommunicatorType >

          typename SolverStarter >

class HeatEquationBenchmarkSetter

{

   public:
@@ -78,12 +77,12 @@ class HeatEquationBenchmarkSetter 
             if( boundaryConditionsType == "dirichlet" )

             {

                typedef Operators::DirichletBoundaryConditions< MeshType, Constant, MeshType::getMeshDimension(), Real, Index > BoundaryConditions;

                typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator, CommunicatorType > Problem;

                typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;

                SolverStarter solverStarter;

                return solverStarter.template run< Problem >( parameters );

             }

             /*typedef Operators::NeumannBoundaryConditions< MeshType, Constant, Real, Index > BoundaryConditions;

             typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator, CommunicatorType > Problem;

             typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;

             SolverStarter solverStarter;

             return solverStarter.template run< Problem >( parameters );*/

          }
@@ -91,12 +90,12 @@ class HeatEquationBenchmarkSetter 
          if( boundaryConditionsType == "dirichlet" )

          {

             typedef Operators::DirichletBoundaryConditions< MeshType, MeshFunction, MeshType::getMeshDimension(), Real, Index > BoundaryConditions;

             typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator, CommunicatorType > Problem;

             typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;

             SolverStarter solverStarter;

             return solverStarter.template run< Problem >( parameters );

          }

          typedef Operators::NeumannBoundaryConditions< MeshType, MeshFunction, Real, Index > BoundaryConditions;

          typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator, CommunicatorType > Problem;

          typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;

          SolverStarter solverStarter;

          return solverStarter.template run< Problem >( parameters );*/

          return false;

src/Benchmarks/LinearSolvers/benchmarks.h

+1 −1
Original line number Diff line number Diff line
@@ -36,7 +36,7 @@ void barrier( const Matrix& matrix ) 
template< typename Matrix >

void barrier( const Matrices::DistributedMatrix< Matrix >& matrix )

{

   TNL::MPI::Barrier( matrix.getCommunicationGroup() );

   TNL::MPI::Barrier( matrix.getCommunicator() );

}



template< typename Device >