Commit fe38b075 authored by Jakub Klinkovský's avatar Jakub Klinkovský
Browse files

MPI refactoring 

- removed wrapper functions: AllGroup, NullGroup, NullRequest
- added MPI_COMM_WORLD and other handles to MPI/DummyDefs.h
- renamed getCommunicationGroup to getCommunicator in all data structures
- improved naming to match the MPI terminology: communicator instead of group
parent 55d3074b

src/Benchmarks/DistSpMV/tnl-benchmark-distributed-spmv.h

+8 −8
Original line number Diff line number Diff line
@@ -108,7 +108,7 @@ benchmarkDistributedSpmv( Benchmark& benchmark, 
   // benchmark function

   auto compute = [&]() {

      matrix.vectorProduct( x, y );

      TNL::MPI::Barrier( matrix.getCommunicationGroup() );

      TNL::MPI::Barrier( matrix.getCommunicator() );

   };



   benchmark.time< typename Matrix::DeviceType >( reset, performer, compute );
@@ -223,13 +223,13 @@ struct SpmvBenchmark 
                   VectorType& vector )

   {

      // set up the distributed matrix

      const auto group = TNL::MPI::AllGroup();

      const auto localRange = Partitioner::splitRange( matrix.getRows(), group );

      DistributedMatrix distributedMatrix( localRange, matrix.getRows(), matrix.getColumns(), group );

      DistributedVector distributedVector( localRange, 0, matrix.getRows(), group );

      const auto communicator = MPI_COMM_WORLD;

      const auto localRange = Partitioner::splitRange( matrix.getRows(), communicator );

      DistributedMatrix distributedMatrix( localRange, matrix.getRows(), matrix.getColumns(), communicator );

      DistributedVector distributedVector( localRange, 0, matrix.getRows(), communicator );



      // copy the row lengths from the global matrix to the distributed matrix

      DistributedRowLengths distributedRowLengths( localRange, 0, matrix.getRows(), group );

      DistributedRowLengths distributedRowLengths( localRange, 0, matrix.getRows(), communicator );

      for( IndexType i = 0; i < distributedMatrix.getLocalMatrix().getRows(); i++ ) {

         const auto gi = distributedMatrix.getLocalRowRange().getGlobalIndex( i );

         distributedRowLengths[ gi ] = matrix.getRowCapacity( gi );
@@ -265,8 +265,8 @@ struct SpmvBenchmark 
      DistributedVector distributedY;

      distributedY.setLike( distributedVector );

      distributedMatrix.vectorProduct( distributedVector, distributedY );

      const int rank = TNL::MPI::GetRank( distributedMatrix.getCommunicationGroup() );

      const int nproc = TNL::MPI::GetSize( distributedMatrix.getCommunicationGroup() );

      const int rank = TNL::MPI::GetRank( distributedMatrix.getCommunicator() );

      const int nproc = TNL::MPI::GetSize( distributedMatrix.getCommunicator() );

      typename VectorType::ViewType subY( &y[ Partitioner::getOffset( matrix.getRows(), rank, nproc ) ],

                                          Partitioner::getSizeForRank( matrix.getRows(), rank, nproc ) );

      TNL_ASSERT_EQ( distributedY.getLocalView(), subY, "WRONG RESULT !!!" );

src/Benchmarks/LinearSolvers/benchmarks.h

+1 −1
Original line number Diff line number Diff line
@@ -36,7 +36,7 @@ void barrier( const Matrix& matrix ) 
template< typename Matrix >

void barrier( const Matrices::DistributedMatrix< Matrix >& matrix )

{

   TNL::MPI::Barrier( matrix.getCommunicationGroup() );

   TNL::MPI::Barrier( matrix.getCommunicator() );

}



template< typename Device >

src/Benchmarks/LinearSolvers/tnl-benchmark-linear-solvers.h

+6 −6
Original line number Diff line number Diff line
@@ -430,14 +430,14 @@ struct LinearSolversBenchmark 
                   const VectorType& b )

   {

      // set up the distributed matrix

      const auto group = TNL::MPI::AllGroup();

      const auto localRange = Partitioner::splitRange( matrixPointer->getRows(), group );

      SharedPointer< DistributedMatrix > distMatrixPointer( localRange, matrixPointer->getRows(), matrixPointer->getColumns(), group );

      DistributedVector dist_x0( localRange, 0, matrixPointer->getRows(), group );

      DistributedVector dist_b( localRange, 0, matrixPointer->getRows(), group );

      const auto communicator = MPI_COMM_WORLD;

      const auto localRange = Partitioner::splitRange( matrixPointer->getRows(), communicator );

      SharedPointer< DistributedMatrix > distMatrixPointer( localRange, matrixPointer->getRows(), matrixPointer->getColumns(), communicator );

      DistributedVector dist_x0( localRange, 0, matrixPointer->getRows(), communicator );

      DistributedVector dist_b( localRange, 0, matrixPointer->getRows(), communicator );



      // copy the row capacities from the global matrix to the distributed matrix

      DistributedRowLengths distributedRowLengths( localRange, 0, matrixPointer->getRows(), group );

      DistributedRowLengths distributedRowLengths( localRange, 0, matrixPointer->getRows(), communicator );

      for( IndexType i = 0; i < distMatrixPointer->getLocalMatrix().getRows(); i++ ) {

         const auto gi = distMatrixPointer->getLocalRowRange().getGlobalIndex( i );

         distributedRowLengths[ gi ] = matrixPointer->getRowCapacity( gi );

src/Benchmarks/ODESolvers/Euler.hpp

+1 −1
Original line number Diff line number Diff line
@@ -200,7 +200,7 @@ void Euler< Problem, SolverMonitor >::computeNewTimeLevel( DofVectorPointer& u, 
   }



   localResidue /= tau * ( RealType ) size;

   TNL::MPI::Allreduce( &localResidue, &currentResidue, 1, MPI_SUM, TNL::MPI::AllGroup() );

   TNL::MPI::Allreduce( &localResidue, &currentResidue, 1, MPI_SUM, MPI_COMM_WORLD );

   //std::cerr << "Local residue = " << localResidue << " - globalResidue = " << currentResidue << std::endl;

}

src/Benchmarks/ODESolvers/Merson.hpp

+2 −2
Original line number Diff line number Diff line
@@ -403,7 +403,7 @@ typename Problem :: RealType Merson< Problem, SolverMonitor >::computeError( con 
      }

#endif

   }

   TNL::MPI::Allreduce( &eps, &maxEps, 1, MPI_MAX, TNL::MPI::AllGroup() );

   TNL::MPI::Allreduce( &eps, &maxEps, 1, MPI_MAX, MPI_COMM_WORLD );

   return maxEps;

}
@@ -465,7 +465,7 @@ void Merson< Problem, SolverMonitor >::computeNewTimeLevel( const RealType time, 
   }



   localResidue /= tau * ( RealType ) size;

   TNL::MPI::Allreduce( &localResidue, &currentResidue, 1, MPI_SUM, TNL::MPI::AllGroup() );

   TNL::MPI::Allreduce( &localResidue, &currentResidue, 1, MPI_SUM, MPI_COMM_WORLD );

/*#ifdef USE_MPI

   TNLMPI::Allreduce( localResidue, currentResidue, 1, MPI_SUM);

#else