Commit ee2fd25d authored by Jakub Klinkovský's avatar Jakub Klinkovský
Browse files

MPI refactoring: removed MpiCommunicator from algebraic data structures 

This affects DistributedArray, DistributedArrayView, DistributedVector,
DistributedVectorView and DistributedMatrix. Allocators were added to
DistributedArray and DistributedVector. Also updated all benchmarks and
unit tests.
parent 3c5d17e3

src/Benchmarks/DistSpMV/tnl-benchmark-distributed-spmv.h

+14 −16
Original line number Diff line number Diff line
@@ -19,8 +19,8 @@ 
#include <TNL/Config/parseCommandLine.h>

#include <TNL/Devices/Host.h>

#include <TNL/Devices/Cuda.h>

#include <TNL/Communicators/MpiCommunicator.h>

#include <TNL/MPI/ScopedInitializer.h>

#include <TNL/MPI/Config.h>

#include <TNL/Containers/Partitioner.h>

#include <TNL/Containers/DistributedVector.h>

#include <TNL/Matrices/DistributedMatrix.h>
@@ -38,8 +38,6 @@ using SegmentsType = TNL::Algorithms::Segments::SlicedEllpack< _Device, _Index, 
using namespace TNL;

using namespace TNL::Benchmarks;



using CommunicatorType = Communicators::MpiCommunicator;





template< typename Matrix, typename Vector >

void
@@ -110,7 +108,7 @@ benchmarkDistributedSpmv( Benchmark& benchmark, 
   // benchmark function

   auto compute = [&]() {

      matrix.vectorProduct( x, y );

      Matrix::CommunicatorType::Barrier( matrix.getCommunicationGroup() );

      TNL::MPI::Barrier( matrix.getCommunicationGroup() );

   };



   benchmark.time< typename Matrix::DeviceType >( reset, performer, compute );
@@ -150,9 +148,9 @@ struct SpmvBenchmark 
   using IndexType = typename MatrixType::IndexType;

   using VectorType = Containers::Vector< RealType, DeviceType, IndexType >;



   using Partitioner = Containers::Partitioner< IndexType, CommunicatorType >;

   using DistributedMatrix = Matrices::DistributedMatrix< MatrixType, CommunicatorType >;

   using DistributedVector = Containers::DistributedVector< RealType, DeviceType, IndexType, CommunicatorType >;

   using Partitioner = Containers::Partitioner< IndexType >;

   using DistributedMatrix = Matrices::DistributedMatrix< MatrixType >;

   using DistributedVector = Containers::DistributedVector< RealType, DeviceType, IndexType >;

   using DistributedRowLengths = typename DistributedMatrix::CompressedRowLengthsVector;



   static bool
@@ -169,7 +167,7 @@ struct SpmvBenchmark 
      matrix.getCompressedRowLengths( rowLengths );

      const IndexType maxRowLength = max( rowLengths );



      const String name = String( (CommunicatorType::isDistributed()) ? "DistSpMV" : "SpMV" )

      const String name = String( (TNL::MPI::GetSize() > 1) ? "DistSpMV" : "SpMV" )

                          + " (" + parameters.getParameter< String >( "name" ) + "): ";

      benchmark.newBenchmark( name, metadata );

      benchmark.setMetadataColumns( Benchmark::MetadataColumns({
@@ -189,13 +187,13 @@ struct SpmvBenchmark 
         getTrivialOrdering( matrix, perm, iperm );

         MatrixType matrix_perm;

         Matrices::reorderSparseMatrix( matrix, matrix_perm, perm, iperm );

         if( CommunicatorType::isDistributed() )

         if( TNL::MPI::GetSize() > 1 )

            runDistributed( benchmark, metadata, parameters, matrix_perm, vector );

         else

            runNonDistributed( benchmark, metadata, parameters, matrix_perm, vector );

      }

      else {

         if( CommunicatorType::isDistributed() )

         if( TNL::MPI::GetSize() > 1 )

            runDistributed( benchmark, metadata, parameters, matrix, vector );

         else

            runNonDistributed( benchmark, metadata, parameters, matrix, vector );
@@ -225,7 +223,7 @@ struct SpmvBenchmark 
                   VectorType& vector )

   {

      // set up the distributed matrix

      const auto group = CommunicatorType::AllGroup;

      const auto group = TNL::MPI::AllGroup();

      const auto localRange = Partitioner::splitRange( matrix.getRows(), group );

      DistributedMatrix distributedMatrix( localRange, matrix.getRows(), matrix.getColumns(), group );

      DistributedVector distributedVector( localRange, 0, matrix.getRows(), group );
@@ -267,8 +265,8 @@ struct SpmvBenchmark 
      DistributedVector distributedY;

      distributedY.setLike( distributedVector );

      distributedMatrix.vectorProduct( distributedVector, distributedY );

      const int rank = CommunicatorType::GetRank( distributedMatrix.getCommunicationGroup() );

      const int nproc = CommunicatorType::GetSize( distributedMatrix.getCommunicationGroup() );

      const int rank = TNL::MPI::GetRank( distributedMatrix.getCommunicationGroup() );

      const int nproc = TNL::MPI::GetSize( distributedMatrix.getCommunicationGroup() );

      typename VectorType::ViewType subY( &y[ Partitioner::getOffset( matrix.getRows(), rank, nproc ) ],

                                          Partitioner::getSizeForRank( matrix.getRows(), rank, nproc ) );

      TNL_ASSERT_EQ( distributedY.getLocalView(), subY, "WRONG RESULT !!!" );
@@ -294,7 +292,7 @@ configSetup( Config::ConfigDescription & config ) 
   config.addDelimiter( "Device settings:" );

   Devices::Host::configSetup( config );

   Devices::Cuda::configSetup( config );

   CommunicatorType::configSetup( config );

   TNL::MPI::configSetup( config );

}



int
@@ -310,14 +308,14 @@ main( int argc, char* argv[] ) 
   configSetup( conf_desc );



   TNL::MPI::ScopedInitializer mpi(argc, argv);

   const int rank = CommunicatorType::GetRank( CommunicatorType::AllGroup );

   const int rank = TNL::MPI::GetRank();



   if( ! parseCommandLine( argc, argv, conf_desc, parameters ) )

      return EXIT_FAILURE;



   if( ! Devices::Host::setup( parameters ) ||

       ! Devices::Cuda::setup( parameters ) ||

       ! CommunicatorType::setup( parameters ) )

       ! TNL::MPI::setup( parameters ) )

      return EXIT_FAILURE;



   const String & logFileName = parameters.getParameter< String >( "log-file" );

src/Benchmarks/LinearSolvers/benchmarks.h

+3 −3
Original line number Diff line number Diff line
@@ -33,10 +33,10 @@ void barrier( const Matrix& matrix ) 
{

}



template< typename Matrix, typename Communicator >

void barrier( const Matrices::DistributedMatrix< Matrix, Communicator >& matrix )

template< typename Matrix >

void barrier( const Matrices::DistributedMatrix< Matrix >& matrix )

{

   Communicator::Barrier( matrix.getCommunicationGroup() );

   TNL::MPI::Barrier( matrix.getCommunicationGroup() );

}



template< typename Device >

src/Benchmarks/LinearSolvers/tnl-benchmark-linear-solvers.h

+11 −13
Original line number Diff line number Diff line
@@ -24,8 +24,8 @@ 
#include <TNL/Config/parseCommandLine.h>

#include <TNL/Devices/Host.h>

#include <TNL/Devices/Cuda.h>

#include <TNL/Communicators/MpiCommunicator.h>

#include <TNL/MPI/ScopedInitializer.h>

#include <TNL/MPI/Config.h>

#include <TNL/Containers/Partitioner.h>

#include <TNL/Containers/DistributedVector.h>

#include <TNL/Matrices/DistributedMatrix.h>
@@ -65,8 +65,6 @@ using namespace TNL; 
using namespace TNL::Benchmarks;

using namespace TNL::Pointers;



using CommunicatorType = Communicators::MpiCommunicator;





static const std::set< std::string > valid_solvers = {

   "gmres",
@@ -333,9 +331,9 @@ struct LinearSolversBenchmark 
   using IndexType = typename MatrixType::IndexType;

   using VectorType = Containers::Vector< RealType, DeviceType, IndexType >;



   using Partitioner = Containers::Partitioner< IndexType, CommunicatorType >;

   using DistributedMatrix = Matrices::DistributedMatrix< MatrixType, CommunicatorType >;

   using DistributedVector = Containers::DistributedVector< RealType, DeviceType, IndexType, CommunicatorType >;

   using Partitioner = Containers::Partitioner< IndexType >;

   using DistributedMatrix = Matrices::DistributedMatrix< MatrixType >;

   using DistributedVector = Containers::DistributedVector< RealType, DeviceType, IndexType >;

   using DistributedRowLengths = typename DistributedMatrix::CompressedRowLengthsVector;



   static bool
@@ -383,7 +381,7 @@ struct LinearSolversBenchmark 
      matrixPointer->getCompressedRowLengths( rowLengths );

      const IndexType maxRowLength = max( rowLengths );



      const String name = String( (CommunicatorType::isDistributed()) ? "Distributed linear solvers" : "Linear solvers" )

      const String name = String( (TNL::MPI::GetSize() > 1) ? "Distributed linear solvers" : "Linear solvers" )

                          + " (" + parameters.getParameter< String >( "name" ) + "): ";

      benchmark.newBenchmark( name, metadata );

      benchmark.setMetadataColumns( Benchmark::MetadataColumns({
@@ -408,13 +406,13 @@ struct LinearSolversBenchmark 
         Matrices::reorderSparseMatrix( *matrixPointer, *matrix_perm, perm, iperm );

         Matrices::reorderArray( x0, x0_perm, perm );

         Matrices::reorderArray( b, b_perm, perm );

         if( CommunicatorType::isDistributed() )

         if( TNL::MPI::GetSize() > 1 )

            runDistributed( benchmark, metadata, parameters, matrix_perm, x0_perm, b_perm );

         else

            runNonDistributed( benchmark, metadata, parameters, matrix_perm, x0_perm, b_perm );

      }

      else {

         if( CommunicatorType::isDistributed() )

         if( TNL::MPI::GetSize() > 1 )

            runDistributed( benchmark, metadata, parameters, matrixPointer, x0, b );

         else

            runNonDistributed( benchmark, metadata, parameters, matrixPointer, x0, b );
@@ -432,7 +430,7 @@ struct LinearSolversBenchmark 
                   const VectorType& b )

   {

      // set up the distributed matrix

      const auto group = CommunicatorType::AllGroup;

      const auto group = TNL::MPI::AllGroup();

      const auto localRange = Partitioner::splitRange( matrixPointer->getRows(), group );

      SharedPointer< DistributedMatrix > distMatrixPointer( localRange, matrixPointer->getRows(), matrixPointer->getColumns(), group );

      DistributedVector dist_x0( localRange, 0, matrixPointer->getRows(), group );
@@ -567,7 +565,7 @@ configSetup( Config::ConfigDescription& config ) 
   config.addDelimiter( "Device settings:" );

   Devices::Host::configSetup( config );

   Devices::Cuda::configSetup( config );

   CommunicatorType::configSetup( config );

   TNL::MPI::configSetup( config );



   config.addDelimiter( "Linear solver settings:" );

   Solvers::IterativeSolver< double, int >::configSetup( config );
@@ -593,13 +591,13 @@ main( int argc, char* argv[] ) 
   configSetup( conf_desc );



   TNL::MPI::ScopedInitializer mpi(argc, argv);

   const int rank = CommunicatorType::GetRank( CommunicatorType::AllGroup );

   const int rank = TNL::MPI::GetRank();



   if( ! parseCommandLine( argc, argv, conf_desc, parameters ) )

      return EXIT_FAILURE;

   if( ! Devices::Host::setup( parameters ) ||

       ! Devices::Cuda::setup( parameters ) ||

       ! CommunicatorType::setup( parameters ) )

       ! TNL::MPI::setup( parameters ) )

      return EXIT_FAILURE;



   const String & logFileName = parameters.getParameter< String >( "log-file" );

src/Benchmarks/ODESolvers/Euler.hpp

+1 −1
Original line number Diff line number Diff line
@@ -200,7 +200,7 @@ void Euler< Problem, SolverMonitor >::computeNewTimeLevel( DofVectorPointer& u, 
   }



   localResidue /= tau * ( RealType ) size;

   Problem::CommunicatorType::Allreduce( &localResidue, &currentResidue, 1, MPI_SUM, Problem::CommunicatorType::AllGroup );

   TNL::MPI::Allreduce( &localResidue, &currentResidue, 1, MPI_SUM, TNL::MPI::AllGroup() );

   //std::cerr << "Local residue = " << localResidue << " - globalResidue = " << currentResidue << std::endl;

}

src/Benchmarks/ODESolvers/Merson.hpp

+5 −5
Original line number Diff line number Diff line
@@ -403,7 +403,7 @@ typename Problem :: RealType Merson< Problem, SolverMonitor >::computeError( con 
      }

#endif

   }

   Problem::CommunicatorType::Allreduce( &eps, &maxEps, 1, MPI_MAX, Problem::CommunicatorType::AllGroup );

   TNL::MPI::Allreduce( &eps, &maxEps, 1, MPI_MAX, TNL::MPI::AllGroup() );

   return maxEps;

}
@@ -465,7 +465,7 @@ void Merson< Problem, SolverMonitor >::computeNewTimeLevel( const RealType time, 
   }



   localResidue /= tau * ( RealType ) size;

   Problem::CommunicatorType::Allreduce( &localResidue, &currentResidue, 1, MPI_SUM, Problem::CommunicatorType::AllGroup);

   TNL::MPI::Allreduce( &localResidue, &currentResidue, 1, MPI_SUM, TNL::MPI::AllGroup() );

/*#ifdef USE_MPI

   TNLMPI::Allreduce( localResidue, currentResidue, 1, MPI_SUM);

#else