Commit afce4ec6 authored by Jakub Klinkovský's avatar Jakub Klinkovský
Browse files

Merge branch 'cineca/mpi' into develop

* cineca/mpi:
  Added specialization of MPIDataType for bool
  Added missing operations to MpiDefs.h
  Added assertions to MpiCommunicator.h checking that MPI functions are not called with the null group
  Whitespace and typo fixes in MpiCommunicator
  Fixed the TNL_MPI_PRINT macro
  Fixed signature of the Allreduce method of communicators
  Added NullGroup to communicators because it is useful for initializations
  Removed empty file and directory in UnitTests
parents 1ec0be26 7570300a
Loading
Loading
Loading
Loading
+55 −36
Original line number Diff line number Diff line
@@ -24,7 +24,6 @@
    typedef struct __attribute__((__packed__))  {
       char name[MPI_MAX_PROCESSOR_NAME];
    } procName;

#endif

#endif
@@ -58,7 +57,13 @@ class MpiCommunicator
      inline static MPI_Datatype MPIDataType( const double* ) { return MPI_DOUBLE; };
      inline static MPI_Datatype MPIDataType( const long double* ) { return MPI_LONG_DOUBLE; };

      // TODO: How to deal with bool
      // TODO: tested with MPI_LOR and MPI_LAND, but there should probably be unit tests for all operations
      inline static MPI_Datatype MPIDataType( const bool* )
      {
         // sizeof(bool) is implementation-defined: https://stackoverflow.com/a/4897859
         static_assert( sizeof(bool) == 1, "The programmer did not count with systems where sizeof(bool) != 1." );
         return MPI_CHAR;
      };

      using Request = MPI_Request;
      using CommunicationGroup = MPI_Comm;
@@ -143,6 +148,7 @@ class MpiCommunicator
         MPI_Init( &argc, &argv );
         NullRequest=MPI_REQUEST_NULL;
         AllGroup=MPI_COMM_WORLD;
         NullGroup=MPI_COMM_NULL;
         redirect = true;

         selectGPU();
@@ -194,10 +200,10 @@ class MpiCommunicator
      static bool IsInitialized()
      {
#ifdef HAVE_MPI
         int inicialized, finalized;
         MPI_Initialized(&inicialized);
         int initialized, finalized;
         MPI_Initialized(&initialized);
         MPI_Finalized(&finalized);
         return inicialized && !finalized;
         return initialized && !finalized;
#else
        return false;
#endif
@@ -207,6 +213,7 @@ class MpiCommunicator
      {
#ifdef HAVE_MPI
        TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");
        TNL_ASSERT_NE(group, NullGroup, "GetRank cannot be called with NullGroup");
        int rank;
        MPI_Comm_rank(group,&rank);
        return rank;
@@ -219,6 +226,7 @@ class MpiCommunicator
      {
#ifdef HAVE_MPI
         TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");
         TNL_ASSERT_NE(group, NullGroup, "GetSize cannot be called with NullGroup");
         int size;
         MPI_Comm_size(group,&size);
         return size;
@@ -251,11 +259,12 @@ class MpiCommunicator
#endif
        }

         static void Barrier(CommunicationGroup comm)
         static void Barrier(CommunicationGroup group)
         {
#ifdef HAVE_MPI
            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");
            MPI_Barrier(comm);
            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");
            TNL_ASSERT_NE(group, NullGroup, "Barrier cannot be called with NullGroup");
            MPI_Barrier(group);
#else
            throw Exceptions::MPISupportMissing();
#endif
@@ -265,7 +274,8 @@ class MpiCommunicator
         static Request ISend( const T *data, int count, int dest, CommunicationGroup group)
         {
#ifdef HAVE_MPI
            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");
            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");
            TNL_ASSERT_NE(group, NullGroup, "ISend cannot be called with NullGroup");
            Request req;
            MPI_Isend((void*) data, count, MPIDataType(data) , dest, 0, group, &req);
            return req;
@@ -278,7 +288,8 @@ class MpiCommunicator
         static Request IRecv( const T *data, int count, int src, CommunicationGroup group)
         {
#ifdef HAVE_MPI
            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");
            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");
            TNL_ASSERT_NE(group, NullGroup, "IRecv cannot be called with NullGroup");
            Request req;
            MPI_Irecv((void*) data, count, MPIDataType(data) , src, 0, group, &req);
            return req;
@@ -290,7 +301,7 @@ class MpiCommunicator
         static void WaitAll(Request *reqs, int length)
         {
#ifdef HAVE_MPI
            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");
            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");
            MPI_Waitall(length, reqs, MPI_STATUSES_IGNORE);
#else
            throw Exceptions::MPISupportMissing();
@@ -301,7 +312,8 @@ class MpiCommunicator
        static void Bcast(  T& data, int count, int root,CommunicationGroup group)
        {
#ifdef HAVE_MPI
        TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");
           TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");
           TNL_ASSERT_NE(group, NullGroup, "BCast cannot be called with NullGroup");
           MPI_Bcast((void*) &data, count,  MPIDataType(data), root, group);
#else
           throw Exceptions::MPISupportMissing();
@@ -309,14 +321,15 @@ class MpiCommunicator
        }

        template< typename T >
        static void Allreduce( T* data,
        static void Allreduce( const T* data,
                               T* reduced_data,
                               int count,
                               const MPI_Op &op,
                               CommunicationGroup group)
        {
#ifdef HAVE_MPI
            MPI_Allreduce( (void*) data, (void*) reduced_data,count,MPIDataType(data),op,group);
            TNL_ASSERT_NE(group, NullGroup, "Allreduce cannot be called with NullGroup");
            MPI_Allreduce( (const void*) data, (void*) reduced_data,count,MPIDataType(data),op,group);
#else
            throw Exceptions::MPISupportMissing();
#endif
@@ -332,6 +345,7 @@ class MpiCommunicator
                    CommunicationGroup group)
         {
#ifdef HAVE_MPI
            TNL_ASSERT_NE(group, NullGroup, "Reduce cannot be called with NullGroup");
            MPI_Reduce( (void*) data, (void*) reduced_data,count,MPIDataType(data),op,root,group);
#else
            throw Exceptions::MPISupportMissing();
@@ -350,6 +364,7 @@ class MpiCommunicator
                                  CommunicationGroup group )
         {
#ifdef HAVE_MPI
            TNL_ASSERT_NE(group, NullGroup, "SendReceive cannot be called with NullGroup");
            MPI_Status status;
            MPI_Sendrecv( ( void* ) sendData,
                          sendCount,
@@ -396,9 +411,11 @@ class MpiCommunicator
#ifdef HAVE_MPI
      static MPI_Request NullRequest;
      static MPI_Comm AllGroup;
      static MPI_Comm NullGroup;
#else
      static int NullRequest;
      static int AllGroup;
      static int NullGroup;
#endif
    private :
      static std::streambuf *psbuf;
@@ -454,9 +471,11 @@ class MpiCommunicator
#ifdef HAVE_MPI
MPI_Request MpiCommunicator::NullRequest;
MPI_Comm MpiCommunicator::AllGroup;
MPI_Comm MpiCommunicator::NullGroup;
#else
int MpiCommunicator::NullRequest;
int MpiCommunicator::AllGroup;
int MpiCommunicator::NullGroup;
#endif
std::streambuf *MpiCommunicator::psbuf;
std::streambuf *MpiCommunicator::backup;
@@ -540,6 +559,6 @@ for( int j = 0; j < TNL::Communicators::MpiCommunicator::GetSize( TNL::Communica
                   << TNL::Communicators::MpiCommunicator::GetSize( TNL::Communicators::MpiCommunicator::AllGroup )      \
                   << " : " << message << std::endl;                                                                     \
      }                                                                                                                  \
      TNL::Communicators::MpiCommunicator::Barrier( Communicator::AllGroup );                                            \
      TNL::Communicators::MpiCommunicator::Barrier( TNL::Communicators::MpiCommunicator::AllGroup );                     \
   }
+15 −2
Original line number Diff line number Diff line
@@ -11,5 +11,18 @@
#pragma once

#ifndef HAVE_MPI
enum MPI_Op { MPI_SUM, MPI_MAX };
enum MPI_Op {
   MPI_MAX,
   MPI_MIN,
   MPI_SUM,
   MPI_PROD,
   MPI_LAND,
   MPI_BAND,
   MPI_LOR,
   MPI_BOR,
   MPI_LXOR,
   MPI_BXOR,
   MPI_MINLOC,
   MPI_MAXLOC,
};
#endif
+7 −3
Original line number Diff line number Diff line
@@ -31,6 +31,7 @@ class NoDistrCommunicator
      typedef int CommunicationGroup;
      static Request NullRequest;
      static CommunicationGroup AllGroup;
      static CommunicationGroup NullGroup;

      static void configSetup( Config::ConfigDescription& config, const String& prefix = "" ){};
 
@@ -43,6 +44,8 @@ class NoDistrCommunicator
      static void Init(int argc, char **argv, bool redirect=false)
      {
          NullRequest = -1;
          AllGroup = 1;
          NullGroup = 0;
      }
      
      static void setRedirection( bool redirect_ ) {}
@@ -105,13 +108,13 @@ class NoDistrCommunicator
      }

      template< typename T >
      static void Allreduce( T* data,
      static void Allreduce( const T* data,
                             T* reduced_data,
                             int count,
                             const MPI_Op &op,
                             CommunicationGroup group )
      {
         memcpy( ( void* ) reduced_data, ( void* ) data, count * sizeof( T ) );
         memcpy( ( void* ) reduced_data, ( const void* ) data, count * sizeof( T ) );
      }

      template< typename T >
@@ -136,6 +139,7 @@ class NoDistrCommunicator

  int NoDistrCommunicator::NullRequest;
  int NoDistrCommunicator::AllGroup;
  int NoDistrCommunicator::NullGroup;

} // namespace <unnamed>
} // namespace Communicators
+0 −1
Original line number Diff line number Diff line
@@ -3,7 +3,6 @@ if( ${WITH_TESTS} )
ADD_SUBDIRECTORY( Containers )
ADD_SUBDIRECTORY( Functions )
ADD_SUBDIRECTORY( Matrices )
ADD_SUBDIRECTORY( Mpi )
ADD_SUBDIRECTORY( Meshes )

ADD_EXECUTABLE( UniquePointerTest UniquePointerTest.cpp )

src/UnitTests/Mpi/CMakeLists.txt

deleted100644 → 0
+0 −0

Empty file deleted.