Commit afce4ec6 authored by Jakub Klinkovský's avatar Jakub Klinkovský
Browse files

Merge branch 'cineca/mpi' into develop 

* cineca/mpi:
  Added specialization of MPIDataType for bool
  Added missing operations to MpiDefs.h
  Added assertions to MpiCommunicator.h checking that MPI functions are not called with the null group
  Whitespace and typo fixes in MpiCommunicator
  Fixed the TNL_MPI_PRINT macro
  Fixed signature of the Allreduce method of communicators
  Added NullGroup to communicators because it is useful for initializations
  Removed empty file and directory in UnitTests
parents 1ec0be26 7570300a

src/TNL/Communicators/MpiCommunicator.h

+55 −36
Original line number Diff line number Diff line
@@ -24,7 +24,6 @@ 
    typedef struct __attribute__((__packed__))  {

       char name[MPI_MAX_PROCESSOR_NAME];

    } procName;



#endif



#endif
@@ -58,7 +57,13 @@ class MpiCommunicator 
      inline static MPI_Datatype MPIDataType( const double* ) { return MPI_DOUBLE; };

      inline static MPI_Datatype MPIDataType( const long double* ) { return MPI_LONG_DOUBLE; };



      // TODO: How to deal with bool

      // TODO: tested with MPI_LOR and MPI_LAND, but there should probably be unit tests for all operations

      inline static MPI_Datatype MPIDataType( const bool* )

      {

         // sizeof(bool) is implementation-defined: https://stackoverflow.com/a/4897859

         static_assert( sizeof(bool) == 1, "The programmer did not count with systems where sizeof(bool) != 1." );

         return MPI_CHAR;

      };



      using Request = MPI_Request;

      using CommunicationGroup = MPI_Comm;
@@ -143,6 +148,7 @@ class MpiCommunicator 
         MPI_Init( &argc, &argv );

         NullRequest=MPI_REQUEST_NULL;

         AllGroup=MPI_COMM_WORLD;

         NullGroup=MPI_COMM_NULL;

         redirect = true;



         selectGPU();
@@ -194,10 +200,10 @@ class MpiCommunicator 
      static bool IsInitialized()

      {

#ifdef HAVE_MPI

         int inicialized, finalized;

         MPI_Initialized(&inicialized);

         int initialized, finalized;

         MPI_Initialized(&initialized);

         MPI_Finalized(&finalized);

         return inicialized && !finalized;

         return initialized && !finalized;

#else

        return false;

#endif
@@ -207,6 +213,7 @@ class MpiCommunicator 
      {

#ifdef HAVE_MPI

        TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");

        TNL_ASSERT_NE(group, NullGroup, "GetRank cannot be called with NullGroup");

        int rank;

        MPI_Comm_rank(group,&rank);

        return rank;
@@ -219,6 +226,7 @@ class MpiCommunicator 
      {

#ifdef HAVE_MPI

         TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");

         TNL_ASSERT_NE(group, NullGroup, "GetSize cannot be called with NullGroup");

         int size;

         MPI_Comm_size(group,&size);

         return size;
@@ -251,11 +259,12 @@ class MpiCommunicator 
#endif

        }



         static void Barrier(CommunicationGroup comm)

         static void Barrier(CommunicationGroup group)

         {

#ifdef HAVE_MPI

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");

            MPI_Barrier(comm);

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");

            TNL_ASSERT_NE(group, NullGroup, "Barrier cannot be called with NullGroup");

            MPI_Barrier(group);

#else

            throw Exceptions::MPISupportMissing();

#endif
@@ -265,7 +274,8 @@ class MpiCommunicator 
         static Request ISend( const T *data, int count, int dest, CommunicationGroup group)

         {

#ifdef HAVE_MPI

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");

            TNL_ASSERT_NE(group, NullGroup, "ISend cannot be called with NullGroup");

            Request req;

            MPI_Isend((void*) data, count, MPIDataType(data) , dest, 0, group, &req);

            return req;
@@ -278,7 +288,8 @@ class MpiCommunicator 
         static Request IRecv( const T *data, int count, int src, CommunicationGroup group)

         {

#ifdef HAVE_MPI

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");

            TNL_ASSERT_NE(group, NullGroup, "IRecv cannot be called with NullGroup");

            Request req;

            MPI_Irecv((void*) data, count, MPIDataType(data) , src, 0, group, &req);

            return req;
@@ -290,7 +301,7 @@ class MpiCommunicator 
         static void WaitAll(Request *reqs, int length)

         {

#ifdef HAVE_MPI

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");

            MPI_Waitall(length, reqs, MPI_STATUSES_IGNORE);

#else

            throw Exceptions::MPISupportMissing();
@@ -301,7 +312,8 @@ class MpiCommunicator 
        static void Bcast(  T& data, int count, int root,CommunicationGroup group)

        {

#ifdef HAVE_MPI

        TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");

           TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");

           TNL_ASSERT_NE(group, NullGroup, "BCast cannot be called with NullGroup");

           MPI_Bcast((void*) &data, count,  MPIDataType(data), root, group);

#else

           throw Exceptions::MPISupportMissing();
@@ -309,14 +321,15 @@ class MpiCommunicator 
        }



        template< typename T >

        static void Allreduce( T* data,

        static void Allreduce( const T* data,

                               T* reduced_data,

                               int count,

                               const MPI_Op &op,

                               CommunicationGroup group)

        {

#ifdef HAVE_MPI

            MPI_Allreduce( (void*) data, (void*) reduced_data,count,MPIDataType(data),op,group);

            TNL_ASSERT_NE(group, NullGroup, "Allreduce cannot be called with NullGroup");

            MPI_Allreduce( (const void*) data, (void*) reduced_data,count,MPIDataType(data),op,group);

#else

            throw Exceptions::MPISupportMissing();

#endif
@@ -332,6 +345,7 @@ class MpiCommunicator 
                    CommunicationGroup group)

         {

#ifdef HAVE_MPI

            TNL_ASSERT_NE(group, NullGroup, "Reduce cannot be called with NullGroup");

            MPI_Reduce( (void*) data, (void*) reduced_data,count,MPIDataType(data),op,root,group);

#else

            throw Exceptions::MPISupportMissing();
@@ -350,6 +364,7 @@ class MpiCommunicator 
                                  CommunicationGroup group )

         {

#ifdef HAVE_MPI

            TNL_ASSERT_NE(group, NullGroup, "SendReceive cannot be called with NullGroup");

            MPI_Status status;

            MPI_Sendrecv( ( void* ) sendData,

                          sendCount,
@@ -396,9 +411,11 @@ class MpiCommunicator 
#ifdef HAVE_MPI

      static MPI_Request NullRequest;

      static MPI_Comm AllGroup;

      static MPI_Comm NullGroup;

#else

      static int NullRequest;

      static int AllGroup;

      static int NullGroup;

#endif

    private :

      static std::streambuf *psbuf;
@@ -454,9 +471,11 @@ class MpiCommunicator 
#ifdef HAVE_MPI

MPI_Request MpiCommunicator::NullRequest;

MPI_Comm MpiCommunicator::AllGroup;

MPI_Comm MpiCommunicator::NullGroup;

#else

int MpiCommunicator::NullRequest;

int MpiCommunicator::AllGroup;

int MpiCommunicator::NullGroup;

#endif

std::streambuf *MpiCommunicator::psbuf;

std::streambuf *MpiCommunicator::backup;
@@ -540,6 +559,6 @@ for( int j = 0; j < TNL::Communicators::MpiCommunicator::GetSize( TNL::Communica 
                   << TNL::Communicators::MpiCommunicator::GetSize( TNL::Communicators::MpiCommunicator::AllGroup )      \

                   << " : " << message << std::endl;                                                                     \

      }                                                                                                                  \

      TNL::Communicators::MpiCommunicator::Barrier( Communicator::AllGroup );                                            \

      TNL::Communicators::MpiCommunicator::Barrier( TNL::Communicators::MpiCommunicator::AllGroup );                     \

   }

src/TNL/Communicators/MpiDefs.h

+15 −2
Original line number Diff line number Diff line
@@ -11,5 +11,18 @@ 
#pragma once



#ifndef HAVE_MPI

enum MPI_Op { MPI_SUM, MPI_MAX };

enum MPI_Op {

   MPI_MAX,

   MPI_MIN,

   MPI_SUM,

   MPI_PROD,

   MPI_LAND,

   MPI_BAND,

   MPI_LOR,

   MPI_BOR,

   MPI_LXOR,

   MPI_BXOR,

   MPI_MINLOC,

   MPI_MAXLOC,

};

#endif

src/TNL/Communicators/NoDistrCommunicator.h

+7 −3
Original line number Diff line number Diff line
@@ -31,6 +31,7 @@ class NoDistrCommunicator 
      typedef int CommunicationGroup;

      static Request NullRequest;

      static CommunicationGroup AllGroup;

      static CommunicationGroup NullGroup;



      static void configSetup( Config::ConfigDescription& config, const String& prefix = "" ){};
@@ -43,6 +44,8 @@ class NoDistrCommunicator 
      static void Init(int argc, char **argv, bool redirect=false)

      {

          NullRequest = -1;

          AllGroup = 1;

          NullGroup = 0;

      }

      

      static void setRedirection( bool redirect_ ) {}
@@ -105,13 +108,13 @@ class NoDistrCommunicator 
      }



      template< typename T >

      static void Allreduce( T* data,

      static void Allreduce( const T* data,

                             T* reduced_data,

                             int count,

                             const MPI_Op &op,

                             CommunicationGroup group )

      {

         memcpy( ( void* ) reduced_data, ( void* ) data, count * sizeof( T ) );

         memcpy( ( void* ) reduced_data, ( const void* ) data, count * sizeof( T ) );

      }



      template< typename T >
@@ -136,6 +139,7 @@ class NoDistrCommunicator 


  int NoDistrCommunicator::NullRequest;

  int NoDistrCommunicator::AllGroup;

  int NoDistrCommunicator::NullGroup;



} // namespace <unnamed>

} // namespace Communicators

src/UnitTests/CMakeLists.txt

+0 −1
Original line number Diff line number Diff line
@@ -3,7 +3,6 @@ if( ${WITH_TESTS} ) 
ADD_SUBDIRECTORY( Containers )

ADD_SUBDIRECTORY( Functions )

ADD_SUBDIRECTORY( Matrices )

ADD_SUBDIRECTORY( Mpi )

ADD_SUBDIRECTORY( Meshes )



ADD_EXECUTABLE( UniquePointerTest UniquePointerTest.cpp )

src/UnitTests/Mpi/CMakeLists.txt

deleted100644 → 0
+0 −0

Empty file deleted.