Commit cc3b9348 authored by Jakub Klinkovský's avatar Jakub Klinkovský
Browse files

MPI refactoring: replaced DimsCreate and CreateNewGroup in MpiCommunicator... 

MPI refactoring: replaced DimsCreate and CreateNewGroup in MpiCommunicator with plain function wrappers
parent db5c4615

src/TNL/Communicators/MpiCommunicator.h

+0 −41
Original line number Diff line number Diff line
@@ -175,47 +175,6 @@ class MpiCommunicator 
         MPI::Alltoall( sendData, sendCount, receiveData, receiveCount, group );

      }





      //dim-number of dimensions, distr array of guess distr - 0 for computation

      //distr array will be filled by computed distribution

      //more information in MPI documentation

      static void DimsCreate(int nproc, int dim, int *distr)

      {

#ifdef HAVE_MPI

         int sum = 0, prod = 1;

         for( int i = 0;i < dim; i++ ) {

            sum += distr[ i ];

            prod *= distr[ i ];

         }

         if( prod != 0 && prod != GetSize( AllGroup ) )

            throw std::logic_error( "The program tries to call MPI_Dims_create with wrong dimensions."

                                    "Non of the dimensions is zero and product of all dimensions does "

                                    "not fit with number of MPI processes." );

         if(sum==0) {

            for(int i=0;i<dim-1;i++)

               distr[i]=1;

            distr[dim-1]=0;

         }



         MPI_Dims_create(nproc, dim, distr);

#else

         for(int i=0;i<dim;i++)

            distr[i]=1;

#endif

      }



      static void CreateNewGroup( bool meToo, int myRank, CommunicationGroup &oldGroup, CommunicationGroup &newGroup )

      {

#ifdef HAVE_MPI

         if(meToo)

            MPI_Comm_split(oldGroup, 1, myRank, &newGroup);

         else

            MPI_Comm_split(oldGroup, MPI_UNDEFINED, GetRank(oldGroup), &newGroup);

#else

         newGroup=oldGroup;

#endif

      }



#ifdef HAVE_MPI

      static MPI_Comm AllGroup;

      static MPI_Comm NullGroup;

src/TNL/MPI/DummyDefs.h

+12 −0
Original line number Diff line number Diff line
@@ -36,4 +36,16 @@ enum { 
   MPI_THREAD_SERIALIZED,

   MPI_THREAD_MULTIPLE

};



// Miscellaneous constants

#define MPI_ANY_SOURCE         -1                      /* match any source rank */

#define MPI_PROC_NULL          -2                      /* rank of null process */

#define MPI_ROOT               -4                      /* special value for intercomms */

#define MPI_ANY_TAG            -1                      /* match any message tag */

#define MPI_UNDEFINED          -32766                  /* undefined stuff */

#define MPI_DIST_GRAPH         3                       /* dist graph topology */

#define MPI_CART               1                       /* cartesian topology */

#define MPI_GRAPH              2                       /* graph topology */

#define MPI_KEYVAL_INVALID     -1                      /* invalid key value */



#endif

src/TNL/MPI/Wrappers.h

+50 −0
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ 
#pragma once



#include <iostream>

#include <stdexcept>



#ifdef HAVE_MPI

   #include <mpi.h>
@@ -156,6 +157,55 @@ inline int GetSize( MPI_Comm group = AllGroup() ) 
#endif

}



// wrappers for MPI helper functions



inline MPI_Comm Comm_split( MPI_Comm comm, int color, int key )

{

#ifdef HAVE_MPI

   MPI_Comm newcomm;

   MPI_Comm_split( comm, color, key, &newcomm );

   return newcomm;

#else

   return comm;

#endif

}



/**

 * \brief Wrapper for \ref MPI_Dims_create.

 *

 * \param nproc - number of processes in the group to be distributed

 * \param ndims - number of dimensions of the Cartesian grid

 * \param dims - distribution of processes into the \e dim-dimensional

 *               Cartesian grid (array of length \e ndims)

 *

 * Negative input values of \e dims[i] are erroneous. An error will occur if

 * \e nproc is not a multiple of the product of all non-zero values \e dims[i].

 *

 * See the MPI documentation for more information.

 */

inline void Compute_dims( int nproc, int ndims, int* dims )

{

#ifdef HAVE_MPI

   int prod = 1;

   for( int i = 0; i < ndims; i++ ) {

      if( dims[ i ] < 0 )

         throw std::invalid_argument( "Negative value passed to MPI::Compute_dims in the dims array argument." );

      if( dims[ i ] > 0 )

         prod *= dims[ i ];

   }



   if( nproc % prod != 0 )

      throw std::logic_error( "The program tries to call MPI_Dims_create with wrong dimensions."

            "The product of the non-zero values dims[i] is " + std::to_string(prod) + " and the "

            "number of processes (" + std::to_string(nproc) + ") is not a multiple of the product." );



   MPI_Dims_create( nproc, ndims, dims );

#else

   for( int i = 0; i < ndims; i++)

      dims[ i ] = 1;

#endif

}



// wrappers for MPI communication functions



inline void Barrier( MPI_Comm group = AllGroup() )