Commit f2ea6987 authored by Tomáš Oberhuber's avatar Tomáš Oberhuber
Browse files

Merge branch 'develop' into operators-optimizations

parents a4e310a8 e3b5d764

.gitlab-ci.yml

+4 −1
Original line number Diff line number Diff line
@@ -26,7 +26,10 @@ stages: 
.build_template_def: &build_template

    stage: build

    script:

        - export NUM_CORES=$(grep "core id" /proc/cpuinfo | wc -l)

        # all cores including hyperthreading

#        - export NUM_CORES=$(grep "core id" /proc/cpuinfo | wc -l)

#       # all pyhsical cores

        - export NUM_CORES=$(grep "core id" /proc/cpuinfo | sort -u | wc -l)

        - export MAKEFLAGS="-l$(echo 1.5*$NUM_CORES | bc) -j$NUM_CORES"

        - mkdir -p "./builddir/$CI_JOB_NAME"

        - pushd "./builddir/$CI_JOB_NAME"

src/Python/pytnl/tnl/SparseMatrix.h

+1 −1
Original line number Diff line number Diff line
@@ -99,7 +99,7 @@ void export_Matrix( py::module & m, const char* name ) 
        // TODO: these two don't work

        //.def("addMatrix",           &Matrix::addMatrix)

        //.def("getTransposition",    &Matrix::getTransposition)

        .def("performSORIteration", &Matrix::template performSORIteration< VectorType >)

        .def("performSORIteration", &Matrix::template performSORIteration< VectorType, VectorType >)

//        .def("assign",              &Matrix::operator=)

        .def("assign", []( Matrix& matrix, const Matrix& other ) -> Matrix& {

                return matrix = other;

src/TNL/CMakeLists.txt

+0 −1
Original line number Diff line number Diff line
@@ -20,7 +20,6 @@ SET( CURRENT_DIR ${CMAKE_SOURCE_DIR}/src/TNL ) 


set( headers

     Assert.h

     Constants.h

     CudaSharedMemory.h

     CudaStreamPool.h

     File.h

src/TNL/Communicators/MpiCommunicator.h

+55 −36
Original line number Diff line number Diff line
@@ -24,7 +24,6 @@ 
    typedef struct __attribute__((__packed__))  {

       char name[MPI_MAX_PROCESSOR_NAME];

    } procName;



#endif



#endif
@@ -58,7 +57,13 @@ class MpiCommunicator 
      inline static MPI_Datatype MPIDataType( const double* ) { return MPI_DOUBLE; };

      inline static MPI_Datatype MPIDataType( const long double* ) { return MPI_LONG_DOUBLE; };



      // TODO: How to deal with bool

      // TODO: tested with MPI_LOR and MPI_LAND, but there should probably be unit tests for all operations

      inline static MPI_Datatype MPIDataType( const bool* )

      {

         // sizeof(bool) is implementation-defined: https://stackoverflow.com/a/4897859

         static_assert( sizeof(bool) == 1, "The programmer did not count with systems where sizeof(bool) != 1." );

         return MPI_CHAR;

      };



      using Request = MPI_Request;

      using CommunicationGroup = MPI_Comm;
@@ -143,6 +148,7 @@ class MpiCommunicator 
         MPI_Init( &argc, &argv );

         NullRequest=MPI_REQUEST_NULL;

         AllGroup=MPI_COMM_WORLD;

         NullGroup=MPI_COMM_NULL;

         redirect = true;



         selectGPU();
@@ -194,10 +200,10 @@ class MpiCommunicator 
      static bool IsInitialized()

      {

#ifdef HAVE_MPI

         int inicialized, finalized;

         MPI_Initialized(&inicialized);

         int initialized, finalized;

         MPI_Initialized(&initialized);

         MPI_Finalized(&finalized);

         return inicialized && !finalized;

         return initialized && !finalized;

#else

        return false;

#endif
@@ -207,6 +213,7 @@ class MpiCommunicator 
      {

#ifdef HAVE_MPI

        TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");

        TNL_ASSERT_NE(group, NullGroup, "GetRank cannot be called with NullGroup");

        int rank;

        MPI_Comm_rank(group,&rank);

        return rank;
@@ -219,6 +226,7 @@ class MpiCommunicator 
      {

#ifdef HAVE_MPI

         TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");

         TNL_ASSERT_NE(group, NullGroup, "GetSize cannot be called with NullGroup");

         int size;

         MPI_Comm_size(group,&size);

         return size;
@@ -251,11 +259,12 @@ class MpiCommunicator 
#endif

        }



         static void Barrier(CommunicationGroup comm)

         static void Barrier(CommunicationGroup group)

         {

#ifdef HAVE_MPI

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");

            MPI_Barrier(comm);

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");

            TNL_ASSERT_NE(group, NullGroup, "Barrier cannot be called with NullGroup");

            MPI_Barrier(group);

#else

            throw Exceptions::MPISupportMissing();

#endif
@@ -265,7 +274,8 @@ class MpiCommunicator 
         static Request ISend( const T *data, int count, int dest, CommunicationGroup group)

         {

#ifdef HAVE_MPI

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");

            TNL_ASSERT_NE(group, NullGroup, "ISend cannot be called with NullGroup");

            Request req;

            MPI_Isend((void*) data, count, MPIDataType(data) , dest, 0, group, &req);

            return req;
@@ -278,7 +288,8 @@ class MpiCommunicator 
         static Request IRecv( const T *data, int count, int src, CommunicationGroup group)

         {

#ifdef HAVE_MPI

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");

            TNL_ASSERT_NE(group, NullGroup, "IRecv cannot be called with NullGroup");

            Request req;

            MPI_Irecv((void*) data, count, MPIDataType(data) , src, 0, group, &req);

            return req;
@@ -290,7 +301,7 @@ class MpiCommunicator 
         static void WaitAll(Request *reqs, int length)

         {

#ifdef HAVE_MPI

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");

            TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");

            MPI_Waitall(length, reqs, MPI_STATUSES_IGNORE);

#else

            throw Exceptions::MPISupportMissing();
@@ -301,7 +312,8 @@ class MpiCommunicator 
        static void Bcast(  T& data, int count, int root,CommunicationGroup group)

        {

#ifdef HAVE_MPI

        TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not inicialized");

           TNL_ASSERT_TRUE(IsInitialized(), "Fatal Error - MPI communicator is not initialized");

           TNL_ASSERT_NE(group, NullGroup, "BCast cannot be called with NullGroup");

           MPI_Bcast((void*) &data, count,  MPIDataType(data), root, group);

#else

           throw Exceptions::MPISupportMissing();
@@ -309,14 +321,15 @@ class MpiCommunicator 
        }



        template< typename T >

        static void Allreduce( T* data,

        static void Allreduce( const T* data,

                               T* reduced_data,

                               int count,

                               const MPI_Op &op,

                               CommunicationGroup group)

        {

#ifdef HAVE_MPI

            MPI_Allreduce( (void*) data, (void*) reduced_data,count,MPIDataType(data),op,group);

            TNL_ASSERT_NE(group, NullGroup, "Allreduce cannot be called with NullGroup");

            MPI_Allreduce( (const void*) data, (void*) reduced_data,count,MPIDataType(data),op,group);

#else

            throw Exceptions::MPISupportMissing();

#endif
@@ -332,6 +345,7 @@ class MpiCommunicator 
                    CommunicationGroup group)

         {

#ifdef HAVE_MPI

            TNL_ASSERT_NE(group, NullGroup, "Reduce cannot be called with NullGroup");

            MPI_Reduce( (void*) data, (void*) reduced_data,count,MPIDataType(data),op,root,group);

#else

            throw Exceptions::MPISupportMissing();
@@ -350,6 +364,7 @@ class MpiCommunicator 
                                  CommunicationGroup group )

         {

#ifdef HAVE_MPI

            TNL_ASSERT_NE(group, NullGroup, "SendReceive cannot be called with NullGroup");

            MPI_Status status;

            MPI_Sendrecv( ( void* ) sendData,

                          sendCount,
@@ -396,9 +411,11 @@ class MpiCommunicator 
#ifdef HAVE_MPI

      static MPI_Request NullRequest;

      static MPI_Comm AllGroup;

      static MPI_Comm NullGroup;

#else

      static int NullRequest;

      static int AllGroup;

      static int NullGroup;

#endif

    private :

      static std::streambuf *psbuf;
@@ -454,9 +471,11 @@ class MpiCommunicator 
#ifdef HAVE_MPI

MPI_Request MpiCommunicator::NullRequest;

MPI_Comm MpiCommunicator::AllGroup;

MPI_Comm MpiCommunicator::NullGroup;

#else

int MpiCommunicator::NullRequest;

int MpiCommunicator::AllGroup;

int MpiCommunicator::NullGroup;

#endif

std::streambuf *MpiCommunicator::psbuf;

std::streambuf *MpiCommunicator::backup;
@@ -540,6 +559,6 @@ for( int j = 0; j < TNL::Communicators::MpiCommunicator::GetSize( TNL::Communica 
                   << TNL::Communicators::MpiCommunicator::GetSize( TNL::Communicators::MpiCommunicator::AllGroup )      \

                   << " : " << message << std::endl;                                                                     \

      }                                                                                                                  \

      TNL::Communicators::MpiCommunicator::Barrier( Communicator::AllGroup );                                            \

      TNL::Communicators::MpiCommunicator::Barrier( TNL::Communicators::MpiCommunicator::AllGroup );                     \

   }

src/TNL/Communicators/MpiDefs.h

+15 −2
Original line number Diff line number Diff line
@@ -11,5 +11,18 @@ 
#pragma once



#ifndef HAVE_MPI

enum MPI_Op { MPI_SUM, MPI_MAX };

enum MPI_Op {

   MPI_MAX,

   MPI_MIN,

   MPI_SUM,

   MPI_PROD,

   MPI_LAND,

   MPI_BAND,

   MPI_LOR,

   MPI_BOR,

   MPI_LXOR,

   MPI_BXOR,

   MPI_MINLOC,

   MPI_MAXLOC,

};

#endif