Commit 12fbb34d authored by Vít Hanousek's avatar Vít Hanousek
Browse files

Merge branch 'mpi-new' into MPI-explosive 

Conflicts:
	src/TNL/Communicators/MpiCommunicator.h
	src/UnitTests/Mpi/DistributedGridIOTest.cpp
parents ee06e044 c6d405cf

CMakeLists.txt

+7 −3
Original line number Diff line number Diff line
@@ -92,15 +92,19 @@ endif() 
if( ${CXX_COMPILER_NAME} STREQUAL "mpic++" )

   message( "MPI compiler detected."    )

   set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_MPI" )

   set( CUDA_HOST_COMPILER "mpic++" )



endif()



####

# Check for MPI

# Check for MPI -- not working

#

#find_package( MPI )

#if( MPI_CXX_FOUND )

   # set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_MPI" )

   # message( "MPI headers found -- ${MPI_CXX_INCLUDE_PATH}")

   # message( "MPI link flags  -- ${MPI_CXX_LINK_FLAGS}")

   # message( "MPI libraries-- ${MPI_CXX_LIBRARIES}")

#endif()



#####

src/TNL/Communicators/MpiCommunicator.h

+17 −18
Original line number Diff line number Diff line
@@ -26,17 +26,17 @@ namespace Communicators { 
    {



        private:

        inline static MPI_Datatype MPIDataType( const signed char ) { return MPI_CHAR; };

        inline static MPI_Datatype MPIDataType( const signed short int ) { return MPI_SHORT; };

        inline static MPI_Datatype MPIDataType( const signed int ) { return MPI_INT; };

        inline static MPI_Datatype MPIDataType( const signed long int ) { return MPI_LONG; };

        inline static MPI_Datatype MPIDataType( const unsigned char ) { return MPI_UNSIGNED_CHAR; };

        inline static MPI_Datatype MPIDataType( const unsigned short int ) { return MPI_UNSIGNED_SHORT; };

        inline static MPI_Datatype MPIDataType( const unsigned int ) { return MPI_UNSIGNED; };

        inline static MPI_Datatype MPIDataType( const unsigned long int ) { return MPI_UNSIGNED_LONG; };

        inline static MPI_Datatype MPIDataType( const float ) { return MPI_FLOAT; };

        inline static MPI_Datatype MPIDataType( const double ) { return MPI_DOUBLE; };

        inline static MPI_Datatype MPIDataType( const long double ) { return MPI_LONG_DOUBLE; };

        inline static MPI_Datatype MPIDataType( const signed char* ) { return MPI_CHAR; };

        inline static MPI_Datatype MPIDataType( const signed short int* ) { return MPI_SHORT; };

        inline static MPI_Datatype MPIDataType( const signed int* ) { return MPI_INT; };

        inline static MPI_Datatype MPIDataType( const signed long int* ) { return MPI_LONG; };

        inline static MPI_Datatype MPIDataType( const unsigned char *) { return MPI_UNSIGNED_CHAR; };

        inline static MPI_Datatype MPIDataType( const unsigned short int* ) { return MPI_UNSIGNED_SHORT; };

        inline static MPI_Datatype MPIDataType( const unsigned int* ) { return MPI_UNSIGNED; };

        inline static MPI_Datatype MPIDataType( const unsigned long int* ) { return MPI_UNSIGNED_LONG; };

        inline static MPI_Datatype MPIDataType( const float* ) { return MPI_FLOAT; };

        inline static MPI_Datatype MPIDataType( const double* ) { return MPI_DOUBLE; };

        inline static MPI_Datatype MPIDataType( const long double* ) { return MPI_LONG_DOUBLE; };

        

        public:
@@ -79,7 +79,6 @@ namespace Communicators { 
                    std::cout.rdbuf(psbuf);

                }

            }



        };



        static void Finalize()
@@ -126,14 +125,14 @@ namespace Communicators { 
        template <typename T>

        static Request ISend( const T *data, int count, int dest)

        {

                return MPI::COMM_WORLD.Isend((void*) data, count, MPIDataType(*data) , dest, 0);

        };    

                return MPI::COMM_WORLD.Isend((void*) data, count, MPIDataType(data) , dest, 0);

        }    



        template <typename T>

        static Request IRecv( const T *data, int count, int src)

        {

                return MPI::COMM_WORLD.Irecv((void*) data, count, MPIDataType(*data) , src, 0);

        };

                return MPI::COMM_WORLD.Irecv((void*) data, count, MPIDataType(data) , src, 0);

        }



        static void WaitAll(Request *reqs, int length)

        {
@@ -144,7 +143,7 @@ namespace Communicators { 
        static void Bcast(  T& data, int count, int root)

        {

                MPI::COMM_WORLD.Bcast((void*) &data, count,  MPIDataType(data), root);

        };

        }



      /*  template< typename T >

        static void Allreduce( T& data,

src/TNL/Containers/Array_impl.h

+1 −0
Original line number Diff line number Diff line
@@ -547,6 +547,7 @@ boundLoad( File& file ) 
   return true;

}





template< typename Element,

          typename Device,

          typename Index >

src/TNL/Meshes/DistributedMeshes/BufferEntitiesHelper.h

0 → 100644
+125 −0
Original line number Diff line number Diff line 
/***************************************************************************

                          BufferEntittiesHelper.h  -  description

                             -------------------

    begin                : March 1, 2018

    copyright            : (C) 2018 by Tomas Oberhuber

    email                : tomas.oberhuber@fjfi.cvut.cz

 ***************************************************************************/



/* See Copyright Notice in tnl/Copyright */



#pragma once



#include <TNL/Devices/Host.h>

#include <TNL/Devices/Cuda.h>

#include <TNL/ParallelFor.h>



namespace TNL {

namespace Meshes { 

namespace DistributedMeshes { 





template < typename MeshFunctionType,

           int dim,

           typename RealType=typename MeshFunctionType::MeshType::RealType,

           typename Device=typename MeshFunctionType::MeshType::DeviceType,

           typename Index=typename MeshFunctionType::MeshType::GlobalIndexType >

class BufferEntitiesHelper

{

};



//======================================== 1D ====================================================



//host

template < typename MeshFunctionType, typename RealType, typename Device, typename Index >

class BufferEntitiesHelper<MeshFunctionType,1,RealType,Device,Index>

{

    public:

    static void BufferEntities(MeshFunctionType meshFunction, RealType * buffer, Index beginx, Index sizex, bool tobuffer)

    {

        auto mesh = meshFunction.getMesh();

        RealType* meshFunctionData = meshFunction.getData().getData();

        auto kernel = [tobuffer, mesh, buffer, meshFunctionData, beginx] __cuda_callable__ ( Index j )

        {

            typename MeshFunctionType::MeshType::Cell entity(mesh);

            entity.getCoordinates().x()=beginx+j;

            entity.refresh();

            if(tobuffer)

                buffer[j]=meshFunctionData[entity.getIndex()];

            else

                meshFunctionData[entity.getIndex()]=buffer[j];

        };

        ParallelFor< Device >::exec( 0, sizex, kernel );

    };  

};





//======================================== 2D ====================================================

template <typename MeshFunctionType, typename RealType, typename Device, typename Index  > 

class BufferEntitiesHelper<MeshFunctionType,2,RealType,Device,Index>

{

    public:

    static void BufferEntities(MeshFunctionType meshFunction, RealType * buffer, Index beginx, Index beginy, Index sizex, Index sizey,bool tobuffer)

    {

        auto mesh=meshFunction.getMesh();

        RealType *meshFunctionData=meshFunction.getData().getData();

        auto kernel = [tobuffer, mesh, buffer, meshFunctionData, beginx, sizex, beginy] __cuda_callable__ ( Index i, Index j )

        {

            typename MeshFunctionType::MeshType::Cell entity(mesh);

            entity.getCoordinates().x()=beginx+j;

            entity.getCoordinates().y()=beginy+i;				

            entity.refresh();

            if(tobuffer)

                    buffer[i*sizex+j]=meshFunctionData[entity.getIndex()];

            else

                    meshFunctionData[entity.getIndex()]=buffer[i*sizex+j];

        };

        

        ParallelFor2D< Device >::exec( 0, 0, sizey, sizex, kernel );       

        

    };

};





//======================================== 3D ====================================================

template <typename MeshFunctionType, typename RealType, typename Device, typename Index >

class BufferEntitiesHelper<MeshFunctionType,3,RealType,Device,Index>

{

    public:

    static void BufferEntities(MeshFunctionType meshFunction, RealType * buffer, Index beginx, Index beginy, Index beginz, Index sizex, Index sizey, Index sizez, bool tobuffer)

    {



        auto mesh=meshFunction.getMesh();

        RealType * meshFunctionData=meshFunction.getData().getData();

        auto kernel = [tobuffer, mesh, buffer, meshFunctionData, beginx, sizex, beginy, sizey, beginz] __cuda_callable__ ( Index k, Index i, Index j )

        {

            typename MeshFunctionType::MeshType::Cell entity(mesh);

            entity.getCoordinates().x()=beginx+j;

            entity.getCoordinates().z()=beginz+k;

            entity.getCoordinates().y()=beginy+i;

            entity.refresh();

            if(tobuffer)

                    buffer[k*sizex*sizey+i*sizex+j]=meshFunctionData[entity.getIndex()];

            else

                    meshFunctionData[entity.getIndex()]=buffer[k*sizex*sizey+i*sizex+j];

        };



        ParallelFor3D< Device >::exec( 0, 0, 0, sizez, sizey, sizex, kernel ); 



        /*for(int k=0;k<sizez;k++)

        {

            for(int i=0;i<sizey;i++)

            {

                for(int j=0;j<sizex;j++)

                {

                        kernel(k,i,j);

                }

            }

        }*/

    };

};





} // namespace DistributedMeshes

} // namespace Meshes

} // namespace TNL

src/TNL/Meshes/DistributedMeshes/CopyEntitiesHelper.h

0 → 100644
+136 −0
Original line number Diff line number Diff line 
/***************************************************************************

                          CopyEntitiesHelper.h  -  description

                             -------------------

    begin                : March 8, 2018

    copyright            : (C) 2018 by Tomas Oberhuber

    email                : tomas.oberhuber@fjfi.cvut.cz

 ***************************************************************************/



/* See Copyright Notice in tnl/Copyright */



#pragma once



#include <TNL/Devices/Host.h>

#include <TNL/Devices/Cuda.h>

#include <TNL/ParallelFor.h>



namespace TNL {

namespace Meshes { 

namespace DistributedMeshes { 



template<typename MeshFunctionType,

         int dim=MeshFunctionType::getMeshDimension()>

class CopyEntitiesHelper

{

    public:

    typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;

    static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)

    {

    }



};





template<typename MeshFunctionType>

class CopyEntitiesHelper<MeshFunctionType, 1>

{

    public:

    typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;

    typedef typename MeshFunctionType::MeshType::Cell Cell;

    typedef typename MeshFunctionType::MeshType::GlobalIndexType Index;



    static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)

    {        

        auto toData=to.getData().getData();

        auto fromData=from.getData().getData();

        auto fromMesh=from.getMesh();

        auto toMesh=to.getMesh();

        auto kernel = [fromData,toData, fromMesh, toMesh, fromBegin, toBegin] __cuda_callable__ ( Index i )

        {

            Cell fromEntity(fromMesh);

            Cell toEntity(toMesh);

            toEntity.getCoordinates().x()=toBegin.x()+i;            

            toEntity.refresh();

            fromEntity.getCoordinates().x()=fromBegin.x()+i;            

            fromEntity.refresh();

            toData[toEntity.getIndex()]=fromData[fromEntity.getIndex()];

        };

        ParallelFor< typename MeshFunctionType::MeshType::DeviceType >::exec( (Index)0, (Index)size.x(), kernel );



    }



};





template<typename MeshFunctionType>



class CopyEntitiesHelper<MeshFunctionType,2>

{

    public:

    typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;

    typedef typename MeshFunctionType::MeshType::Cell Cell;

    typedef typename MeshFunctionType::MeshType::GlobalIndexType Index;



    static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)

    {

        auto toData=to.getData().getData();

        auto fromData=from.getData().getData();

        auto fromMesh=from.getMesh();

        auto toMesh=to.getMesh();

        auto kernel = [fromData,toData, fromMesh, toMesh, fromBegin, toBegin] __cuda_callable__ ( Index j, Index i )

        {

            Cell fromEntity(fromMesh);

            Cell toEntity(toMesh);

            toEntity.getCoordinates().x()=toBegin.x()+i;

            toEntity.getCoordinates().y()=toBegin.y()+j;            

            toEntity.refresh();

            fromEntity.getCoordinates().x()=fromBegin.x()+i;

            fromEntity.getCoordinates().y()=fromBegin.y()+j;            

            fromEntity.refresh();

            toData[toEntity.getIndex()]=fromData[fromEntity.getIndex()];

        };

        ParallelFor2D< typename MeshFunctionType::MeshType::DeviceType >::exec( (Index)0,(Index)0,(Index)size.y(), (Index)size.x(), kernel );



    }



};





template<typename MeshFunctionType>

class CopyEntitiesHelper<MeshFunctionType,3>

{

    public:

    typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;

    typedef typename MeshFunctionType::MeshType::Cell Cell;

    typedef typename MeshFunctionType::MeshType::GlobalIndexType Index;



    static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)

    {

        auto toData=to.getData().getData();

        auto fromData=from.getData().getData();

        auto fromMesh=from.getMesh();

        auto toMesh=to.getMesh();

        auto kernel = [fromData,toData, fromMesh, toMesh, fromBegin, toBegin] __cuda_callable__ ( Index k, Index j, Index i )

        {

            Cell fromEntity(fromMesh);

            Cell toEntity(toMesh);

            toEntity.getCoordinates().x()=toBegin.x()+i;

            toEntity.getCoordinates().y()=toBegin.y()+j;

            toEntity.getCoordinates().z()=toBegin.z()+k;                                

            toEntity.refresh();

            fromEntity.getCoordinates().x()=fromBegin.x()+i;

            fromEntity.getCoordinates().y()=fromBegin.y()+j;

            fromEntity.getCoordinates().z()=fromBegin.z()+k;            

            fromEntity.refresh();

            toData[toEntity.getIndex()]=fromData[fromEntity.getIndex()];

        };

        ParallelFor3D< typename MeshFunctionType::MeshType::DeviceType >::exec( (Index)0,(Index)0,(Index)0,(Index)size.z() ,(Index)size.y(), (Index)size.x(), kernel );

    }

};









} // namespace DistributedMeshes

} // namespace Meshes

} // namespace TNL