Merge branch 'mpi-new' into MPI-explosive (4abeee6c) · Commits · TNL / tnl-dev

src/TNL/Communicators/MpiCommunicator.h

+28 −26

Original line number	Diff line number	Diff line
		@@ -21,45 +21,45 @@ namespace Communicators {
		{

		private:
		inline MPI_Datatype MPIDataType( const signed char ) { return MPI_CHAR; };
		inline MPI_Datatype MPIDataType( const signed short int ) { return MPI_SHORT; };
		inline MPI_Datatype MPIDataType( const signed int ) { return MPI_INT; };
		inline MPI_Datatype MPIDataType( const signed long int ) { return MPI_LONG; };
		inline MPI_Datatype MPIDataType( const unsigned char ) { return MPI_UNSIGNED_CHAR; };
		inline MPI_Datatype MPIDataType( const unsigned short int ) { return MPI_UNSIGNED_SHORT; };
		inline MPI_Datatype MPIDataType( const unsigned int ) { return MPI_UNSIGNED; };
		inline MPI_Datatype MPIDataType( const unsigned long int ) { return MPI_UNSIGNED_LONG; };
		inline MPI_Datatype MPIDataType( const float ) { return MPI_FLOAT; };
		inline MPI_Datatype MPIDataType( const double ) { return MPI_DOUBLE; };
		inline MPI_Datatype MPIDataType( const long double ) { return MPI_LONG_DOUBLE; };
		inline static MPI_Datatype MPIDataType( const signed char ) { return MPI_CHAR; };
		inline static MPI_Datatype MPIDataType( const signed short int ) { return MPI_SHORT; };
		inline static MPI_Datatype MPIDataType( const signed int ) { return MPI_INT; };
		inline static MPI_Datatype MPIDataType( const signed long int ) { return MPI_LONG; };
		inline static MPI_Datatype MPIDataType( const unsigned char ) { return MPI_UNSIGNED_CHAR; };
		inline static MPI_Datatype MPIDataType( const unsigned short int ) { return MPI_UNSIGNED_SHORT; };
		inline static MPI_Datatype MPIDataType( const unsigned int ) { return MPI_UNSIGNED; };
		inline static MPI_Datatype MPIDataType( const unsigned long int ) { return MPI_UNSIGNED_LONG; };
		inline static MPI_Datatype MPIDataType( const float ) { return MPI_FLOAT; };
		inline static MPI_Datatype MPIDataType( const double ) { return MPI_DOUBLE; };
		inline static MPI_Datatype MPIDataType( const long double ) { return MPI_LONG_DOUBLE; };

		public:

		typedef MPI::Request Request;
		MPI::Request NullRequest;
		static MPI::Request NullRequest;

		void Init(int argc, char **argv)
		static void Init(int argc, char **argv)
		{
		MPI::Init(argc,argv);
		NullRequest=MPI::REQUEST_NULL;
		};

		void Finalize()
		static void Finalize()
		{
		MPI::Finalize();
		};

		bool IsInitialized()
		static bool IsInitialized()
		{
		return MPI::Is_initialized();
		};

		int GetRank()
		static int GetRank()
		{
		return MPI::COMM_WORLD.Get_rank();
		};

		int GetSize()
		static int GetSize()
		{
		return MPI::COMM_WORLD.Get_size();
		};
		@@ -67,41 +67,41 @@ namespace Communicators {
		//dim-number of dimesions, distr array of guess distr - 0 for computation
		//distr array will be filled by computed distribution
		//more information in MPI documentation
		void DimsCreate(int nproc, int dim, int *distr)
		static void DimsCreate(int nproc, int dim, int *distr)
		{
		MPI_Dims_create(nproc, dim, distr);
		};

		void Barrier()
		static void Barrier()
		{
		MPI::COMM_WORLD.Barrier();
		};

		template <typename T>
		Request ISend( const T *data, int count, int dest)
		static Request ISend( const T *data, int count, int dest)
		{
		return MPI::COMM_WORLD.Isend((void) data, count, MPIDataType(data) , dest, 0);
		};

		template <typename T>
		Request IRecv( const T *data, int count, int src)
		static Request IRecv( const T *data, int count, int src)
		{
		return MPI::COMM_WORLD.Irecv((void) data, count, MPIDataType(data) , src, 0);
		};

		void WaitAll(Request *reqs, int length)
		static void WaitAll(Request *reqs, int length)
		{
		MPI::Request::Waitall(length, reqs);
		};

		template< typename T >
		void Bcast( T& data, int count, int root)
		static void Bcast( T& data, int count, int root)
		{
		MPI::COMM_WORLD.Bcast((void*) &data, count, MPIDataType(data), root);
		};

		/* template< typename T >
		void Allreduce( T& data,
		static void Allreduce( T& data,
		T& reduced_data,
		int count,
		const MPI_Op &op)
		@@ -110,7 +110,7 @@ namespace Communicators {
		};

		template< typename T >
		void Reduce( T& data,
		static void Reduce( T& data,
		T& reduced_data,
		int count,
		MPI_Op &op,
		@@ -120,6 +120,8 @@ namespace Communicators {
		};*/
		};

		MPI::Request MpiCommunicator::NullRequest;

		}//namespace Communicators
		} // namespace TNL

src/TNL/Communicators/NoDistrCommunicator.h

+17 −14

Original line number	Diff line number	Diff line
		@@ -20,32 +20,33 @@ namespace Communicators {
		public:

		typedef int Request;
		Request NullRequest;
		static Request NullRequest;

		void Init(int argc, char **argv)
		static void Init(int argc, char **argv)
		{
		NullRequest=-1;
		};

		void Finalize()
		static void Finalize()
		{
		};

		bool IsInitialized()
		static bool IsInitialized()
		{
		return true;
		};

		int GetRank()
		static int GetRank()
		{
		return 0;
		};

		int GetSize()
		static int GetSize()
		{
		return 1;
		};

		void DimsCreate(int nproc, int dim, int *distr)
		static void DimsCreate(int nproc, int dim, int *distr)
		{
		for(int i=0;i<dim;i++)
		{
		@@ -53,33 +54,33 @@ namespace Communicators {
		}
		};

		void Barrier()
		static void Barrier()
		{
		};

		template <typename T>
		Request ISend( const T *data, int count, int dest)
		static Request ISend( const T *data, int count, int dest)
		{
		return 1;
		};

		template <typename T>
		Request IRecv( const T *data, int count, int src)
		static Request IRecv( const T *data, int count, int src)
		{
		return 1;
		};

		void WaitAll(Request *reqs, int length)
		static void WaitAll(Request *reqs, int length)
		{
		};

		template< typename T >
		void Bcast( T& data, int count, int root)
		static void Bcast( T& data, int count, int root)
		{
		};

		/* template< typename T >
		void Allreduce( T& data,
		static void Allreduce( T& data,
		T& reduced_data,
		int count,
		const MPI_Op &op)
		@@ -88,7 +89,7 @@ namespace Communicators {
		};

		template< typename T >
		void Reduce( T& data,
		static void Reduce( T& data,
		T& reduced_data,
		int count,
		MPI_Op &op,
		@@ -98,6 +99,8 @@ namespace Communicators {
		};*/
		};

		int NoDistrCommunicator::NullRequest;

		} // namespace Communicators
		} // namespace TNL

src/TNL/Functions/MeshFunction.h

+2 −2

Original line number	Diff line number	Diff line
		@@ -160,8 +160,8 @@ class MeshFunction :

		using Object::boundLoad;

		template< typename Communicator>
		void Synchronize(Communicator &comm);
		template< typename CommunicatorType>
		void Synchronize();


		protected:

src/TNL/Functions/MeshFunction_impl.h

+3 −3

Original line number	Diff line number	Diff line
		@@ -543,15 +543,15 @@ write( const String& fileName,
		template< typename Mesh,
		int MeshEntityDimension,
		typename Real >
		template< typename Communicator>
		template< typename CommunicatorType>
		void
		MeshFunction< Mesh, MeshEntityDimension, Real >::
		Synchronize(Communicator &comm)
		Synchronize()
		{
		auto distrMesh = this->getMesh().GetDistMesh();
		if(distrMesh != NULL && distrMesh->IsDistributed())
		{
		this->synchronizer.Synchronize(comm,*this);
		this->synchronizer.Synchronize<CommunicatorType>(*this);
		}
		}

src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h

+28 −28

Original line number	Diff line number	Diff line
		@@ -96,8 +96,8 @@ private:
		};
		};

		template<typename Communicator>
		void Synchronize(Communicator &comm, MeshFunctionType &meshfunction)
		template<typename CommunicatorType>
		void Synchronize(MeshFunctionType &meshfunction)
		{
		TNL_ASSERT_TRUE(isSet,"Synchronizer is not set, but used to Synchronize");

		@@ -129,33 +129,33 @@ private:
		}

		//async send
		typename Communicator::Request req[4];
		typename CommunicatorType::Request req[4];

		//send everithing, recieve everything
		if(left!=-1)
		{
		req[0]=comm.ISend(leftsendbuf, size, left);
		req[2]=comm.IRecv(leftrcvbuf, size, left);
		req[0]=CommunicatorType::ISend(leftsendbuf, size, left);
		req[2]=CommunicatorType::IRecv(leftrcvbuf, size, left);
		}
		else
		{
		req[0]=comm.NullRequest;
		req[2]=comm.NullRequest;
		req[0]=CommunicatorType::NullRequest;
		req[2]=CommunicatorType::NullRequest;
		}

		if(right!=-1)
		{
		req[1]=comm.ISend(rightsendbuf, size, right);
		req[3]=comm.IRecv(rightrcvbuf, size, right);
		req[1]=CommunicatorType::ISend(rightsendbuf, size, right);
		req[3]=CommunicatorType::IRecv(rightrcvbuf, size, right);
		}
		else
		{
		req[1]=comm.NullRequest;
		req[3]=comm.NullRequest;
		req[1]=CommunicatorType::NullRequest;
		req[3]=CommunicatorType::NullRequest;
		}

		//wait until send and recv is done
		comm.WaitAll(req, 4);
		CommunicatorType::WaitAll(req, 4);

		//copy data form rcv buffers
		if(left!=-1)
		@@ -297,8 +297,8 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 2, GridReal, D
		DeleteBuffers();
		}

		template<typename Communicator>
		void Synchronize(Communicator &comm, MeshFunctionType &meshfunction)
		template<typename CommunicatorType>
		void Synchronize( MeshFunctionType &meshfunction)
		{

		TNL_ASSERT_TRUE(isSet,"Synchronizer is not set, but used to Synchronize");
		@@ -315,23 +315,23 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 2, GridReal, D
		neighbor);

		//async send and rcv
		typename Communicator::Request req[16];
		typename CommunicatorType::Request req[16];

		//send everithing, recieve everything
		for(int i=0;i<8;i++)
		if(neighbor[i]!=-1)
		{
		req[i]=comm.ISend(sendbuffs[i], sizes[i], neighbor[i]);
		req[8+i]=comm.IRecv(rcvbuffs[i], sizes[i], neighbor[i]);
		req[i]=CommunicatorType::ISend(sendbuffs[i], sizes[i], neighbor[i]);
		req[8+i]=CommunicatorType::IRecv(rcvbuffs[i], sizes[i], neighbor[i]);
		}
		else
		{
		req[i]=comm.NullRequest;
		req[8+i]=comm.NullRequest;
		req[i]=CommunicatorType::NullRequest;
		req[8+i]=CommunicatorType::NullRequest;
		}

		//wait until send is done
		comm.WaitAll(req,16);
		CommunicatorType::WaitAll(req,16);

		//copy data form rcv buffers
		CopyBuffers(meshfunction, rcvbuffs, false,
		@@ -515,8 +515,8 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 3, GridReal, D
		}
		}

		template<typename Communicator>
		void Synchronize(Communicator &comm,MeshFunctionType &meshfunction)
		template<typename CommunicatorType>
		void Synchronize(MeshFunctionType &meshfunction)
		{

		TNL_ASSERT_TRUE(isSet,"Synchronizer is not set, but used to Synchronize");
		@@ -534,23 +534,23 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 3, GridReal, D
		neighbor);

		//async send and rcv
		typename Communicator::Request req[52];
		typename CommunicatorType::Request req[52];

		//send everithing, recieve everything
		for(int i=0;i<26;i++)
		if(neighbor[i]!=-1)
		{
		req[i]=comm.ISend(sendbuffs[i], sizes[i], neighbor[i]);
		req[26+i]=comm.IRecv(rcvbuffs[i], sizes[i], neighbor[i]);
		req[i]=CommunicatorType::ISend(sendbuffs[i], sizes[i], neighbor[i]);
		req[26+i]=CommunicatorType::IRecv(rcvbuffs[i], sizes[i], neighbor[i]);
		}
		else
		{
		req[i]=comm.NullRequest;
		req[26+i]=comm.NullRequest;
		req[i]=CommunicatorType::NullRequest;
		req[26+i]=CommunicatorType::NullRequest;
		}

		//wait until send is done
		comm.WaitAll(req,52);
		CommunicatorType::WaitAll(req,52);

		//copy data form rcv buffers
		CopyBuffers(meshfunction, rcvbuffs, false,