Merge branch 'mpi-explosive' of geraldine.fjfi.cvut.cz:/local/projects/tnl/tnl into mpi-explosive (7481e64f) · Commits · TNL / tnl-dev

examples/heat-equation/tnl-heat-equation.h

+4 −1

Original line number	Diff line number	Diff line
		@@ -49,6 +49,9 @@ class heatEquationConfig
		config.addEntry< double >( "boundary-conditions-constant", "This sets a value in case of the constant boundary conditions." );
		config.addEntry< double >( "right-hand-side-constant", "This sets a constant value for the right-hand side.", 0.0 );
		config.addEntry< String >( "initial-condition", "File with the initial condition.", "initial.tnl");
		config.addEntry< String >( "distributed-grid-io-type", "Choose Distributed Grid IO Type", "LocalCopy");
		config.addEntryEnum< String >( "LocalCopy" );
		config.addEntryEnum< String >( "MpiIO" );
		};
		};

src/TNL/Communicators/MpiCommunicator.h

+58 −1

Original line number	Diff line number	Diff line
		@@ -12,9 +12,20 @@

		#include <iostream>
		#include <fstream>
		#include <cstring>

		#ifdef HAVE_MPI
		#include <mpi.h>

		#ifdef HAVE_CUDA
		#include <TNL/Devices/Cuda.h>

		typedef struct __attribute__((__packed__)) {
		char name[MPI_MAX_PROCESSOR_NAME];
		} procName;

		#endif

		#endif

		#include <TNL/String.h>
		@@ -23,6 +34,8 @@
		#include <TNL/Config/ConfigDescription.h>
		#include <TNL/Exceptions/MPISupportMissing.h>



		namespace TNL {
		namespace Communicators {

		@@ -79,6 +92,8 @@ class MpiCommunicator
		MPI::Init( argc, argv );
		NullRequest=MPI::REQUEST_NULL;
		redirect = true;

		selectGPU();
		#endif
		}

		@@ -274,13 +289,55 @@ class MpiCommunicator
		#else
		static int NullRequest;
		#endif

		private :
		static std::streambuf *psbuf;
		static std::streambuf *backup;
		static std::ofstream filestr;
		static bool redirect;
		static bool inited;

		static void selectGPU(void)
		{
		#ifdef HAVE_MPI
		#ifdef HAVE_CUDA
		int count,rank, gpuCount, gpuNumber;
		MPI_Comm_size(MPI_COMM_WORLD,&count);
		MPI_Comm_rank(MPI_COMM_WORLD,&rank);

		cudaGetDeviceCount(&gpuCount);

		procName names[count];

		int i=0;
		int len;
		MPI_Get_processor_name(names[rank].name, &len);

		for(i=0;i<count;i++)
		std::memcpy(names[i].name,names[rank].name,len+1);

		MPI_Alltoall( (void*)names ,MPI_MAX_PROCESSOR_NAME,MPI_CHAR,
		(void*)names,MPI_MAX_PROCESSOR_NAME,MPI_CHAR,
		MPI_COMM_WORLD);

		int nodeRank=0;
		for(i=0;i<rank;i++)
		{
		if(std::strcmp(names[rank].name,names[i].name)==0)
		nodeRank++;
		}

		gpuNumber=nodeRank % gpuCount;

		cudaSetDevice(gpuNumber);
		TNL_CHECK_CUDA_DEVICE;

		//std::cout<<"Node: " << rank << " gpu: " << gpuNumber << std::endl;

		#endif
		#endif
		}


		};

		#ifdef HAVE_MPI

src/TNL/Meshes/DistributedMeshes/DistributedGridIO.h

+46 −23

Original line number	Diff line number	Diff line
		@@ -12,12 +12,6 @@

		#include <iostream>

		#ifdef HAVE_MPI
		#ifdef MPIIO
		#include <mpi.h>
		#endif
		#endif

		#include <TNL/File.h>
		#include <TNL/Communicators/MpiCommunicator.h>
		#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
		@@ -164,7 +158,6 @@ class DistributedGridIO<MeshFunctionType,LocalCopy,Device>
		*/

		#ifdef HAVE_MPI
		#ifdef MPIIO
		template<typename MeshFunctionType>
		class DistributedGridIO_MPIIOBase
		{
		@@ -404,21 +397,33 @@ class DistributedGridIO_MPIIOBase
		};

		};
		#endif

		template<typename MeshFunctionType>
		class DistributedGridIO<MeshFunctionType,MpiIO,TNL::Devices::Cuda>
		{
		public:
		static bool save(const String& fileName, MeshFunctionType &meshFunction)
		{
		#ifdef HAVE_MPI
		if(Communicators::MpiCommunicator::IsInitialized())//i.e. - isUsed
		{
		using HostVectorType = Containers::Vector<typename MeshFunctionType::RealType, Devices::Host, typename MeshFunctionType::IndexType >;
		HostVectorType hostVector;
		hostVector=meshFunction.getData();
		typename MeshFunctionType::RealType * data=hostVector.getData();
		return DistributedGridIO_MPIIOBase<MeshFunctionType>::save(fileName,meshFunction,data);
		}
		#endif
		std::cout << "MPIIO can be used only with MPICommunicator." << std::endl;
		return false;

		};

		static bool load(const String& fileName,MeshFunctionType &meshFunction)
		{
		#ifdef HAVE_MPI
		if(Communicators::MpiCommunicator::IsInitialized())//i.e. - isUsed
		{
		using HostVectorType = Containers::Vector<typename MeshFunctionType::RealType, Devices::Host, typename MeshFunctionType::IndexType >;
		HostVectorType hostVector;
		@@ -427,6 +432,10 @@ class DistributedGridIO<MeshFunctionType,MpiIO,TNL::Devices::Cuda>
		DistributedGridIO_MPIIOBase<MeshFunctionType>::load(fileName,meshFunction,data);
		meshFunction.getData()=hostVector;
		return true;
		}
		#endif
		std::cout << "MPIIO can be used only with MPICommunicator." << std::endl;
		return false;
		};

		};
		@@ -436,21 +445,35 @@ class DistributedGridIO<MeshFunctionType,MpiIO,TNL::Devices::Host>
		{
		public:
		static bool save(const String& fileName, MeshFunctionType &meshFunction)
		{
		#ifdef HAVE_MPI
		if(Communicators::MpiCommunicator::IsInitialized())//i.e. - isUsed
		{
		typename MeshFunctionType::RealType * data=meshFunction.getData().getData();
		return DistributedGridIO_MPIIOBase<MeshFunctionType>::save(fileName,meshFunction,data);
		}
		#endif
		std::cout << "MPIIO can be used only with MPICommunicator." << std::endl;
		return false;

		};

		static bool load(const String& fileName,MeshFunctionType &meshFunction)
		{
		#ifdef HAVE_MPI
		if(Communicators::MpiCommunicator::IsInitialized())//i.e. - isUsed
		{
		double * data=meshFunction.getData().getData();
		return DistributedGridIO_MPIIOBase<MeshFunctionType>::load(fileName,meshFunction,data);
		}
		#endif
		std::cout << "MPIIO can be used only with MPICommunicator." << std::endl;
		return false;
		};

		};

		#endif
		#endif

		}
		}
		}

src/TNL/Problems/HeatEquationProblem.h

+5 −0

Original line number	Diff line number	Diff line
		@@ -25,6 +25,8 @@
		#include <TNL/Solvers/PDE/LinearSystemAssembler.h>
		#include <TNL/Solvers/PDE/BackwardTimeDiscretisation.h>

		#include <TNL/Meshes/DistributedMeshes/DistributedGridIO.h>

		namespace TNL {
		namespace Problems {

		@@ -133,6 +135,9 @@ class HeatEquationProblem : public PDEProblem< Mesh,
		RightHandSide,
		Solvers::PDE::BackwardTimeDiscretisation,
		DofVectorType > systemAssembler;

		Meshes::DistributedMeshes::DistrGridIOTypes distributedIOType;

		};

		} // namespace Problems

src/TNL/Problems/HeatEquationProblem_impl.h

+17 −8

Original line number	Diff line number	Diff line
		@@ -24,10 +24,6 @@

		#include "HeatEquationProblem.h"

		//#define MPIIO
		#include <TNL/Meshes/DistributedMeshes/DistributedGridIO.h>


		namespace TNL {
		namespace Problems {

		@@ -99,6 +95,13 @@ setup( const MeshPointer& meshPointer,
		std::cerr << "I was not able to initialize the right-hand side function." << std::endl;
		return false;
		}

		String param=parameters.getParameter< String >( "distributed-grid-io-type" );
		if(param=="MpiIO")
		distributedIOType=Meshes::DistributedMeshes::MpiIO;
		if(param=="LocalCopy")
		distributedIOType=Meshes::DistributedMeshes::LocalCopy;

		return true;
		}

		@@ -149,6 +152,9 @@ setInitialCondition( const Config::ParameterContainer& parameters,
		if(CommunicatorType::isDistributed())
		{
		std::cout<<"Nodes Distribution: " << uPointer->getMesh().getDistributedMesh()->printProcessDistr() << std::endl;
		if(distributedIOType==Meshes::DistributedMeshes::MpiIO)
		Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType,Meshes::DistributedMeshes::MpiIO> ::load(initialConditionFile, *uPointer );
		if(distributedIOType==Meshes::DistributedMeshes::LocalCopy)
		Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType,Meshes::DistributedMeshes::LocalCopy> ::load(initialConditionFile, *uPointer );
		uPointer->template synchronize<CommunicatorType>();
		}
		@@ -214,6 +220,9 @@ makeSnapshot( const RealType& time,

		if(CommunicatorType::isDistributed())
		{
		if(distributedIOType==Meshes::DistributedMeshes::MpiIO)
		Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType,Meshes::DistributedMeshes::MpiIO> ::save(fileName.getFileName(), *uPointer );
		if(distributedIOType==Meshes::DistributedMeshes::LocalCopy)
		Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType,Meshes::DistributedMeshes::LocalCopy> ::save(fileName.getFileName(), *uPointer );
		}
		else