Distributed VectorField load with test. (24dcc35c) · Commits · TNL / tnl-dev

src/TNL/Meshes/DistributedMeshes/DistributedGridIO_MeshFunction.h

+2 −0

Original line number	Diff line number	Diff line
		@@ -340,6 +340,8 @@ class DistributedGridIO_MPIIOBase
		MPI_Datatype atype;
		int dataCount=CreateDataTypes(distrGrid,&ftype,&atype);

		MPI_File_set_view(file,0,MPI_BYTE,MPI_BYTE,"native",MPI_INFO_NULL);

		int headerSize=0;

		if(Communicators::MpiCommunicator::GetRank(group)==0)

src/TNL/Meshes/DistributedMeshes/DistributedGridIO_VectorField.h

+54 −40

Original line number	Diff line number	Diff line
		@@ -27,8 +27,6 @@ class DistributedGridIO<Functions::VectorField<Size,MeshFunctionType>,MpiIO,Devi
		static bool save(const String& fileName, Functions::VectorField<Size,MeshFunctionType> &vectorField)
		{
		#ifdef HAVE_MPI
		std::cout << "saving VCT filed" << endl;

		if(Communicators::MpiCommunicator::IsInitialized())//i.e. - isUsed
		{
		auto *distrGrid=vectorField.getMesh().getDistributedMesh();
		@@ -83,7 +81,7 @@ class DistributedGridIO<Functions::VectorField<Size,MeshFunctionType>,MpiIO,Devi
		MPI_Get_count(&wstatus,MPI_CHAR,&count);
		size+=count*sizeof(char);

		//Meshfunction type
		//VectorField type
		String vectorFieldSerializationType=vectorField.getSerializationTypeVirtual();
		int vectorFieldSerializationTypeLength=vectorFieldSerializationType.getLength();
		MPI_File_write(file,&vectorFieldSerializationTypeLength,1,MPI_INT,&wstatus);
		@@ -96,52 +94,68 @@ class DistributedGridIO<Functions::VectorField<Size,MeshFunctionType>,MpiIO,Devi
		return size;
		}

		/* static bool load(const String& fileName,MeshFunctionType &meshFunction)
		static unsigned int readVectorFieldHeader(MPI_File &file,Functions::VectorField<Size,MeshFunctionType> &vectorField)
		{
		MPI_Status rstatus;
		char buffer[255];
		int size=0;
		int count=0;

		MPI_File_read(file, (void *)buffer,5, MPI_CHAR, &rstatus);//MAGIC
		size+=5*sizeof(char);
		MPI_File_read(file, (void *)&count,1, MPI_INT, &rstatus);//SIZE OF DATATYPE
		size+=1*sizeof(int);
		MPI_File_read(file, (void *)&buffer,count, MPI_CHAR, &rstatus);//DATATYPE
		size+=count*sizeof(char);

		return size;
		}

		public:
		static bool load(const String& fileName, Functions::VectorField<Size,MeshFunctionType> &vectorField)
		{
		#ifdef HAVE_MPI
		if(Communicators::MpiCommunicator::IsInitialized())//i.e. - isUsed
		{
		using HostVectorType = Containers::Vector<typename MeshFunctionType::RealType, Devices::Host, typename MeshFunctionType::IndexType >;
		HostVectorType hostVector;
		hostVector.setLike(meshFunction.getData());
		double * data=hostVector.getData();
		DistributedGridIO_MPIIOBase<MeshFunctionType>::load(fileName,meshFunction,data);
		meshFunction.getData()=hostVector;
		return true;
		auto *distrGrid=vectorField.getMesh().getDistributedMesh();
		if(distrGrid==NULL)
		{
		return vectorField.save(fileName);
		}
		#endif
		std::cout << "MPIIO can be used only with MPICommunicator." << std::endl;
		return false;
		};

		#ifdef HAVE MPI
		private:
		static unsigned int writeVectorFieldHeader(MPI_File &file,Functions::VectorField<Size,MeshFunctionType> &vectorField )
		{
		unsigned int size=0;
		int count;
		MPI_Status wstatus;
		MPI_Comm group=((MPI_Comm)(distrGrid->getCommunicationGroup()));

		//Magic
		MPI_File_write( file, const_cast< void* >( ( const void* ) "TNLMN" ), 5, MPI_CHAR,&wstatus );
		MPI_Get_count(&wstatus,MPI_CHAR,&count);
		size+=count*sizeof(char);
		//write
		MPI_File file;
		MPI_File_open( group,
		const_cast< char* >( fileName.getString() ),
		MPI_MODE_RDONLY,
		MPI_INFO_NULL,
		&file);

		//Meshfunction type
		String vectorFieldSerializationType=vectorField.getSerializationTypeVirtual();
		int vectorFieldSerializationTypeLength=vectorFieldSerializationType.getLength();
		MPI_File_write(file,&vectorFieldSerializationTypeLength,1,MPI_INT,&wstatus);
		MPI_Get_count(&wstatus,MPI_INT,&count);
		size+=count*sizeof(int);
		MPI_File_write(file,vectorFieldSerializationType.getString(),vectorFieldSerializationType.getLength(),MPI_CHAR,&wstatus);
		MPI_Get_count(&wstatus,MPI_CHAR,&count);
		size+=count*sizeof(char);

		return size;
		int offset=0; //global offset -> every meshfunctoion creates it's own datatypes we need manage global offset
		if(Communicators::MpiCommunicator::GetRank(group)==0)
		offset+=readVectorFieldHeader(file,vectorField);
		MPI_Bcast(&offset, 1, MPI_INT,0, group);

		};
		for( int i = 0; i < vectorField.getVectorDimension(); i++ )
		{
		typename MeshFunctionType::RealType * data=vectorField[i]->getData().getData(); //here manage data transfer Device...
		int size = DistributedGridIO_MPIIOBase<MeshFunctionType>::load(file,*(vectorField[i]),data,offset);
		offset+=size;
		if( size==0 )
		return false;
		}

		MPI_File_close(&file);
		return true;
		}
		#endif
		*/
		std::cout << "MPIIO can be used only with MPICommunicator." << std::endl;
		return false;

		};

		};

src/UnitTests/Mpi/DistributedVectorFieldIO_MPIIOTest.cpp

+8 −8

Original line number	Diff line number	Diff line
		@@ -27,21 +27,21 @@ TEST( DistributedVectorFieldIO_MPIIO, Save_3D )
		TestDistributedVectorFieldMPIIO<3,2,Host>::TestSave();
		}

		/*
		TEST( DistributedGridIO, Load_1D )

		TEST( DistributedVectorFieldIO_MPIIO, Load_1D )
		{
		TestDistributedVectorFieldMPIIO<1,Host>::TestLoad();
		TestDistributedVectorFieldMPIIO<1,2,Host>::TestLoad();
		}

		TEST( DistributedGridIO, Load_2D )
		TEST( DistributedVectorFieldIO_MPIIO, Load_2D )
		{
		TestDistributedVectorFieldMPIIO<2,Host>::TestLoad();
		TestDistributedVectorFieldMPIIO<2,3,Host>::TestLoad();
		}

		TEST( DistributedGridIO, Load_2D )
		TEST( DistributedVectorFieldIO_MPIIO, Load_3D )
		{
		TestDistributedVectorFieldMPIIO<3,Host>::TestLoad();
		}*/
		TestDistributedVectorFieldMPIIO<3,2,Host>::TestLoad();
		}


		#else

src/UnitTests/Mpi/DistributedVectorFieldIO_MPIIOTestBase.h

+23 −17

Original line number	Diff line number	Diff line
		@@ -111,7 +111,7 @@ class TestDistributedVectorFieldMPIIO{
		}
		};

		/* static void TestLoad()
		static void TestLoad()
		{
		SharedPointer< LinearFunctionType, Device > linearFunctionPtr;
		MeshFunctionEvaluator< MeshFunctionType, LinearFunctionType > linearFunctionEvaluator;
		@@ -137,44 +137,50 @@ class TestDistributedVectorFieldMPIIO{
		//Prepare file
		if(CommunicatorType::GetRank(CommunicatorType::AllGroup)==0)
		{
		DofType saveDof(globalGrid->template getEntitiesCount< Cell >());
		DofType saveDof(vctdim*(globalGrid->template getEntitiesCount< Cell >()));

		SharedPointer<MeshFunctionType> saveMeshFunctionptr;
		saveMeshFunctionptr->bind(globalGrid,saveDof);
		linearFunctionEvaluator.evaluateAllEntities(saveMeshFunctionptr , linearFunctionPtr);
		VectorFieldType saveVectorField;
		saveVectorField.bind(globalGrid,saveDof);
		for(int i=0;i<vctdim;i++)
		linearFunctionEvaluator.evaluateAllEntities(saveVectorField[i] , linearFunctionPtr);

		File file;
		file.open( FileName, IOMode::write );
		saveMeshFunctionptr->save(file);
		saveVectorField.save(file);
		file.close();
		}

		SharedPointer<MeshType> loadGridptr;
		SharedPointer<MeshFunctionType> loadMeshFunctionptr;
		VectorFieldType loadVectorField;
		distrgrid.setupGrid(*loadGridptr);

		DofType loadDof(loadGridptr->template getEntitiesCount< Cell >());
		DofType loadDof(vctdim*(loadGridptr->template getEntitiesCount< Cell >()));
		loadDof.setValue(0);
		loadMeshFunctionptr->bind(loadGridptr,loadDof);
		loadVectorField.bind(loadGridptr,loadDof);

		DistributedGridIO<VectorFieldType,MpiIO> ::load(FileName, loadVectorField );

		DistributedGridIO<MeshFunctionType,MpiIO> ::load(FileName, *loadMeshFunctionptr );
		loadMeshFunctionptr->template synchronize<CommunicatorType>(); //need synchronization for overlaps to be filled corectly in loadDof
		for(int i=0;i<vctdim;i++)
		(loadVectorField[i])->template synchronize<CommunicatorType>(); //need synchronization for overlaps to be filled corectly in loadDof

		SharedPointer<MeshType> evalGridPtr;
		SharedPointer<MeshFunctionType> evalMeshFunctionptr;
		VectorFieldType evalVectorField;
		distrgrid.setupGrid(*evalGridPtr);

		DofType evalDof(evalGridPtr->template getEntitiesCount< Cell >());
		DofType evalDof(vctdim*(evalGridPtr->template getEntitiesCount< Cell >()));
		evalDof.setValue(-1);
		evalMeshFunctionptr->bind(evalGridPtr,evalDof);
		evalVectorField.bind(evalGridPtr,evalDof);

		linearFunctionEvaluator.evaluateAllEntities(evalMeshFunctionptr , linearFunctionPtr);
		evalMeshFunctionptr->template synchronize<CommunicatorType>();
		for(int i=0;i<vctdim;i++)
		{
		linearFunctionEvaluator.evaluateAllEntities(evalVectorField[i] , linearFunctionPtr);
		(evalVectorField[i])->template synchronize<CommunicatorType>();
		}

		for(int i=0;i<evalDof.getSize();i++)
		{
		EXPECT_EQ( evalDof.getElement(i), loadDof.getElement(i)) << "Compare Loaded and evaluated Dof Failed for: "<< i;
		}

		}*/
		}
		};