3D distributed Grid - 3D Grid Traversars supports distributed grid now. (80d70074) · Commits · TNL / tnl-dev

src/TNL/Meshes/GridDetails/Traverser_Grid3D_impl.h

+183 −13

Original line number	Diff line number	Diff line
		@@ -12,6 +12,8 @@

		#include <TNL/Meshes/GridDetails/GridTraverser.h>

		#include "Traverser_Grid3D.h"

		namespace TNL {
		namespace Meshes {

		@@ -34,6 +36,9 @@ processBoundaryEntities( const GridPointer& gridPointer,
		*/
		static_assert( GridEntity::getEntityDimension() == 3, "The entity has wrong dimension." );

		auto distributedgrid=gridPointer->GetDistGrid();
		if(distributedgrid==nullptr\|\|!distributedgrid->isMPIUsed())
		{
		GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, true, 1, 1, 1 >(
		gridPointer,
		CoordinatesType( 0, 0, 0 ),
		@@ -41,6 +46,76 @@ processBoundaryEntities( const GridPointer& gridPointer,
		userDataPointer,
		0 );
		}
		else
		{
		//MPI
		#define HAVE_MPI
		#ifdef HAVE_MPI
		int* neighbors=distributedgrid->getNeighbors();
		if(neighbors[West]==-1)
		{
		GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
		gridPointer,
		CoordinatesType(0,0,0),
		CoordinatesType(0,gridPointer->getDimensions().y()-1,gridPointer->getDimensions().z()-1),
		userDataPointer,
		0 );
		}

		if(neighbors[East]==-1)
		{
		GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
		gridPointer,
		CoordinatesType(gridPointer->getDimensions().x()-1,0,0),
		CoordinatesType(gridPointer->getDimensions().x()-1,gridPointer->getDimensions().y()-1,gridPointer->getDimensions().z()-1),
		userDataPointer,
		0 );
		}

		if(neighbors[Nord]==-1)
		{
		GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
		gridPointer,
		CoordinatesType(0,0,0),
		CoordinatesType(gridPointer->getDimensions().x()-1,0,gridPointer->getDimensions().z()-1),
		userDataPointer,
		0 );
		}

		if(neighbors[South]==-1)
		{
		GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
		gridPointer,
		CoordinatesType(0,gridPointer->getDimensions().y()-1,0),
		CoordinatesType(gridPointer->getDimensions().x()-1,gridPointer->getDimensions().y()-1,gridPointer->getDimensions().z()-1),
		userDataPointer,
		0 );
		}

		if(neighbors[Bottom]==-1)
		{
		GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
		gridPointer,
		CoordinatesType(0,0,0),
		CoordinatesType(gridPointer->getDimensions().x()-1,gridPointer->getDimensions().y()-1,0),
		userDataPointer,
		0 );
		}

		if(neighbors[Top]==-1)
		{
		GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
		gridPointer,
		CoordinatesType(0,0,gridPointer->getDimensions().z()-1),
		CoordinatesType(gridPointer->getDimensions().x()-1,gridPointer->getDimensions().y()-1,gridPointer->getDimensions().z()-1),
		userDataPointer,
		0 );
		}

		#endif

		}
		}

		template< typename Real,
		typename Device,
		@@ -58,6 +133,9 @@ processInteriorEntities( const GridPointer& gridPointer,
		*/
		static_assert( GridEntity::getEntityDimension() == 3, "The entity has wrong dimension." );

		auto distributedgrid=gridPointer->GetDistGrid();
		if(distributedgrid==nullptr\|\|!distributedgrid->isMPIUsed())
		{
		GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
		gridPointer,
		CoordinatesType( 1, 1, 1 ),
		@@ -65,6 +143,52 @@ processInteriorEntities( const GridPointer& gridPointer,
		userDataPointer,
		0 );
		}
		else
		{
		#ifdef HAVE_MPI
		int* neighbors=distributedgrid->getNeighbors();
		CoordinatesType begin( distributedgrid->getOverlap());
		CoordinatesType end( gridPointer->getDimensions() - distributedgrid->getOverlap()- CoordinatesType(1,1,1) );
		if(neighbors[West]==-1)
		{
		begin.x()= 1 ;
		}

		if(neighbors[East]==-1)
		{
		end.x()=gridPointer->getDimensions().x()-2;
		}

		if(neighbors[Nord]==-1)
		{
		begin.y()= 1 ;
		}

		if(neighbors[South]==-1)
		{
		end.y()=gridPointer->getDimensions().y()-2;
		}

		if(neighbors[Bottom]==-1)
		{
		begin.z()= 1 ;
		}

		if(neighbors[Top]==-1)
		{
		end.z()=gridPointer->getDimensions().z()-2;
		}

		GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
		gridPointer,
		begin,
		end,
		userDataPointer,
		0);
		#endif
		}

		}

		template< typename Real,
		typename Device,
		@@ -82,6 +206,9 @@ processAllEntities( const GridPointer& gridPointer,
		*/
		static_assert( GridEntity::getEntityDimension() == 3, "The entity has wrong dimension." );

		auto distributedgrid=gridPointer->GetDistGrid();
		if(distributedgrid==nullptr\|\|!distributedgrid->isMPIUsed())
		{
		GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
		gridPointer,
		CoordinatesType( 0, 0, 0 ),
		@@ -89,6 +216,49 @@ processAllEntities( const GridPointer& gridPointer,
		userDataPointer,
		0 );
		}
		else
		{
		CoordinatesType begin( distributedgrid->getOverlap());
		CoordinatesType end( gridPointer->getDimensions() - distributedgrid->getOverlap()- CoordinatesType(1,1,1) );
		int* neighbors=distributedgrid->getNeighbors();
		if(neighbors[West]==-1)
		{
		begin.x()= 0 ;
		}

		if(neighbors[East]==-1)
		{
		end.x()=gridPointer->getDimensions().x()-1;
		}

		if(neighbors[Nord]==-1)
		{
		begin.y()= 0 ;
		}

		if(neighbors[South]==-1)
		{
		end.y()=gridPointer->getDimensions().y()-1;
		}

		if(neighbors[Bottom]==-1)
		{
		begin.z()= 0 ;
		}

		if(neighbors[Top]==-1)
		{
		end.z()=gridPointer->getDimensions().z()-1;
		}

		GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
		gridPointer,
		begin,
		end,
		userDataPointer,
		0 );
		}
		}

		/****
		* Grid 3D, faces

tests/mpi/MeshFunctionEvauateTest.cpp

+17 −8

Original line number	Diff line number	Diff line
		@@ -92,6 +92,14 @@ int main ( int argc, char *argv[])

		DofType dof(gridptr->template getEntitiesCount< Cell >());

		int maxx=gridptr->getDimensions().x();
		int maxy=gridptr->getDimensions().y();
		int maxz=gridptr->getDimensions().z();
		for(int k=0;k<maxz;k++)
		for(int j=0;j<maxy;j++)
		for(int i=0;i<maxx;i++)
		dof[kmaxxmaxy+maxx*j+i]=0;

		meshFunctionptr->bind(gridptr,dof);

		SharedPointer< FunctionToEvaluate<double,3>, Host > functionToEvaluate;
		@@ -103,11 +111,12 @@ int main ( int argc, char *argv[])

		for(int i=0;i<cycles;i++)
		{
		zero->Number=MPI::COMM_WORLD.Get_rank();
		//zero->Number=i;
		//zero->Number=MPI::COMM_WORLD.Get_rank();
		zero->Number=-1;
		eval.start();
		//zeroevaluator.evaluateAllEntities( meshFunctionptr , zero );
		evaluator.evaluateAllEntities( meshFunctionptr , functionToEvaluate );
		zeroevaluator.evaluateBoundaryEntities( meshFunctionptr , zero );
		zero->Number=MPI::COMM_WORLD.Get_rank();
		zeroevaluator.evaluateInteriorEntities( meshFunctionptr , zero );
		//zero->Number=-1;
		/*zero->Number=MPI::COMM_WORLD.Get_rank();
		zeroevaluator.evaluateBoundaryEntities( meshFunctionptr , zero );*/
		@@ -116,7 +125,7 @@ int main ( int argc, char *argv[])


		sync.start();
		DistributedGridSynchronizer<DistributedGrid<MeshType>,MeshFunctionType,3>::Synchronize(distrgrid,*meshFunctionptr);
		//DistributedGridSynchronizer<DistributedGrid<MeshType>,MeshFunctionType,3>::Synchronize(distrgrid,*meshFunctionptr);
		MPI::COMM_WORLD.Barrier();
		sync.stop();

		@@ -126,9 +135,9 @@ int main ( int argc, char *argv[])

		#ifdef OUTPUT
		//print local dof
		int maxx=gridptr->getDimensions().x();
		int maxy=gridptr->getDimensions().y();
		int maxz=gridptr->getDimensions().z();
		maxx=gridptr->getDimensions().x();
		maxy=gridptr->getDimensions().y();
		maxz=gridptr->getDimensions().z();

		stringstream sout;
		distrgrid.printcoords(sout);