Partial implementation of 3D distributed grid. není funkční, potřebuju... (6b3e5420) · Commits · TNL / tnl-dev

src/TNL/Meshes/DistributedGrid.h

+344 −13

Original line number	Diff line number	Diff line
		@@ -29,6 +29,7 @@ class DistributedGrid

		};

		#define HAVE_MPI

		#ifndef HAVE_MPI
		template<typename GridType>
		@@ -51,6 +52,16 @@ public:
		};
		};

		template<typename GridType>
		class DistributedGrid <GridType,3>
		{
		public:
		bool isMPIUsed(void)
		{
		return false;
		};
		};

		#else
		//=============================================1D==================================
		template<typename GridType>
		@@ -211,7 +222,9 @@ class DistributedGrid <GridType,2>

		GridType GlobalGrid;
		PointType localorigin;
		CoordinatesType localsize;
		CoordinatesType localsize;//velikost gridu zpracovavane danym uzlem bez prekryvu
		CoordinatesType localbegin;//souradnice zacatku zpracovavane vypoctove oblasi
		CoordinatesType localgridsize;//velikost lokálního gridu včetně překryvů
		CoordinatesType overlap;


		@@ -263,7 +276,11 @@ class DistributedGrid <GridType,2>
		myproccoord[0]=0;
		myproccoord[1]=0;
		localorigin=GlobalGrid.getOrigin();
		localgridsize=GlobalGrid.getDimensions();
		localsize=GlobalGrid.getDimensions();
		localbegin.x()=0;
		localbegin.y()=0;

		return;
		}
		else
		@@ -333,21 +350,325 @@ class DistributedGrid <GridType,2>
		neighbors[DownRight]=getRangOfProcCoord(myproccoord[0]+1,myproccoord[1]+1);


		localbegin=overlap;

		if(neighbors[Left]==-1)
		{
		localorigin.x()+=overlap.x()*GlobalGrid.getSpaceSteps().x();
		localbegin.x()=0;
		}

		if(neighbors[Up]==-1)
		{
		localorigin.y()+=overlap.y()*GlobalGrid.getSpaceSteps().y();
		localbegin.y()=0;
		}

		//Tady je BUG pro distribuci v jednom řádku, jednom sloupci
		if(neighbors[Left]==-1\|\|neighbors[Right]==-1)
		localsize.x()+=overlap.x();
		localgridsize=localsize;
		//Add Overlaps
		if(neighbors[Left]!=-1)
		localgridsize.x()+=overlap.x();
		if(neighbors[Right]!=-1)
		localgridsize.x()+=overlap.x();

		if(neighbors[Up]!=-1)
		localgridsize.y()+=overlap.y();
		if(neighbors[Down]!=-1)
		localgridsize.y()+=overlap.y();
		}



		};

		void SetupGrid( GridType& grid)
		{
		grid.setOrigin(localorigin);
		grid.setDimensions(localgridsize);
		//compute local proporions by sideefect
		grid.setSpaceSteps(GlobalGrid.getSpaceSteps());
		grid.SetDistGrid(this);
		};

		void printcoords(void)
		{
		std::cout<<"("<<myproccoord[0]<<","<<myproccoord[1]<<"):" <<std::endl;
		};

		bool isMPIUsed(void)
		{
		return this->mpiInUse;
		};

		CoordinatesType getOverlap()
		{
		return this->overlap;
		};

		int * getNeighbors()
		{
		return this->neighbors;
		}

		CoordinatesType getLocalSize()
		{
		return this->localsize;
		}


		CoordinatesType getLocalBegin()
		{
		return this->localbegin;
		}

		private:

		int getRangOfProcCoord(int x, int y)
		{
		return y*procsdistr[0]+x;
		}
		};

		//========================3D======================================================
		enum Directions3D { West = 0 , East = 1 , Nord = 2, South=3, Top =4, Bottom=5,
		NordWest=6, NordEast=7, SouthWest=8, SouthEast=9,
		TopWest=10,TopEast=11,TopNord=12,TopSouth=13,
		BottomWest=14,BottomEast=15,BottomNord=16,BottomSouth=17,
		TopNordWest=18,TopNordEast=19,TopSouthWest=20,TopSouthEast=21,
		BottomNordWest=22,BottomNordEast=23,BottomSouthWest=24,BottomSouthEast=25};

		template<typename GridType>
		class DistributedGrid <GridType,3>
		{
		public:

		typedef typename GridType::IndexType IndexType;
		typedef typename GridType::PointType PointType;
		typedef Containers::StaticVector< 3, IndexType > CoordinatesType;


		private :

		GridType GlobalGrid;
		PointType localorigin;
		CoordinatesType localsize;
		CoordinatesType localgridsize;
		CoordinatesType localbegin;
		CoordinatesType overlap;


		IndexType Dimensions;
		bool mpiInUse;

		int rank;
		int nproc;

		int procsdistr[3];
		int myproccoord[3];
		int numberoflarger[3];

		int neighbors[26];

		public:
		//compute everithing
		DistributedGrid(GridType globalGrid,int *distribution=NULL)
		{

		//fuj
		overlap.x()=1;
		overlap.y()=1;
		overlap.z()=1;

		for (int i=0;i<26;i++)
		neighbors[i]=-1;

		Dimensions= GridType::getMeshDimension();
		GlobalGrid=globalGrid;
		//Detect MPI and number of process
		mpiInUse=false;



		if(MPI::Is_initialized())
		{
		rank=MPI::COMM_WORLD.Get_rank();
		this->nproc=MPI::COMM_WORLD.Get_size();
		//use MPI only if have more than one process
		if(this->nproc>1)
		{
		mpiInUse=true;
		}
		}

		if(!mpiInUse)
		{
		//Without MPI
		myproccoord[0]=0;
		myproccoord[1]=0;
		myproccoord[2]=0;

		localorigin=GlobalGrid.getOrigin();
		localsize=GlobalGrid.getDimensions();
		return;
		}
		else
		localsize.x()+=2*overlap.x();
		{
		//With MPI
		//compute node distribution
		if(distribution!=NULL)
		{
		procsdistr[0]=distribution[0];
		procsdistr[1]=distribution[1];
		procsdistr[2]=distribution[2];
		}
		else
		{
		procsdistr[0]=0;
		procsdistr[1]=0;
		procsdistr[2]=0;
		}
		MPI_Dims_create(nproc, 3, procsdistr);
		myproccoord[2]=rank/(procsdistr[0]*procsdistr[1]); // CO je X, Y, Z? x je 0 y je 1 a z je 2, snad...
		myproccoord[1]=(rank%(procsdistr[0]*procsdistr[1]))/procsdistr[0];
		myproccoord[0]=(rank%(procsdistr[0]*procsdistr[1]))%procsdistr[0];


		//compute local mesh size

		numberoflarger[0]=GlobalGrid.getDimensions().x()%procsdistr[0];
		numberoflarger[1]=GlobalGrid.getDimensions().y()%procsdistr[1];
		numberoflarger[2]=GlobalGrid.getDimensions().z()%procsdistr[2];

		localsize.x()=(GlobalGrid.getDimensions().x()/procsdistr[0]);
		localsize.y()=(GlobalGrid.getDimensions().y()/procsdistr[1]);
		localsize.z()=(GlobalGrid.getDimensions().z()/procsdistr[2]);

		if(numberoflarger[0]>myproccoord[0])
		localsize.x()+=1;
		if(numberoflarger[1]>myproccoord[1])
		localsize.y()+=1;
		if(numberoflarger[2]>myproccoord[2])
		localsize.z()+=1;

		if(numberoflarger[0]>myproccoord[0])
		localorigin.x()=GlobalGrid.getOrigin().x()
		+(myproccoord[0]localsize.x()-overlap.x())GlobalGrid.getSpaceSteps().x();
		else
		localorigin.x()=GlobalGrid.getOrigin().x()
		+(numberoflarger[0](localsize.x()+1)+(myproccoord[0]-numberoflarger[0])localsize.x()-overlap.x())
		*GlobalGrid.getSpaceSteps().x();

		if(numberoflarger[1]>myproccoord[1])
		localorigin.y()=GlobalGrid.getOrigin().y()
		+(myproccoord[1]localsize.y()-overlap.y())GlobalGrid.getSpaceSteps().y();
		else
		localorigin.y()=GlobalGrid.getOrigin().y()
		+(numberoflarger[1](localsize.y()+1)+(myproccoord[1]-numberoflarger[1])localsize.y()-overlap.y())
		*GlobalGrid.getSpaceSteps().y();

		if(neighbors[Up]==-1\|\|neighbors[Down]==-1)
		localsize.y()+=overlap.y();
		if(numberoflarger[2]>myproccoord[2])
		localorigin.z()=GlobalGrid.getOrigin().z()
		+(myproccoord[2]localsize.z()-overlap.z())GlobalGrid.getSpaceSteps().z();
		else
		localsize.y()+=2*overlap.y();
		localorigin.z()=GlobalGrid.getOrigin().z()
		+(numberoflarger[2](localsize.z()+1)+(myproccoord[2]-numberoflarger[2])localsize.z()-overlap.z())
		*GlobalGrid.getSpaceSteps().z();


		//nearnodes
		//X Y Z
		if(myproccoord[0]>0)
		neighbors[West]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1],myproccoord[2]);
		if(myproccoord[0]<procsdistr[0]-1)
		neighbors[East]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1],myproccoord[2]);
		if(myproccoord[1]>0)
		neighbors[Nord]=getRankOfProcCoord(myproccoord[0],myproccoord[1]-1,myproccoord[2]);
		if(myproccoord[1]<procsdistr[1]-1)
		neighbors[South]=getRankOfProcCoord(myproccoord[0],myproccoord[1]+1,myproccoord[2]);
		if(myproccoord[2]>0)
		neighbors[Bottom]=getRankOfProcCoord(myproccoord[0],myproccoord[1],myproccoord[2]-1);
		if(myproccoord[2]<procsdistr[2]-1)
		neighbors[Top]=getRankOfProcCoord(myproccoord[0],myproccoord[1],myproccoord[2]+1);
		//XY
		if(myproccoord[0]>0 && myproccoord[1]>0)
		neighbors[NordWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]-1,myproccoord[2]);
		if(myproccoord[0]>0 && myproccoord[1]<procsdistr[1]-1)
		neighbors[SouthWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]+1,myproccoord[2]);
		if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]>0)
		neighbors[NordEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]-1,myproccoord[2]);
		if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]<procsdistr[1]-1)
		neighbors[SouthEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]+1,myproccoord[2]);
		//XZ
		if(myproccoord[0]>0 && myproccoord[2]>0)
		neighbors[TopWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1],myproccoord[2]-1);
		if(myproccoord[0]>0 && myproccoord[2]<procsdistr[2]-1)
		neighbors[BottomWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1],myproccoord[2]+1);
		if(myproccoord[0]<procsdistr[0]-1 && myproccoord[2]>0)
		neighbors[TopEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1],myproccoord[2]-1);
		if(myproccoord[0]<procsdistr[0]-1 && myproccoord[2]<procsdistr[2]-1)
		neighbors[BottomEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1],myproccoord[2]+1);
		//YZ
		if(myproccoord[1]>0 && myproccoord[2]>0)
		neighbors[TopNord]=getRankOfProcCoord(myproccoord[0],myproccoord[1]-1,myproccoord[2]-1);
		if(myproccoord[1]>0 && myproccoord[2]<procsdistr[2]-1)
		neighbors[BottomNord]=getRankOfProcCoord(myproccoord[0],myproccoord[1]-1,myproccoord[2]+1);
		if(myproccoord[1]<procsdistr[1]-1 && myproccoord[2]>0)
		neighbors[TopSouth]=getRankOfProcCoord(myproccoord[0],myproccoord[1]+1,myproccoord[2]-1);
		if(myproccoord[1]<procsdistr[1]-1 && myproccoord[2]<procsdistr[2]-1)
		neighbors[BottomSouth]=getRankOfProcCoord(myproccoord[0],myproccoord[1]+1,myproccoord[2]+1);
		//XYZ
		if(myproccoord[0]>0 && myproccoord[1]>0 && myproccoord[2]>0 )
		neighbors[TopNordWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]-1,myproccoord[2]-1);
		if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]>0 && myproccoord[2]>0 )
		neighbors[TopNordEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]-1,myproccoord[2]-1);
		if(myproccoord[0]>0 && myproccoord[1]<procsdistr[1]-1 && myproccoord[2]>0 )
		neighbors[TopSouthWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]+1,myproccoord[2]-1);
		if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]<procsdistr[1]-1 && myproccoord[2]>0 )
		neighbors[TopSouthEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]+1,myproccoord[2]-1);
		if(myproccoord[0]>0 && myproccoord[1]>0 && myproccoord[2]<procsdistr[2]-1 )
		neighbors[BottomNordWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]-1,myproccoord[2]+1);
		if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]>0 && myproccoord[2]<procsdistr[2]-1 )
		neighbors[BottomNordEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]-1,myproccoord[2]+1);
		if(myproccoord[0]>0 && myproccoord[1]<procsdistr[1]-1 && myproccoord[2]<procsdistr[2]-1 )
		neighbors[BottomSouthWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]+1,myproccoord[2]+1);
		if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]<procsdistr[1]-1 && myproccoord[2]<procsdistr[2]-1 )
		neighbors[BottomSouthEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]+1,myproccoord[2]+1);

		localbegin=overlap;

		if(neighbors[West]==-1)
		{
		localorigin.x()+=overlap.x()*GlobalGrid.getSpaceSteps().x();
		localbegin.x()=0;
		}
		if(neighbors[Nord]==-1)
		{
		localorigin.y()+=overlap.y()*GlobalGrid.getSpaceSteps().y();
		localbegin.y()=0;
		}
		if(neighbors[Top]==-1)
		{
		localorigin.z()+=overlap.z()*GlobalGrid.getSpaceSteps().z();
		localbegin.z()=0;
		}

		localgridsize=localsize;

		if(neighbors[West]!=-1)
		localgridsize.x()+=overlap.x();
		if(neighbors[East]!=-1)
		localgridsize.x()+=overlap.x();

		if(neighbors[Nord]!=-1)
		localgridsize.y()+=overlap.y();
		if(neighbors[South]!=-1)
		localgridsize.y()+=overlap.y();

		if(neighbors[Top]!=-1)
		localgridsize.z()+=overlap.z();
		if(neighbors[Bottom]!=-1)
		localgridsize.z()+=overlap.z();

		}


		@@ -365,7 +686,7 @@ class DistributedGrid <GridType,2>

		void printcoords(void)
		{
		std::cout<<"("<<myproccoord[0]<<","<<myproccoord[1]<<"):" <<std::endl;
		std::cout<<"("<<myproccoord[0]<<","<<myproccoord[1]<<","<<myproccoord[2]<<"):" <<std::endl;
		};

		bool isMPIUsed(void)
		@@ -383,11 +704,21 @@ class DistributedGrid <GridType,2>
		return this->neighbors;
		}

		CoordinatesType getLocalSize()
		{
		return this->localsize;
		}

		CoordinatesType getLocalBegin()
		{
		return this->localbegin;
		}

		private:

		int getRangOfProcCoord(int x, int y)
		int getRankOfProcCoord(int x, int y, int z)
		{
		return y*procsdistr[0]+x;
		return zprocsdistr[0]procsdistr[1]+y*procsdistr[0]+x;
		}
		};

src/TNL/Meshes/DistributedGridSynchronizer.h

+179 −15

File changed.

Preview size limit exceeded, changes collapsed.

tests/mpi/MeshFunctionEvauateTest.cpp

+15 −9

Original line number	Diff line number	Diff line
		@@ -5,6 +5,7 @@
		*/

		#include <iostream>
		#include <sstream>
		#define HAVE_MPI
		#include <TNL/mpi-supp.h>

		@@ -73,8 +74,10 @@ int main ( int argc, char *argv[])
		globalGrid.setDimensions(size,size);
		globalGrid.setDomain(globalOrigin,globalProportions);

		int distr[2]={0,1};
		DistributedGrid<MeshType> distrgrid(globalGrid, distr);
		//int distr[2]={0,1};
		//DistributedGrid<MeshType> distrgrid(globalGrid, distr);

		DistributedGrid<MeshType> distrgrid(globalGrid);

		SharedPointer<MeshType> gridptr;
		SharedPointer<MeshFunctionType> meshFunctionptr;
		@@ -96,12 +99,13 @@ int main ( int argc, char *argv[])

		for(int i=0;i<cycles;i++)
		{
		//zero->Number=MPI::COMM_WORLD.Get_rank();
		zero->Number=i;
		zero->Number=MPI::COMM_WORLD.Get_rank();
		//zero->Number=i;
		eval.start();
		zeroevaluator.evaluateInteriorEntities( meshFunctionptr , zero );
		//evaluator.evaluateAllEntities( meshFunctionptr , functionToEvaluate );
		zero->Number=-1;
		//zero->Number=-1;
		zero->Number=MPI::COMM_WORLD.Get_rank();
		zeroevaluator.evaluateBoundaryEntities( meshFunctionptr , zero );
		MPI::COMM_WORLD.Barrier();
		eval.stop();
		@@ -114,24 +118,26 @@ int main ( int argc, char *argv[])

		sum+=dof[2*gridptr->getDimensions().y()]; //dummy acces to array
		}
		all.stop();

		#ifdef OUTPUT
		//print local dof
		int maxx=gridptr->getDimensions().x();
		int maxy=gridptr->getDimensions().y();
		distrgrid.printcoords();
		stringstream sout;
		for(int j=0;j<maxy;j++)
		{
		for(int i=0;i<maxx;i++)
		{
		cout <<dof[maxx*j+i]<<" ";
		sout <<dof[maxx*j+i]<<" ";
		}
		cout << endl;
		sout << endl;
		}
		cout << endl<<endl;
		cout << sout.str()<< endl<<endl;
		#endif

		all.stop();