Commit f77e2274 authored by Vít Hanousek's avatar Vít Hanousek
Browse files

Working implementation of 3D distributed grid, but 3D traversars do not know... 

Working implementation of 3D distributed grid, but 3D traversars do not know about distributed grid, Process{Boundary/Interrior}Entities do not work correctly with distributed gird.
parent 6b3e5420

src/TNL/Meshes/DistributedGrid.h

+45 −47
Original line number Diff line number Diff line
@@ -29,7 +29,7 @@ class DistributedGrid 


};



#define HAVE_MPI

//#define HAVE_MPI



#ifndef HAVE_MPI

template<typename GridType>
@@ -179,9 +179,9 @@ class DistributedGrid <GridType,1> 
           grid.SetDistGrid(this);

       };

       

       void printcoords(void)

       void printcoords(std::ostream& out)

       {

           std::cout<<rank<<":" <<std::endl;

           out<<rank<<":" <<std::endl;

       };

       

       bool isMPIUsed(void)
@@ -390,9 +390,9 @@ class DistributedGrid <GridType,2> 
           grid.SetDistGrid(this);

       };

       

       void printcoords(void)

       void printcoords(std::ostream& out)

       {

           std::cout<<"("<<myproccoord[0]<<","<<myproccoord[1]<<"):" <<std::endl;

           out<<"("<<myproccoord[0]<<","<<myproccoord[1]<<"):" <<std::endl;

       };

       

       bool isMPIUsed(void)
@@ -432,10 +432,11 @@ class DistributedGrid <GridType,2> 
//========================3D======================================================

enum Directions3D { West = 0 , East = 1 , Nord = 2, South=3, Top =4, Bottom=5, 

                  NordWest=6, NordEast=7, SouthWest=8, SouthEast=9,

                  TopWest=10,TopEast=11,TopNord=12,TopSouth=13,

                  BottomWest=14,BottomEast=15,BottomNord=16,BottomSouth=17,

                  TopNordWest=18,TopNordEast=19,TopSouthWest=20,TopSouthEast=21,

                  BottomNordWest=22,BottomNordEast=23,BottomSouthWest=24,BottomSouthEast=25};

                  BottomWest=10,BottomEast=11,BottomNord=12,BottomSouth=13,

                  TopWest=14,TopEast=15,TopNord=16,TopSouth=17,

                  BottomNordWest=18,BottomNordEast=19,BottomSouthWest=20,BottomSouthEast=21,

                  TopNordWest=22,TopNordEast=23,TopSouthWest=24,TopSouthEast=25

                  };



template<typename GridType>    

class DistributedGrid <GridType,3>
@@ -509,6 +510,7 @@ class DistributedGrid <GridType,3> 
               

               localorigin=GlobalGrid.getOrigin();

               localsize=GlobalGrid.getDimensions();

               localgridsize=localsize;

               return;

           }

           else
@@ -532,9 +534,7 @@ class DistributedGrid <GridType,3> 
               myproccoord[1]=(rank%(procsdistr[0]*procsdistr[1]))/procsdistr[0];

               myproccoord[0]=(rank%(procsdistr[0]*procsdistr[1]))%procsdistr[0];



               

               //compute local mesh size               

               

               numberoflarger[0]=GlobalGrid.getDimensions().x()%procsdistr[0];

               numberoflarger[1]=GlobalGrid.getDimensions().y()%procsdistr[1];

               numberoflarger[2]=GlobalGrid.getDimensions().z()%procsdistr[2];
@@ -574,7 +574,6 @@ class DistributedGrid <GridType,3> 
                                +(numberoflarger[2]*(localsize.z()+1)+(myproccoord[2]-numberoflarger[2])*localsize.z()-overlap.z())

                                *GlobalGrid.getSpaceSteps().z();

               

               

               //nearnodes

               //X Y Z

               if(myproccoord[0]>0)
@@ -589,50 +588,52 @@ class DistributedGrid <GridType,3> 
                   neighbors[Bottom]=getRankOfProcCoord(myproccoord[0],myproccoord[1],myproccoord[2]-1);

               if(myproccoord[2]<procsdistr[2]-1)

                   neighbors[Top]=getRankOfProcCoord(myproccoord[0],myproccoord[1],myproccoord[2]+1);



               //XY

               if(myproccoord[0]>0 && myproccoord[1]>0)

                   neighbors[NordWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]-1,myproccoord[2]);

               if(myproccoord[0]>0 && myproccoord[1]<procsdistr[1]-1)

                   neighbors[SouthWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]+1,myproccoord[2]);

               if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]>0)

                   neighbors[NordEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]-1,myproccoord[2]);

               if(myproccoord[0]>0 && myproccoord[1]<procsdistr[1]-1)

                   neighbors[SouthWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]+1,myproccoord[2]);

               if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]<procsdistr[1]-1)

                   neighbors[SouthEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]+1,myproccoord[2]);             

               //XZ

               if(myproccoord[0]>0 && myproccoord[2]>0)

                   neighbors[TopWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1],myproccoord[2]-1);

               if(myproccoord[0]>0 && myproccoord[2]<procsdistr[2]-1)

                   neighbors[BottomWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1],myproccoord[2]+1);

                   neighbors[BottomWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1],myproccoord[2]-1);

               if(myproccoord[0]<procsdistr[0]-1 && myproccoord[2]>0)

                   neighbors[TopEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1],myproccoord[2]-1);

                   neighbors[BottomEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1],myproccoord[2]-1); 

               if(myproccoord[0]>0 && myproccoord[2]<procsdistr[2]-1)

                   neighbors[TopWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1],myproccoord[2]+1);

              if(myproccoord[0]<procsdistr[0]-1 && myproccoord[2]<procsdistr[2]-1)

                   neighbors[BottomEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1],myproccoord[2]+1);

                   neighbors[TopEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1],myproccoord[2]+1);

               //YZ

               if(myproccoord[1]>0 && myproccoord[2]>0)

                   neighbors[TopNord]=getRankOfProcCoord(myproccoord[0],myproccoord[1]-1,myproccoord[2]-1);

               if(myproccoord[1]>0 && myproccoord[2]<procsdistr[2]-1)

                   neighbors[BottomNord]=getRankOfProcCoord(myproccoord[0],myproccoord[1]-1,myproccoord[2]+1);

                   neighbors[BottomNord]=getRankOfProcCoord(myproccoord[0],myproccoord[1]-1,myproccoord[2]-1);

               if(myproccoord[1]<procsdistr[1]-1 && myproccoord[2]>0)

                   neighbors[TopSouth]=getRankOfProcCoord(myproccoord[0],myproccoord[1]+1,myproccoord[2]-1);

                   neighbors[BottomSouth]=getRankOfProcCoord(myproccoord[0],myproccoord[1]+1,myproccoord[2]-1);

               if(myproccoord[1]>0 && myproccoord[2]<procsdistr[2]-1)

                   neighbors[TopNord]=getRankOfProcCoord(myproccoord[0],myproccoord[1]-1,myproccoord[2]+1);               

               if(myproccoord[1]<procsdistr[1]-1 && myproccoord[2]<procsdistr[2]-1)

                   neighbors[BottomSouth]=getRankOfProcCoord(myproccoord[0],myproccoord[1]+1,myproccoord[2]+1);

                   neighbors[TopSouth]=getRankOfProcCoord(myproccoord[0],myproccoord[1]+1,myproccoord[2]+1);

               //XYZ

               if(myproccoord[0]>0 && myproccoord[1]>0 && myproccoord[2]>0 )

                   neighbors[TopNordWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]-1,myproccoord[2]-1);

                   neighbors[BottomNordWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]-1,myproccoord[2]-1);

               if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]>0 && myproccoord[2]>0 )

                   neighbors[TopNordEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]-1,myproccoord[2]-1);

                   neighbors[BottomNordEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]-1,myproccoord[2]-1);

               if(myproccoord[0]>0 && myproccoord[1]<procsdistr[1]-1 && myproccoord[2]>0 )

                   neighbors[TopSouthWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]+1,myproccoord[2]-1);

                   neighbors[BottomSouthWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]+1,myproccoord[2]-1);

               if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]<procsdistr[1]-1 && myproccoord[2]>0 )

                   neighbors[TopSouthEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]+1,myproccoord[2]-1);

                   neighbors[BottomSouthEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]+1,myproccoord[2]-1);

               if(myproccoord[0]>0 && myproccoord[1]>0 && myproccoord[2]<procsdistr[2]-1 )

                   neighbors[BottomNordWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]-1,myproccoord[2]+1);

                   neighbors[TopNordWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]-1,myproccoord[2]+1);

               if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]>0 && myproccoord[2]<procsdistr[2]-1 )

                   neighbors[BottomNordEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]-1,myproccoord[2]+1);

                   neighbors[TopNordEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]-1,myproccoord[2]+1);

               if(myproccoord[0]>0 && myproccoord[1]<procsdistr[1]-1 && myproccoord[2]<procsdistr[2]-1 )

                   neighbors[BottomSouthWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]+1,myproccoord[2]+1);

                   neighbors[TopSouthWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]+1,myproccoord[2]+1);

               if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]<procsdistr[1]-1 && myproccoord[2]<procsdistr[2]-1 )

                   neighbors[BottomSouthEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]+1,myproccoord[2]+1);                           

                   neighbors[TopSouthEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]+1,myproccoord[2]+1);   



               

               localbegin=overlap;
@@ -646,7 +647,7 @@ class DistributedGrid <GridType,3> 
                   localorigin.y()+=overlap.y()*GlobalGrid.getSpaceSteps().y();

                   localbegin.y()=0;

               }

               if(neighbors[Top]==-1)

               if(neighbors[Bottom]==-1)

               {

                   localorigin.z()+=overlap.z()*GlobalGrid.getSpaceSteps().z();

                   localbegin.z()=0;
@@ -664,29 +665,26 @@ class DistributedGrid <GridType,3> 
               if(neighbors[South]!=-1)

                   localgridsize.y()+=overlap.y();

               

               if(neighbors[Top]!=-1)

                   localgridsize.z()+=overlap.z();

               if(neighbors[Bottom]!=-1)

                   localgridsize.z()+=overlap.z();

               if(neighbors[Top]!=-1)

                   localgridsize.z()+=overlap.z();

               

           }                     

                     

           

                      

       };

       

       void SetupGrid( GridType& grid)

       {

           grid.setOrigin(localorigin);

           grid.setDimensions(localsize);

           grid.setDimensions(localgridsize);

           //compute local proporions by sideefect

           grid.setSpaceSteps(GlobalGrid.getSpaceSteps());

           grid.SetDistGrid(this);

       };

       

       void printcoords(void)

       void printcoords(std::ostream& out )

       {

           std::cout<<"("<<myproccoord[0]<<","<<myproccoord[1]<<","<<myproccoord[2]<<"):" <<std::endl;

           out<<"("<<myproccoord[0]<<","<<myproccoord[1]<<","<<myproccoord[2]<<"):" <<std::endl;

       };

       

       bool isMPIUsed(void)

src/TNL/Meshes/DistributedGridSynchronizer.h

+84 −45

File changed.

Preview size limit exceeded, changes collapsed.

src/TNL/Meshes/GridDetails/Grid3D.h

+16 −1
Original line number Diff line number Diff line
@@ -68,6 +68,9 @@ class Grid< 3, Real, Device, Index > : public Object 


   void setDomain( const PointType& origin,

                   const PointType& proportions );

   

   void setOrigin( const PointType& origin);

   

   __cuda_callable__

   inline const PointType& getOrigin() const;
@@ -94,6 +97,12 @@ class Grid< 3, Real, Device, Index > : public Object 
   __cuda_callable__

   inline const PointType& getSpaceSteps() const;



   inline void setSpaceSteps(const PointType& steps);

   

   void SetDistGrid(DistributedGrid <ThisType,3> * distGrid);

   

   DistributedGrid <ThisType,3> * GetDistGrid(void) const;

   

   template< int xPow, int yPow, int zPow >

   __cuda_callable__

   const RealType& getSpaceStepsProducts() const;
@@ -143,6 +152,10 @@ class Grid< 3, Real, Device, Index > : public Object 


   protected:



   void computeProportions();

       

   void computeSpaceStepPowers();    

       

   void computeSpaceSteps();



   CoordinatesType dimensions;
@@ -160,6 +173,8 @@ class Grid< 3, Real, Device, Index > : public Object 
 

   RealType spaceStepsProducts[ 5 ][ 5 ][ 5 ];

   

   DistributedGrid <ThisType,3> *distGrid;



   template< typename, typename, int >

   friend class GridEntityGetter;

src/TNL/Meshes/GridDetails/Grid3D_impl.h

+57 −1
Original line number Diff line number Diff line
@@ -91,6 +91,17 @@ void Grid< 3, Real, Device, Index > :: computeSpaceSteps() 
      this->spaceSteps.x() = this->proportions.x() / ( Real ) this->getDimensions().x();

      this->spaceSteps.y() = this->proportions.y() / ( Real ) this->getDimensions().y();

      this->spaceSteps.z() = this->proportions.z() / ( Real ) this->getDimensions().z();

      

      this->computeSpaceStepPowers();

      

   }

};



template< typename Real,

          typename Device,

          typename Index >

void Grid< 3, Real, Device, Index > :: computeSpaceStepPowers()

{      

      const RealType& hx = this->spaceSteps.x();

      const RealType& hy = this->spaceSteps.y();

      const RealType& hz = this->spaceSteps.z();
@@ -161,8 +172,37 @@ void Grid< 3, Real, Device, Index > :: computeSpaceSteps() 
         }

      }

}





template< typename Real,

          typename Device,

          typename Index >

void Grid< 3, Real, Device, Index > :: computeProportions()

{

    this->proportions.x()=this->dimensions.x()*this->spaceSteps.x();

    this->proportions.y()=this->dimensions.y()*this->spaceSteps.y();

    this->proportions.z()=this->dimensions.z()*this->spaceSteps.z();

}



template< typename Real,

          typename Device,

          typename Index >

void Grid< 3, Real, Device, Index > :: setOrigin( const PointType& origin)

{

    this->origin=origin;

}



template< typename Real,

          typename Device,

          typename Index >

void Grid< 3, Real, Device, Index > :: setSpaceSteps(const PointType& steps)

{

     this->spaceSteps=steps;    

     computeSpaceStepPowers();

     computeProportions(); 

}





template< typename Real,

          typename Device,

          typename Index >
@@ -453,6 +493,22 @@ template< typename Real, 
   return ::pow( lpNorm, 1.0 / p );

}



template< typename Real,

          typename Device,

          typename Index >

void Grid< 3, Real, Device, Index >:: SetDistGrid(DistributedGrid <ThisType,3> * distGrid)

{

    this->distGrid=distGrid;

}

   

template< typename Real,

          typename Device,

          typename Index >

DistributedGrid <Grid< 3, Real, Device, Index >,3> * Grid< 3, Real, Device, Index >:: GetDistGrid(void) const

{

    return this->distGrid;

}



template< typename Real,

          typename Device,

          typename Index >

tests/mpi/Functions.h

+38 −0
Original line number Diff line number Diff line
@@ -102,3 +102,41 @@ class ZeroFunction<Real,2> : public Functions::Domain< 2, Functions::MeshDomain 
		 

	  }

};



//============================3D============================================================

template <typename Real>

class FunctionToEvaluate<Real,3> : public Functions::Domain< 3, Functions::MeshDomain >

{

   public:

	  typedef Real RealType;

	  FunctionToEvaluate( )

	  {};



	  template< typename EntityType >

	  RealType operator()( const EntityType& meshEntity,

								  const RealType& time = 0.0 ) const

	  {

		 //return meshEntity.getCoordinates().y()*10+meshEntity.getCoordinates().x();

		 return meshEntity.getCenter().z()*10000+meshEntity.getCenter().y()*100+meshEntity.getCenter().x();

	  }

};



template <typename Real>

class ZeroFunction<Real,3> : public Functions::Domain< 3, Functions::MeshDomain >

{

   public:

          typedef Real RealType;

          

          Real Number;

	  ZeroFunction( )

	  {};

          

	  template< typename EntityType >

	  RealType operator()( const EntityType& meshEntity,

								  const RealType& time = 0.0 ) const

	  {

		 //return meshEntity.getCoordinates().y()*10+meshEntity.getCoordinates().x();

		 return this->Number;

		 

	  }

};