Commit e1326cac authored by Vít Hanousek's avatar Vít Hanousek Committed by Jakub Klinkovský
Browse files

Issue #15 - distributed grid modified to expect upper and lower overlap 

 have same sizes at standard communication.
 -distributedGridSynchronizer was moved.
parent 1f622ebb

src/TNL/Meshes/DistributedMeshes/CMakeLists.txt

+0 −3
Original line number Diff line number Diff line
@@ -5,9 +5,6 @@ SET( headers BufferEntitiesHelper.h 
             DistributedGrid.h

             DistributedGrid.hpp

             DistributedGridSynchronizer.h

             DistributedGridSynchronizer_1D.h

             DistributedGridSynchronizer_2D.h

             DistributedGridSynchronizer_3D.h

             DistributedGridIO.h

             DistributedGridIO_MeshFunction.h

             DistributedGridIO_VectorField.h

src/TNL/Meshes/DistributedMeshes/DistributedGrid.h

+1 −8
Original line number Diff line number Diff line
@@ -62,9 +62,7 @@ class DistributedMesh< Grid< Dimension, Real, Device, Index > > 
      const GridType& getGlobalGrid() const;

      

      void setOverlaps( const SubdomainOverlapsType& lower,

                        const SubdomainOverlapsType& upper,

                        const SubdomainOverlapsType& globalLower,

                        const SubdomainOverlapsType& globalUpper );

                        const SubdomainOverlapsType& upper);

      

      void setupGrid( GridType& grid);
@@ -81,11 +79,6 @@ class DistributedMesh< Grid< Dimension, Real, Device, Index > > 
      

      const SubdomainOverlapsType& getUpperOverlap() const;



      //original overlaps set by user - same values return all subdomains..

      const SubdomainOverlapsType& getGlobalLowerOverlap() const;

      

      const SubdomainOverlapsType& getGlobalUpperOverlap() const;



      //number of elements of local sub domain WITHOUT overlap

      // TODO: getSubdomainDimensions

      const CoordinatesType& getLocalSize() const;

src/TNL/Meshes/DistributedMeshes/DistributedGrid.hpp

+1 −22
Original line number Diff line number Diff line
@@ -170,13 +170,8 @@ template< int Dimension, typename Real, typename Device, typename Index > 
void

DistributedMesh< Grid< Dimension, Real, Device, Index > >::

setOverlaps( const SubdomainOverlapsType& lower,

             const SubdomainOverlapsType& upper,

             const SubdomainOverlapsType& globalLower,

             const SubdomainOverlapsType& globalUpper )

             const SubdomainOverlapsType& upper)

{

   this->globalLowerOverlap = globalLower;

   this->globalUpperOverlap = globalUpper;



   this->lowerOverlap = lower;

   this->upperOverlap = upper;
@@ -263,22 +258,6 @@ getUpperOverlap() const 
   return this->upperOverlap;

};



template< int Dimension, typename Real, typename Device, typename Index >     

const typename DistributedMesh< Grid< Dimension, Real, Device, Index > >::CoordinatesType&

DistributedMesh< Grid< Dimension, Real, Device, Index > >::

getGlobalLowerOverlap() const

{

   return this->globalLowerOverlap;

};



template< int Dimension, typename Real, typename Device, typename Index >     

const typename DistributedMesh< Grid< Dimension, Real, Device, Index > >::CoordinatesType&

DistributedMesh< Grid< Dimension, Real, Device, Index > >::

getGlobalUpperOverlap() const

{

   return this->globalUpperOverlap;

};



template< int Dimension, typename Real, typename Device, typename Index >     

const typename DistributedMesh< Grid< Dimension, Real, Device, Index > >::CoordinatesType&

DistributedMesh< Grid< Dimension, Real, Device, Index > >::

src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h

+207 −3
Original line number Diff line number Diff line
@@ -10,6 +10,210 @@ 


#pragma once



//#include <TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_1D.h>

//#include <TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_2D.h>

#include <TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h>
 
#include <TNL/Meshes/Grid.h>

#include <TNL/Containers/Array.h>

#include <TNL/Meshes/DistributedMeshes/BufferEntitiesHelper.h>

#include <TNL/Meshes/DistributedMeshes/Directions.h>



namespace TNL {

namespace Functions{

template< typename Mesh,

          int MeshEntityDimension,

          typename Real  >

class MeshFunction;

}//Functions

}//TNL



namespace TNL {

namespace Meshes { 

namespace DistributedMeshes { 



template <typename RealType,

          int EntityDimension,

          int MeshDimension,

          typename Index,

          typename Device,

          typename GridReal>  

class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension, GridReal, Device, Index >,EntityDimension, RealType>>

{



   public:

      static constexpr int getMeshDimension() { return MeshDimension; };

      static constexpr int getNeighborCount() {return DirectionCount<MeshDimension>::get();};



      typedef typename Grid< MeshDimension, GridReal, Device, Index >::Cell Cell;

      // FIXME: clang does not like this (incomplete type error)

//      typedef typename Functions::MeshFunction< Grid< 3, GridReal, Device, Index >,EntityDimension, RealType> MeshFunctionType;

      typedef typename Grid< MeshDimension, GridReal, Device, Index >::DistributedMeshType DistributedGridType; 

      typedef typename DistributedGridType::CoordinatesType CoordinatesType;

      using SubdomainOverlapsType = typename DistributedGridType::SubdomainOverlapsType;

          

      DistributedMeshSynchronizer()

      {

         isSet = false;

      };



      DistributedMeshSynchronizer( DistributedGridType *distributedGrid )

      {

         isSet = false;

         setDistributedGrid( distributedGrid );

      };



      void setDistributedGrid( DistributedGridType *distributedGrid )

      {

         isSet = true;



         this->distributedGrid = distributedGrid;

         

         const SubdomainOverlapsType& lowerOverlap = this->distributedGrid->getLowerOverlap();

         const SubdomainOverlapsType& upperOverlap = this->distributedGrid->getUpperOverlap();

       

         const CoordinatesType& localBegin = this->distributedGrid->getLocalBegin(); 

         const CoordinatesType& localSize = this->distributedGrid->getLocalSize(); 

         const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize();         



         for( int i=0; i<this->getNeighborCount(); i++ )

         {

            Index sendSize=1;//sended and recieve areas has same size



            //bool isBoundary=( neighbor[ i ] == -1 );

            auto directions=Directions::template getXYZ<this->getMeshDimension()>(i);



            sendDimensions[i]=localSize;//sended and recieve areas has same dimensions

            sendBegin[i]=localBegin;

            recieveBegin[i]=localBegin;



            for(int j=0;j<this->getMeshDimension();j++)

            {

               if(directions[j]==-1)

               {

                  sendDimensions[i][j]=lowerOverlap[j];

                  recieveBegin[i][j]=0;

               }



               if(directions[j]==1)

               {

                  sendDimensions[i][j]=upperOverlap[j];

                  sendBegin[i][j]=localBegin[j]+localSize[j]-upperOverlap[j];

                  recieveBegin[i][j]=localBegin[j]+localSize[j];

               }



               sendSize*=sendDimensions[i][j];

            }



            sendSizes[ i ] = sendSize;

            sendBuffers[ i ].setSize( sendSize );

            recieveBuffers[ i ].setSize( sendSize);

         }

        

     }

        

      template< typename CommunicatorType,

                typename MeshFunctionType,

                typename PeriodicBoundariesMaskPointer = Pointers::SharedPointer< MeshFunctionType > >

      void synchronize( MeshFunctionType &meshFunction,

                        bool periodicBoundaries = false,

                        const PeriodicBoundariesMaskPointer& mask = PeriodicBoundariesMaskPointer( nullptr ) )

      {



         TNL_ASSERT_TRUE( isSet, "Synchronizer is not set, but used to synchronize" );



    	   if( !distributedGrid->isDistributed() ) return;

         

         const int *neighbors = distributedGrid->getNeighbors();

         const int *periodicNeighbors = distributedGrid->getPeriodicNeighbors();

         

        if( periodicBoundaries )

         {

            std::cerr<<"TOTALY DEMAGED by refactorization" << std::endl;

         }         

        

         //fill send buffers

         copyBuffers( meshFunction, 

            sendBuffers, sendBegin,sendDimensions,

            true,

            neighbors,

            periodicBoundaries,

            PeriodicBoundariesMaskPointer( nullptr ) ); // the mask is used only when receiving data );

        

         //async send and receive

         typename CommunicatorType::Request requests[2*this->getNeighborCount()];

         typename CommunicatorType::CommunicationGroup group;

         group=*((typename CommunicatorType::CommunicationGroup *)(distributedGrid->getCommunicationGroup()));

         int requestsCount( 0 );

		                

         //send everything, recieve everything 

         for( int i=0; i<this->getNeighborCount(); i++ )

            if( neighbors[ i ] != -1 )

            {

               requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(),  sendSizes[ i ], neighbors[ i ], group );

               requests[ requestsCount++ ] = CommunicatorType::IRecv( recieveBuffers[ i ].getData(),  sendSizes[ i ], neighbors[ i ], group );

            }

            else if( periodicBoundaries && sendSizes[ i ] !=0 )

      	   {

               requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(),  sendSizes[ i ], periodicNeighbors[ i ], group );

               requests[ requestsCount++ ] = CommunicatorType::IRecv( recieveBuffers[ i ].getData(),  sendSizes[ i ], periodicNeighbors[ i ], group );

            }



        //wait until send is done

        CommunicatorType::WaitAll( requests, requestsCount );



        //copy data from receive buffers

        copyBuffers(meshFunction, 

            recieveBuffers,recieveBegin,sendDimensions  ,          

            false,

            neighbors,

            periodicBoundaries,

            mask );

    }

    

   private:      

      template< typename Real_, 

                typename MeshFunctionType,

                typename PeriodicBoundariesMaskPointer >

      void copyBuffers( 

         MeshFunctionType& meshFunction,

         Containers::Array<Real_, Device, Index>* buffers,

         CoordinatesType* begins,

         CoordinatesType* sizes,

         bool toBuffer,

         const int* neighbor,

         bool periodicBoundaries,

         const PeriodicBoundariesMaskPointer& mask )

      {

         using Helper = BufferEntitiesHelper< MeshFunctionType, PeriodicBoundariesMaskPointer, this->getMeshDimension(), Real_, Device >;

       

         for(int i=0;i<this->getNeighborCount();i++)

         {

            bool isBoundary=( neighbor[ i ] == -1 );           

            if( ! isBoundary || periodicBoundaries )

            {

                  Helper::BufferEntities( meshFunction, mask, buffers[ i ].getData(), isBoundary, begins[i], sizes[i], toBuffer );

            }                  

         }      

      }

    

   private:

   

      Containers::Array<RealType, Device, Index> sendBuffers[getNeighborCount()];

      Containers::Array<RealType, Device, Index> recieveBuffers[getNeighborCount()];

      Containers::StaticArray< getNeighborCount(), int > sendSizes;

      Containers::StaticArray< getNeighborCount(), int > recieveSizes;



  

      CoordinatesType sendDimensions[getNeighborCount()];

      CoordinatesType recieveDimensions[getNeighborCount()];

      CoordinatesType sendBegin[getNeighborCount()];

      CoordinatesType recieveBegin[getNeighborCount()];

      

      DistributedGridType *distributedGrid;



      bool isSet;

    

};





} // namespace DistributedMeshes

} // namespace Meshes

} // namespace TNL

src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_1D.h

deleted100644 → 0
+0 −206
Original line number Diff line number Diff line 
/***************************************************************************

                          DistributedGridSynchronizer_1D.h  -  description

                             -------------------

    begin                : Aug 15, 2018

    copyright            : (C) 2018 by Tomas Oberhuber

    email                : tomas.oberhuber@fjfi.cvut.cz

 ***************************************************************************/



/* See Copyright Notice in tnl/Copyright */



#pragma once



#include <TNL/Meshes/Grid.h>

#include <TNL/Containers/Array.h>

#include <TNL/Meshes/DistributedMeshes/BufferEntitiesHelper.h>





namespace TNL {

namespace Functions{

template< typename Mesh,

          int MeshEntityDimension,

          typename Real  >

class MeshFunction;

}//Functions

}//TNL



namespace TNL {

namespace Meshes { 

namespace DistributedMeshes { 



template <typename Real,

          int EntityDimension,

          typename Index,

          typename Device,

          typename GridReal>  

class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 1, GridReal, Device, Index >, EntityDimension, Real > >

{



   public:

      using RealType = Real;

      typedef typename Grid< 1, GridReal, Device, Index >::Cell Cell;

      // FIXME: clang does not like this (incomplete type error)

//      typedef typename Functions::MeshFunction< Grid< 1, GridReal, Device, Index >,EntityDimension, RealType> MeshFunctionType;

      typedef typename Grid< 1, GridReal, Device, Index >::DistributedMeshType DistributedGridType;

      typedef typename DistributedGridType::CoordinatesType CoordinatesType;

      using SubdomainOverlapsType = typename DistributedGridType::SubdomainOverlapsType;



      DistributedMeshSynchronizer()

      {

          isSet = false;

      };



      DistributedMeshSynchronizer( DistributedGridType *distributedGrid )

      {

          isSet = false;

          setDistributedGrid( distributedGrid );

      };



      void setDistributedGrid( DistributedGridType *distributedGrid)

      {

         isSet=true;



         this->distributedGrid=distributedGrid;



         const SubdomainOverlapsType& lowerOverlap = distributedGrid->getLowerOverlap();

         const SubdomainOverlapsType& upperOverlap = distributedGrid->getUpperOverlap();

         const CoordinatesType& localSize = distributedGrid->getLocalSize(); 

         const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize();

         



         sendBuffers[ ZzYzXm ].setSize( lowerOverlap.x() );

         sendBuffers[ ZzYzXp ].setSize( upperOverlap.x() );

         receiveBuffers[ ZzYzXm ].setSize( lowerOverlap.x() );

         receiveBuffers[ ZzYzXp ].setSize( upperOverlap.x() );         



      };



      template< typename CommunicatorType,

                typename MeshFunctionType,

                typename PeriodicBoundariesMaskPointer = Pointers::SharedPointer< MeshFunctionType > >

      void synchronize( MeshFunctionType &meshFunction,

                        bool periodicBoundaries = false,

                        const PeriodicBoundariesMaskPointer& mask = PeriodicBoundariesMaskPointer( nullptr ) )

      {

         TNL_ASSERT_TRUE( isSet, "Synchronizer is not set, but used to synchronize" );

         

         if( !distributedGrid->isDistributed() )

            return;



         int totalSize = meshFunction.getMesh().getDimensions().x();

         const SubdomainOverlapsType& lowerOverlap = distributedGrid->getLowerOverlap();

         const SubdomainOverlapsType& upperOverlap = distributedGrid->getUpperOverlap();

         const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize();



         leftSource  = lowerOverlap.x();

         rightSource = localGridSize.x() - 2 * upperOverlap.x();

         leftDestination  = 0;

         rightDestination = localGridSize.x() - upperOverlap.x();

         

         const int *neighbors = distributedGrid->getNeighbors();

         const int *periodicNeighbors = distributedGrid->getPeriodicNeighbors();         

         

         if( periodicBoundaries )

         {

            if( neighbors[ ZzYzXm ] == -1 )

               swap( leftSource, leftDestination );

            if( neighbors[ ZzYzXp ] == -1 )

               swap( rightSource, rightDestination );

         }



         copyBuffers( meshFunction, sendBuffers, true,

                      leftSource, rightSource,

                      lowerOverlap, upperOverlap,

                      neighbors,

                      periodicBoundaries,

                      PeriodicBoundariesMaskPointer( nullptr ) ); // the mask is used only when receiving data 



         //async send

         typename CommunicatorType::Request requests[ 4 ];

         typename CommunicatorType::CommunicationGroup group;

         group=*((typename CommunicatorType::CommunicationGroup *)(distributedGrid->getCommunicationGroup()));

         int requestsCount( 0 );

         

         //send everything, recieve everything 

         if( neighbors[ ZzYzXm ] != -1 )

         {

            TNL_ASSERT_GE( sendBuffers[ ZzYzXm ].getSize(), lowerOverlap.x(), "" );

            TNL_ASSERT_GE( receiveBuffers[ ZzYzXm ].getSize(), lowerOverlap.x(), "" );

            requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ ZzYzXm ].getData(), lowerOverlap.x(), neighbors[ ZzYzXm ],0, group );

            requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ ZzYzXm ].getData(), lowerOverlap.x(), neighbors[ ZzYzXm ],0, group );

         }

         else if( periodicBoundaries )

         {

            TNL_ASSERT_GE( sendBuffers[ ZzYzXm ].getSize(), lowerOverlap.x(), "" );

            TNL_ASSERT_GE( receiveBuffers[ ZzYzXm ].getSize(), lowerOverlap.x(), "" );            

            requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ ZzYzXm ].getData(), lowerOverlap.x(), periodicNeighbors[ ZzYzXm ],1, group );

            requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ ZzYzXm ].getData(), lowerOverlap.x(), periodicNeighbors[ ZzYzXm ],1, group );

         }        



         if( neighbors[ ZzYzXp ] != -1 )

         {

            TNL_ASSERT_GE( sendBuffers[ ZzYzXp ].getSize(), upperOverlap.x(), "" );

            TNL_ASSERT_GE( receiveBuffers[ ZzYzXp ].getSize(), upperOverlap.x(), "" );

            requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ ZzYzXp ].getData(), upperOverlap.x(), neighbors[ ZzYzXp ], 0, group );

            requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ ZzYzXp ].getData(), upperOverlap.x(), neighbors[ ZzYzXp ], 0, group );

         }

         else if( periodicBoundaries )

         {

            TNL_ASSERT_GE( sendBuffers[ ZzYzXp ].getSize(), upperOverlap.x(), "" );

            TNL_ASSERT_GE( receiveBuffers[ ZzYzXp ].getSize(), upperOverlap.x(), "" );

            requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ ZzYzXp ].getData(), upperOverlap.x(), periodicNeighbors[ ZzYzXp ], 1, group );

            requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ ZzYzXp ].getData(), upperOverlap.x(), periodicNeighbors[ ZzYzXp ], 1, group );

         }

         

         //wait until send and receive is done

         CommunicatorType::WaitAll( requests, requestsCount );



         copyBuffers( meshFunction, receiveBuffers, false,

            leftDestination, rightDestination,

            lowerOverlap,

            upperOverlap,

            neighbors,

            periodicBoundaries,

            mask );

      }

      

   private:

      template< typename Real_,

                typename MeshFunctionType,

                typename PeriodicBoundariesMaskPointer >

      void copyBuffers( 

         MeshFunctionType& meshFunction,

         TNL::Containers::Array<Real_,Device>* buffers,

         bool toBuffer,

         int left, int right,

         const SubdomainOverlapsType& lowerOverlap,

         const SubdomainOverlapsType& upperOverlap,

         const int* neighbors,

         bool periodicBoundaries,

         const PeriodicBoundariesMaskPointer& mask )

      

      {

         typedef BufferEntitiesHelper< MeshFunctionType, PeriodicBoundariesMaskPointer, 1, Real_, Device > Helper;

         bool leftIsBoundary = ( neighbors[ ZzYzXm ] == -1 );

         bool rightIsBoundary = ( neighbors[ ZzYzXp ] == -1 );

         if( ! leftIsBoundary || periodicBoundaries )

            Helper::BufferEntities( meshFunction, mask, buffers[ ZzYzXm ].getData(), leftIsBoundary, left, lowerOverlap.x(), toBuffer );

         if( ! rightIsBoundary || periodicBoundaries )

            Helper::BufferEntities( meshFunction, mask, buffers[ ZzYzXp ].getData(), rightIsBoundary, right, upperOverlap.x(), toBuffer );

      }



      Containers::Array<RealType, Device> sendBuffers[ 2 ], receiveBuffers[ 2 ];



      DistributedGridType *distributedGrid;



      int leftSource;

      int rightSource;

      int leftDestination;

      int rightDestination;

      

      bool isSet;

};



} // namespace DistributedMeshes

} // namespace Meshes

} // namespace TNL