Commit c907bc84 authored by Jakub Klinkovský's avatar Jakub Klinkovský
Browse files

Refucktored DistributedGrid 

- pointer from Grid to DistributedGrid is gone, instead the
  DistributedGrid has a localGrid attribute
- Grid has new attributes: localBegin, localEnd, interiorBegin,
  interiorEnd (useful for traversers)
- all tests work except for CutMeshFunction
parent 27c6fb8d

src/Examples/Hamilton-Jacobi/Solvers/hamilton-jacobi/tnlDirectEikonalProblem_impl.h

+3 −3
Original line number Diff line number Diff line
@@ -127,13 +127,13 @@ setInitialCondition( const Config::ParameterContainer& parameters, 
  this->initialData->setMesh( this->getMesh() );

  if( CommunicatorType::isDistributed() )

  {

    std::cout<<"Nodes Distribution: " << initialData->getMesh().getDistributedMesh()->printProcessDistr() << std::endl;

    std::cout<<"Nodes Distribution: " << this->distributedMeshPointer->printProcessDistr() << std::endl;

    throw Exceptions::NotImplementedError( "PVTI reader is not implemented yet." );

//    if(distributedIOType==Meshes::DistributedMeshes::MpiIO)

//      Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType,Meshes::DistributedMeshes::MpiIO> ::load(inputFile, *initialData );

//    if(distributedIOType==Meshes::DistributedMeshes::LocalCopy)

//      Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType,Meshes::DistributedMeshes::LocalCopy> ::load(inputFile, *initialData );

    synchronizer.setDistributedGrid( initialData->getMesh().getDistributedMesh() );

    synchronizer.setDistributedGrid( &this->distributedMeshPointer.getData() );

    synchronizer.synchronize( *initialData );

  }

  else
@@ -190,7 +190,7 @@ tnlDirectEikonalProblem< Mesh, Communicator, Anisotropy, Real, Index >:: 
solve( DofVectorPointer& dofs )

{

   FastSweepingMethod< MeshType, Communicator,AnisotropyType > fsm;

   fsm.solve( this->getMesh(), u, anisotropy, initialData );

   fsm.solve( *this->getDistributedMesh(), this->getMesh(), u, anisotropy, initialData );



   makeSnapshot();

   return true;

src/Examples/Hamilton-Jacobi/Solvers/hamilton-jacobi/tnlFastSweepingMethod.h

+52 −53
Original line number Diff line number Diff line
@@ -58,7 +58,8 @@ class FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Communicator, 


    void setMaxIterations( const IndexType& maxIterations );



    void solve( const MeshPointer& mesh,

    void solve( const Meshes::DistributedMeshes::DistributedMesh< MeshType >& distributedMesh,

            const MeshPointer& mesh,

            MeshFunctionPointer& Aux,

            const AnisotropyPointer& anisotropy,

            MeshFunctionPointer& u );
@@ -106,7 +107,8 @@ class FastSweepingMethod< Meshes::Grid< 2, Real, Device, Index >, Communicator, 


    void setMaxIterations( const IndexType& maxIterations );



    void solve( const MeshPointer& mesh,

    void solve( const Meshes::DistributedMeshes::DistributedMesh< MeshType >& distributedMesh,

            const MeshPointer& mesh,

            MeshFunctionPointer& Aux,

            const AnisotropyPointer& anisotropy,

            const MeshFunctionPointer& u );
@@ -118,14 +120,12 @@ class FastSweepingMethod< Meshes::Grid< 2, Real, Device, Index >, Communicator, 


      const IndexType maxIterations;



      void setOverlaps( StaticVector& vecLowerOverlaps, StaticVector& vecUpperOverlaps,

              const MeshPointer& mesh);

      

      bool goThroughSweep( const StaticVector boundsFrom, const StaticVector boundsTo,

              MeshFunctionType& aux, const InterfaceMapType& interfaceMap,

              const AnisotropyPointer& anisotropy );



      void getInfoFromNeighbours( int& calculated, int& calculateAgain, const MeshPointer& mesh );

      void getInfoFromNeighbours( int& calculated, int& calculateAgain,

                                  const Meshes::DistributedMeshes::DistributedMesh< MeshType >& distributedMesh );

};



template< typename Real,
@@ -166,7 +166,8 @@ class FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Communicator, 


    void setMaxIterations( const IndexType& maxIterations );



    void solve( const MeshPointer& mesh,

    void solve( const Meshes::DistributedMeshes::DistributedMesh< MeshType >& distributedMesh,

            const MeshPointer& mesh,

            MeshFunctionPointer& Aux,

            const AnisotropyPointer& anisotropy,

            MeshFunctionPointer& u );
@@ -179,14 +180,12 @@ class FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Communicator, 


      const IndexType maxIterations;



      void setOverlaps( StaticVector& vecLowerOverlaps, StaticVector& vecUpperOverlaps,

              const MeshPointer& mesh);

      

      bool goThroughSweep( const StaticVector boundsFrom, const StaticVector boundsTo,

              MeshFunctionType& aux, const InterfaceMapType& interfaceMap,

              const AnisotropyPointer& anisotropy );



      void getInfoFromNeighbours( int& calculated, int& calculateAgain, const MeshPointer& mesh );

      void getInfoFromNeighbours( int& calculated, int& calculateAgain,

                                  const Meshes::DistributedMeshes::DistributedMesh< MeshType >& distributedMesh );

};

src/Examples/Hamilton-Jacobi/Solvers/hamilton-jacobi/tnlFastSweepingMethod1D_impl.h

+3 −2
Original line number Diff line number Diff line
@@ -58,7 +58,8 @@ template< typename Real, 
          typename Anisotropy >

void

FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Communicator, Anisotropy >::

solve( const MeshPointer& mesh,

solve( const Meshes::DistributedMeshes::DistributedMesh< MeshType >& distributedMesh,

       const MeshPointer& mesh,

       MeshFunctionPointer& Aux,

       const AnisotropyPointer& anisotropy,

       MeshFunctionPointer& u )

src/Examples/Hamilton-Jacobi/Solvers/hamilton-jacobi/tnlFastSweepingMethod2D_impl.h

+16 −32
Original line number Diff line number Diff line
@@ -58,7 +58,8 @@ template< typename Real, 
        typename Anisotropy >

void

FastSweepingMethod< Meshes::Grid< 2, Real, Device, Index >, Communicator, Anisotropy >::

solve( const MeshPointer& mesh,

solve( const Meshes::DistributedMeshes::DistributedMesh< MeshType >& distributedMesh,

       const MeshPointer& mesh,

        MeshFunctionPointer& Aux,

        const AnisotropyPointer& anisotropy,

        const MeshFunctionPointer& u )
@@ -69,8 +70,14 @@ solve( const MeshPointer& mesh, 
  interfaceMapPtr->setMesh( mesh );



  // Setting overlaps ( WITHOUT MPI SHOULD BE 0 )

  StaticVector vecLowerOverlaps, vecUpperOverlaps;

  setOverlaps( vecLowerOverlaps, vecUpperOverlaps, mesh );

  StaticVector vecLowerOverlaps = 0;

  StaticVector vecUpperOverlaps = 0;

  if( CommunicatorType::isDistributed() )

  {

    //Distributed mesh for MPI overlaps (without MPI null pointer)

    vecLowerOverlaps = distributedMesh.getLowerOverlap();

    vecUpperOverlaps = distributedMesh.getUpperOverlap();

  }



  std::cout << "Initiating the interface cells ..." << std::endl;

  BaseType::initInterface( u, auxPtr, interfaceMapPtr, vecLowerOverlaps, vecUpperOverlaps );
@@ -82,7 +89,7 @@ solve( const MeshPointer& mesh, 
  IndexType iteration( 0 );

  InterfaceMapType interfaceMap = *interfaceMapPtr;

  MeshFunctionType aux = *auxPtr;

  synchronizer.setDistributedGrid( aux.getMesh().getDistributedMesh() );

  synchronizer.setDistributedGrid( &distributedMesh );

  synchronizer.synchronize( aux ); //synchronize initialized overlaps



  std::cout << "Calculating the values ..." << std::endl;
@@ -368,7 +375,7 @@ solve( const MeshPointer& mesh, 
/**----------------------MPI-TO-DO---------------------------------------------**/

#ifdef HAVE_MPI

      if( CommunicatorType::isDistributed() ){

        getInfoFromNeighbours( calculatedBefore, calculateMPIAgain, mesh );

        getInfoFromNeighbours( calculatedBefore, calculateMPIAgain, distributedMesh );



        synchronizer.synchronize( aux );

      }
@@ -384,28 +391,6 @@ solve( const MeshPointer& mesh, 


// PROTECTED FUNCTIONS:



template< typename Real, typename Device, typename Index,

          typename Communicator, typename Anisotropy >

void

FastSweepingMethod< Meshes::Grid< 2, Real, Device, Index >, Communicator, Anisotropy >::

setOverlaps( StaticVector& vecLowerOverlaps, StaticVector& vecUpperOverlaps,

              const MeshPointer& mesh)

{

  vecLowerOverlaps[0] = 0; vecLowerOverlaps[1] = 0; vecUpperOverlaps[0] = 0; vecUpperOverlaps[1] = 0;

#ifdef HAVE_MPI

  if( CommunicatorType::isDistributed() ) //If we started solver with MPI

  {

    //Distributed mesh for MPI overlaps (without MPI null pointer)

    const Meshes::DistributedMeshes::DistributedMesh< MeshType >* meshPom = mesh->getDistributedMesh();

    vecLowerOverlaps = meshPom->getLowerOverlap();

    vecUpperOverlaps = meshPom->getUpperOverlap();

  }

#endif

}









template< typename Real, typename Device, typename Index,

          typename Communicator, typename Anisotropy >

bool
@@ -448,14 +433,13 @@ template< typename Real, typename Device, typename Index, 
          typename Communicator, typename Anisotropy >

void

FastSweepingMethod< Meshes::Grid< 2, Real, Device, Index >, Communicator, Anisotropy >::

getInfoFromNeighbours( int& calculatedBefore, int& calculateMPIAgain, const MeshPointer& mesh )

getInfoFromNeighbours( int& calculatedBefore, int& calculateMPIAgain,

                       const Meshes::DistributedMeshes::DistributedMesh< MeshType >& distributedMesh )

{

  Meshes::DistributedMeshes::DistributedMesh< MeshType >* meshDistr = mesh->getDistributedMesh();



  int calculateFromNeighbours[4] = {0,0,0,0};

  const int *neighbours = meshDistr->getNeighbors(); // Getting neighbors of distributed mesh

  const int *neighbours = distributedMesh.getNeighbors(); // Getting neighbors of distributed mesh

  MPI::Request *requestsInformation;

  requestsInformation = new MPI::Request[ meshDistr->getNeighborsCount() ];

  requestsInformation = new MPI::Request[ distributedMesh.getNeighborsCount() ];



  int neighCount = 0; // should this thread calculate again?

src/Examples/Hamilton-Jacobi/Solvers/hamilton-jacobi/tnlFastSweepingMethod3D_impl.h

+19 −36
Original line number Diff line number Diff line
@@ -58,7 +58,8 @@ template< typename Real, 
        typename Anisotropy >

void

FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Communicator, Anisotropy >::

solve( const MeshPointer& mesh,

solve( const Meshes::DistributedMeshes::DistributedMesh< MeshType >& distributedMesh,

       const MeshPointer& mesh,

       MeshFunctionPointer& Aux,

       const AnisotropyPointer& anisotropy,

       MeshFunctionPointer& u )
@@ -69,8 +70,14 @@ solve( const MeshPointer& mesh, 
  interfaceMapPtr->setMesh( mesh );



  // getting overlaps ( WITHOUT MPI SHOULD BE 0 )

  Containers::StaticVector< 3, IndexType > vecLowerOverlaps, vecUpperOverlaps;

  setOverlaps( vecLowerOverlaps, vecUpperOverlaps, mesh );

  StaticVector vecLowerOverlaps = 0;

  StaticVector vecUpperOverlaps = 0;

  if( CommunicatorType::isDistributed() )

  {

    //Distributed mesh for MPI overlaps (without MPI null pointer)

    vecLowerOverlaps = distributedMesh.getLowerOverlap();

    vecUpperOverlaps = distributedMesh.getUpperOverlap();

  }



  std::cout << "Initiating the interface cells ..." << std::endl;

  BaseType::initInterface( u, auxPtr, interfaceMapPtr, vecLowerOverlaps, vecUpperOverlaps );
@@ -81,7 +88,7 @@ solve( const MeshPointer& mesh, 
  IndexType iteration( 0 );

  MeshFunctionType aux = *auxPtr;

  InterfaceMapType interfaceMap = * interfaceMapPtr;

  synchronizer.setDistributedGrid( aux.getMesh().getDistributedMesh() );

  synchronizer.setDistributedGrid( &distributedMesh );

  synchronizer.synchronize( aux ); //synchronization of intial conditions



  while( iteration < this->maxIterations )
@@ -357,7 +364,7 @@ solve( const MeshPointer& mesh, 
#ifdef HAVE_MPI

      if( CommunicatorType::isDistributed() )

      {

        getInfoFromNeighbours( calculatedBefore, calculateMPIAgain, mesh );

        getInfoFromNeighbours( calculatedBefore, calculateMPIAgain, distributedMesh );



        // synchronizate the overlaps

        synchronizer.synchronize( aux );
@@ -379,29 +386,6 @@ solve( const MeshPointer& mesh, 


// PROTECTED FUNCTIONS:



template< typename Real, typename Device, typename Index,

          typename Communicator, typename Anisotropy >

void

FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Communicator, Anisotropy >::

setOverlaps( StaticVector& vecLowerOverlaps, StaticVector& vecUpperOverlaps,

              const MeshPointer& mesh)

{

  vecLowerOverlaps[0] = 0; vecLowerOverlaps[1] = 0; vecLowerOverlaps[2] = 0;

  vecUpperOverlaps[0] = 0; vecUpperOverlaps[1] = 0; vecUpperOverlaps[2] = 0;

#ifdef HAVE_MPI

  if( CommunicatorType::isDistributed() ) //If we started solver with MPI

  {

    //Distributed mesh for MPI overlaps (without MPI null pointer)

    const Meshes::DistributedMeshes::DistributedMesh< MeshType >* meshPom = mesh->getDistributedMesh();

    vecLowerOverlaps = meshPom->getLowerOverlap();

    vecUpperOverlaps = meshPom->getUpperOverlap();

  }

#endif

}









template< typename Real, typename Device, typename Index,

          typename Communicator, typename Anisotropy >

bool
@@ -450,15 +434,14 @@ template< typename Real, typename Device, typename Index, 
          typename Communicator, typename Anisotropy >

void

FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Communicator, Anisotropy >::

getInfoFromNeighbours( int& calculatedBefore, int& calculateMPIAgain, const MeshPointer& mesh )

getInfoFromNeighbours( int& calculatedBefore, int& calculateMPIAgain,

                       const Meshes::DistributedMeshes::DistributedMesh< MeshType >& distributedMesh )

{

  Meshes::DistributedMeshes::DistributedMesh< MeshType >* meshDistr = mesh->getDistributedMesh();



  int calculateFromNeighbours[6] = {0,0,0,0,0,0};



  const int *neighbours = meshDistr->getNeighbors(); // Getting neighbors of distributed mesh

  const int *neighbours = distributedMesh.getNeighbors(); // Getting neighbors of distributed mesh

  MPI::Request *requestsInformation;

  requestsInformation = new MPI::Request[ meshDistr->getNeighborsCount() ];

  requestsInformation = new MPI::Request[ distributedMesh.getNeighborsCount() ];



  int neighCount = 0; // should this thread calculate again?