diff --git a/CMakeLists.txt b/CMakeLists.txt index 82ae1a00819e154db93c1b4b1d139801ac3f897d..8ed064eb1de0ebce012daf328173b420ada0f0a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,7 +79,7 @@ endif() # set Debug/Release options set( CMAKE_CXX_FLAGS "-std=c++11 -pthread -Wall -Wno-unused-local-typedefs -Wno-unused-variable" ) -set( CMAKE_CXX_FLAGS_DEBUG "-g -rdynamic -ftemplate-backtrace-limit=0" ) +set( CMAKE_CXX_FLAGS_DEBUG "-g" ) set( CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native -mtune=native -DNDEBUG" ) #set( CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native -mtune=native -DNDEBUG -ftree-vectorizer-verbose=1 -ftree-vectorize -fopt-info-vec-missed -funroll-loops" ) # pass -rdynamic only in Debug mode @@ -229,7 +229,7 @@ if( ${WITH_CUDA} ) endif() endif() endif() - set( CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} ; ${CUDA_ARCH} -D_FORCE_INLINES -lineinfo ) + set( CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} ; ${CUDA_ARCH} -D_FORCE_INLINES ) # TODO: this is necessary only due to a bug in cmake set( CUDA_ADD_LIBRARY_OPTIONS -shared ) endif() diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_1D.h b/src/TNL/Meshes/DistributedMeshes/DistributedGrid_1D.h deleted file mode 100644 index b78e0d71e4c3b1982ef30df981c8d181f0e57dda..0000000000000000000000000000000000000000 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_1D.h +++ /dev/null @@ -1,55 +0,0 @@ -/*************************************************************************** - DistributedGrid_1D.h - description - ------------------- - begin : January 09, 2018 - copyright : (C) 2018 by Tomas Oberhuber - email : tomas.oberhuber@fjfi.cvut.cz - ***************************************************************************/ - -/* See Copyright Notice in tnl/Copyright */ - -#pragma once - -#include <TNL/Meshes/Grid.h> -#include <TNL/Logger.h> -#include <TNL/Meshes/DistributedMeshes/DistributedGrid_Base.h> - -namespace TNL { -namespace Meshes { -namespace DistributedMeshes { - -template< typename RealType, typename Device, typename Index > -class DistributedMesh< Grid< 1, RealType, Device, Index > > : public DistributedGrid_Base<1, RealType, Device, Index > -{ - - public: - /* using typename DistributedGrid_Base<1, RealType, Device, Index >::IndexType; - using typename DistributedGrid_Base<1, RealType, Device, Index >::GridType; - using typename DistributedGrid_Base<1, RealType, Device, Index >::PointType; - using typename DistributedGrid_Base<1, RealType, Device, Index >::CoordinatesType;*/ - - typedef typename DistributedGrid_Base<1, RealType, Device, Index >::CoordinatesType CoordinatesType; - typedef typename DistributedGrid_Base<1, RealType, Device, Index >::IndexType IndexType; - typedef typename DistributedGrid_Base<1, RealType, Device, Index >::GridType GridType; - typedef typename DistributedGrid_Base<1, RealType, Device, Index >::PointType PointType; - typedef typename DistributedGrid_Base<1, RealType, Device, Index >::SubdomainOverlapsType SubdomainOverlapsType; - - bool setup( const Config::ParameterContainer& parameters, - const String& prefix ); - - void setupGrid( GridType& grid ); - - String printProcessCoords() const; - - String printProcessDistr() const; - - void writeProlog( Logger& logger ) const; - -}; - - } //namespace DistributedMeshes - } // namespace Meshes -} // namespace TNL - -#include <TNL/Meshes/DistributedMeshes/DistributedGrid_1D.hpp> - diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_1D.hpp b/src/TNL/Meshes/DistributedMeshes/DistributedGrid_1D.hpp deleted file mode 100644 index 8a64ba7ec6576914824bce10a7e250202d6fee79..0000000000000000000000000000000000000000 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_1D.hpp +++ /dev/null @@ -1,70 +0,0 @@ -/*************************************************************************** - DistributedGrid_1D.hpp - description - ------------------- - begin : January 09, 2018 - copyright : (C) 2018 by Tomas Oberhuber - email : tomas.oberhuber@fjfi.cvut.cz - ***************************************************************************/ - -/* See Copyright Notice in tnl/Copyright */ - -#pragma once - -#include <cstdlib> - -namespace TNL { - namespace Meshes { - namespace DistributedMeshes { - -template< typename RealType, typename Device, typename Index > -bool -DistributedMesh< Grid< 1, RealType, Device, Index > >:: -setup( const Config::ParameterContainer& parameters, - const String& prefix ) -{ - this->domainDecomposition.x() = parameters.getParameter< int >( "grid-domain-decomposition-x" ); - return true; -} - -template< typename RealType, typename Device, typename Index > -void -DistributedMesh< Grid< 1, RealType, Device, Index > >:: -setupGrid( GridType& grid) -{ - TNL_ASSERT_TRUE(this->isSet,"DistributedGrid is not set, but used by SetupGrid"); - grid.setOrigin(this->localOrigin); - grid.setDimensions(this->localGridSize); - //compute local proportions by sideefect - grid.setSpaceSteps(this->spaceSteps); - grid.setDistMesh(this); -}; - -template< typename RealType, typename Device, typename Index > -String -DistributedMesh< Grid< 1, RealType, Device, Index > >:: -printProcessCoords() const -{ - return convertToString(this->rank); -}; - -template< typename RealType, typename Device, typename Index > -String -DistributedMesh< Grid< 1, RealType, Device, Index > >:: -printProcessDistr() const -{ - return convertToString(this->nproc); -}; - -template< typename RealType, typename Device, typename Index > -void -DistributedMesh< Grid< 1, RealType, Device, Index > >:: -writeProlog( Logger& logger ) const -{ - this->globalGrid.writeProlog( logger ); - logger.writeParameter( "Domain decomposition:", this->getDomainDecomposition() ); -} - - } //namespace DistributedMeshes - } // namespace Meshes -} // namespace TNL - diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_2D.h b/src/TNL/Meshes/DistributedMeshes/DistributedGrid_2D.h deleted file mode 100644 index 0d75c700791bca41e526284381f45760adf96544..0000000000000000000000000000000000000000 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_2D.h +++ /dev/null @@ -1,53 +0,0 @@ -/*************************************************************************** - DistributedGrid_2D.h - description - ------------------- - begin : January 09, 2018 - copyright : (C) 2018 by Tomas Oberhuber - email : tomas.oberhuber@fjfi.cvut.cz - ***************************************************************************/ - -/* See Copyright Notice in tnl/Copyright */ - -#pragma once - -#include <TNL/Meshes/Grid.h> -#include <TNL/Meshes/DistributedMeshes/DistributedGrid_Base.h> - -namespace TNL { -namespace Meshes { -namespace DistributedMeshes { - - -template< typename RealType, typename Device, typename Index > -class DistributedMesh< Grid< 2, RealType, Device, Index > >: public DistributedGrid_Base<2, RealType, Device, Index > -{ - public: - -/* using typename DistributedGrid_Base<2, RealType, Device, Index >::IndexType; - using typename DistributedGrid_Base<2, RealType, Device, Index >::GridType; - using typename DistributedGrid_Base<2, RealType, Device, Index >::PointType; - using typename DistributedGrid_Base<2, RealType, Device, Index >::CoordinatesType;*/ - typedef typename DistributedGrid_Base<2, RealType, Device, Index >::CoordinatesType CoordinatesType; - typedef typename DistributedGrid_Base<2, RealType, Device, Index >::IndexType IndexType; - typedef typename DistributedGrid_Base<2, RealType, Device, Index >::GridType GridType; - typedef typename DistributedGrid_Base<2, RealType, Device, Index >::PointType PointType; - typedef typename DistributedGrid_Base<2, RealType, Device, Index >::SubdomainOverlapsType SubdomainOverlapsType; - - bool setup( const Config::ParameterContainer& parameters, - const String& prefix ); - - void setupGrid( GridType& grid ); - - String printProcessCoords() const; - - String printProcessDistr() const; - - void writeProlog( Logger& logger ) const; - -}; - -} // namespace DistributedMeshes -} // namespace Meshes -} // namespace TNL - -#include <TNL/Meshes/DistributedMeshes/DistributedGrid_2D.hpp> diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_2D.hpp b/src/TNL/Meshes/DistributedMeshes/DistributedGrid_2D.hpp deleted file mode 100644 index b03060ebd23f6aede823d0efc7b873a72a4f490b..0000000000000000000000000000000000000000 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_2D.hpp +++ /dev/null @@ -1,73 +0,0 @@ -/*************************************************************************** - DistributedGrid_2D.hpp - description - ------------------- - begin : January 09, 2018 - copyright : (C) 2018 by Tomas Oberhuber - email : tomas.oberhuber@fjfi.cvut.cz - ***************************************************************************/ - -/* See Copyright Notice in tnl/Copyright */ - -#include <cstdlib> - -#include <TNL/Meshes/DistributedMeshes/DistributedGrid_Base.h> -#include <TNL/Communicators/MpiCommunicator.h> - -#pragma once - -namespace TNL { - namespace Meshes { - namespace DistributedMeshes { - -template< typename Real, typename Device, typename Index > -bool -DistributedMesh< Grid< 2, Real, Device, Index > >:: -setup( const Config::ParameterContainer& parameters, - const String& prefix ) -{ - this->domainDecomposition.x() = parameters.getParameter< int >( "grid-domain-decomposition-x" ); - this->domainDecomposition.y() = parameters.getParameter< int >( "grid-domain-decomposition-y" ); - return true; -} - -template< typename Real, typename Device, typename Index > -void -DistributedMesh< Grid< 2, Real, Device, Index > >:: -setupGrid( GridType& grid ) -{ - TNL_ASSERT_TRUE(this->isSet,"DistributedGrid is not set, but used by SetupGrid"); - grid.setOrigin( this->localOrigin ); - grid.setDimensions( this->localGridSize ); - //compute local proporions by sideefect - grid.setSpaceSteps( this->spaceSteps ); - grid.setDistMesh(this); -}; - -template< typename Real, typename Device, typename Index > -String -DistributedMesh< Grid< 2, Real, Device, Index > >:: -printProcessCoords() const -{ - return convertToString(this->subdomainCoordinates[0])+String("-")+convertToString(this->subdomainCoordinates[1]); -}; - -template< typename Real, typename Device, typename Index > -String -DistributedMesh< Grid< 2, Real, Device, Index > >:: -printProcessDistr() const -{ - return convertToString(this->domainDecomposition[0])+String("-")+convertToString(this->domainDecomposition[1]); -}; - - -template< typename Real, typename Device, typename Index > -void -DistributedMesh< Grid< 2, Real, Device, Index > >:: -writeProlog( Logger& logger ) const -{ - logger.writeParameter( "Domain decomposition:", this->getDomainDecomposition() ); -}; - - } //namespace DistributedMeshes - } // namespace Meshes -} // namespace TNL diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_3D.h b/src/TNL/Meshes/DistributedMeshes/DistributedGrid_3D.h deleted file mode 100644 index 31c7782027cb5c495356f116f6a589c9131d33b5..0000000000000000000000000000000000000000 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_3D.h +++ /dev/null @@ -1,57 +0,0 @@ -/*************************************************************************** - DistributedGrid_3D.h - description - ------------------- - begin : January 15, 2018 - copyright : (C) 2018 by Tomas Oberhuber - email : tomas.oberhuber@fjfi.cvut.cz - ***************************************************************************/ - -/* See Copyright Notice in tnl/Copyright */ - -#pragma once - -#include <TNL/Config/ConfigDescription.h> -#include <TNL/Meshes/Grid.h> -#include <TNL/Meshes/DistributedMeshes/DistributedGrid_Base.h> - -namespace TNL { -namespace Meshes { -namespace DistributedMeshes { - - -template< typename RealType, typename Device, typename Index > -class DistributedMesh<Grid< 3, RealType, Device, Index >> : public DistributedGrid_Base<3, RealType, Device, Index > -{ - - public: - - //using typename DistributedGrid_Base<3, RealType, Device, Index >::IndexType; - //using typename DistributedGrid_Base<3, RealType, Device, Index >::GridType; - //using typename DistributedGrid_Base<3, RealType, Device, Index >::PointType; - //using typename DistributedGrid_Base<3, RealType, Device, Index >::CoordinatesType; - typedef typename DistributedGrid_Base<3, RealType, Device, Index >::CoordinatesType CoordinatesType; - typedef typename DistributedGrid_Base<3, RealType, Device, Index >::IndexType IndexType; - typedef typename DistributedGrid_Base<3, RealType, Device, Index >::GridType GridType; - typedef typename DistributedGrid_Base<3, RealType, Device, Index >::PointType PointType; - typedef typename DistributedGrid_Base<3, RealType, Device, Index >::SubdomainOverlapsType SubdomainOverlapsType; - - static void configSetup( Config::ConfigDescription& config ); - - bool setup( const Config::ParameterContainer& parameters, - const String& prefix ); - - void setupGrid( GridType& grid); - - String printProcessCoords() const; - - String printProcessDistr() const; - - void writeProlog( Logger& logger ); - -}; - -} // namespace DistributedMeshes -} // namespace Meshes -} // namespace TNL - -#include <TNL/Meshes/DistributedMeshes/DistributedGrid_3D.hpp> diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_3D.hpp b/src/TNL/Meshes/DistributedMeshes/DistributedGrid_3D.hpp deleted file mode 100644 index b8799f4ed4133a891ff1b28f4fb385e84b429c50..0000000000000000000000000000000000000000 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_3D.hpp +++ /dev/null @@ -1,79 +0,0 @@ -/*************************************************************************** - DistributedGrid_3D.hpp - description - ------------------- - begin : January 09, 2018 - copyright : (C) 2018 by Tomas Oberhuber - email : tomas.oberhuber@fjfi.cvut.cz - ***************************************************************************/ - -/* See Copyright Notice in tnl/Copyright */ - -#pragma once - - -namespace TNL { - namespace Meshes { - namespace DistributedMeshes { - -template< typename RealType, typename Device, typename Index > -void -DistributedMesh< Grid< 3, RealType, Device, Index > >:: -configSetup( Config::ConfigDescription& config ) -{ - config.addEntry< int >( "grid-domain-decomposition-x", "Number of grid subdomains along x-axis.", 0 ); - config.addEntry< int >( "grid-domain-decomposition-y", "Number of grid subdomains along y-axis.", 0 ); - config.addEntry< int >( "grid-domain-decomposition-z", "Number of grid subdomains along z-axis.", 0 ); -} - -template< typename RealType, typename Device, typename Index > -bool -DistributedMesh< Grid< 3, RealType, Device, Index > >:: -setup( const Config::ParameterContainer& parameters, - const String& prefix ) -{ - this->domainDecomposition.x() = parameters.getParameter< int >( "grid-domain-decomposition-x" ); - this->domainDecomposition.y() = parameters.getParameter< int >( "grid-domain-decomposition-y" ); - this->domainDecomposition.z() = parameters.getParameter< int >( "grid-domain-decomposition-z" ); - return true; -} - -template< typename RealType, typename Device, typename Index > -void -DistributedMesh< Grid< 3, RealType, Device, Index > >:: -setupGrid( GridType& grid) -{ - TNL_ASSERT_TRUE(this->isSet,"DistributedGrid is not set, but used by SetupGrid"); - grid.setOrigin(this->localOrigin); - grid.setDimensions(this->localGridSize); - //compute local proportions by side efect - grid.setSpaceSteps(this->spaceSteps); - grid.setDistMesh(this); -}; - -template< typename RealType, typename Device, typename Index > -String -DistributedMesh< Grid< 3, RealType, Device, Index > >:: -printProcessCoords() const -{ - return convertToString(this->subdomainCoordinates[0])+String("-")+convertToString(this->subdomainCoordinates[1])+String("-")+convertToString(this->subdomainCoordinates[2]); -}; - -template< typename RealType, typename Device, typename Index > -String -DistributedMesh< Grid< 3, RealType, Device, Index > >:: -printProcessDistr() const -{ - return convertToString(this->domainDecomposition[0])+String("-")+convertToString(this->domainDecomposition[1])+String("-")+convertToString(this->domainDecomposition[2]); -}; - -template< typename RealType, typename Device, typename Index > -void -DistributedMesh< Grid< 3, RealType, Device, Index > >:: -writeProlog( Logger& logger ) -{ - logger.writeParameter( "Domain decomposition:", this->getDomainDecomposition() ); -} - - } //namespace DistributedMeshes - } // namespace Meshes -} // namespace TNL diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_Base.h b/src/TNL/Meshes/DistributedMeshes/DistributedGrid_Base.h deleted file mode 100644 index b3e958a42843ee60ed14512d46272b013e67cf9f..0000000000000000000000000000000000000000 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_Base.h +++ /dev/null @@ -1,150 +0,0 @@ -/*************************************************************************** - DistributedGrid_Base.h - part common for all Dimensionensions - ------------------- - begin : July 07, 2018 - copyright : (C) 2018 by Tomas Oberhuber - email : tomas.oberhuber@fjfi.cvut.cz - ***************************************************************************/ - - -/* See Copyright Notice in tnl/Copyright */ - -#pragma once - -#include <TNL/Meshes/Grid.h> -#include <TNL/Logger.h> -#include <TNL/Meshes/DistributedMeshes/Directions.h> - - -namespace TNL { -namespace Meshes { -namespace DistributedMeshes { - - - -template< int Dimension, - typename RealType, - typename Device, - typename Index > -class DistributedGrid_Base -{ - public: - - typedef Index IndexType; - typedef Grid< Dimension, RealType, Device, IndexType > GridType; - typedef typename GridType::PointType PointType; - typedef Containers::StaticVector< Dimension, IndexType > CoordinatesType; - typedef Containers::StaticVector< Dimension, IndexType > SubdomainOverlapsType; - - static constexpr int getMeshDimension() { return Dimension; }; - - static constexpr int getNeighborsCount() { return DirectionCount<Dimension>::get(); } //c++14 may use Directions::pow3(Dimension)-1 - - DistributedGrid_Base(); - - ~DistributedGrid_Base(); - - void setDomainDecomposition( const CoordinatesType& domainDecomposition ); - - const CoordinatesType& getDomainDecomposition() const; - - template< typename CommunicatorType > - void setGlobalGrid( const GridType& globalGrid ); - - void setOverlaps( const SubdomainOverlapsType& lower, - const SubdomainOverlapsType& upper ); - - bool isDistributed() const; - - // TODO: replace it with getLowerOverlap() and getUpperOverlap() - const CoordinatesType& getOverlap() const; - - //number of elements of local sub domain WITHOUT overlap - const CoordinatesType& getLocalSize() const; - - //Dimensionensions of global grid - const CoordinatesType& getGlobalSize() const; - - const GridType& getGlobalGrid() const; - - //coordinates of begin of local subdomain without overlaps in global grid - const CoordinatesType& getGlobalBegin() const; - - //number of elements of local sub domain WITH overlap - const CoordinatesType& getLocalGridSize() const; - - //coordinates of begin of local subdomain without overlaps in local grid - const CoordinatesType& getLocalBegin() const; - - const CoordinatesType& getSubdomainCoordinates() const; - - const PointType& getLocalOrigin() const; - const PointType& getSpaceSteps() const; - - //aka MPI-communcicator - void setCommunicationGroup(void * group); - void * getCommunicationGroup() const; - - template< int EntityDimension > - IndexType getEntitiesCount() const; - - template< typename Entity > - IndexType getEntitiesCount() const; - - const int* getNeighbors() const; - - const int* getPeriodicNeighbors() const; - - template<typename CommunicatorType, typename DistributedGridType> - bool SetupByCut(DistributedGridType &inputDistributedGrid, - Containers::StaticVector<Dimension, int> savedDimensions, - Containers::StaticVector<DistributedGridType::getMeshDimension()-Dimension,int> reducedDimensions, - Containers::StaticVector<DistributedGridType::getMeshDimension()-Dimension,IndexType> fixedIndexs); - - int getRankOfProcCoord(const CoordinatesType &nodeCoordinates) const; - - public: - - bool isThereNeighbor(const CoordinatesType &direction) const; - - void setupNeighbors(); - - void print( ostream& str ) const; - - GridType globalGrid; - PointType localOrigin; - CoordinatesType localBegin; - CoordinatesType localSize; - CoordinatesType localGridSize; - CoordinatesType overlap; - //CoordinatesType globalDimensions; - CoordinatesType globalBegin; - PointType spaceSteps; - - SubdomainOverlapsType lowerOverlap, upperOverlap; - - CoordinatesType domainDecomposition; - CoordinatesType subdomainCoordinates; - - int neighbors[ getNeighborsCount() ]; - - int periodicNeighbors[ getNeighborsCount() ]; - - IndexType Dimensions; - bool distributed; - - int rank; - int nproc; - - bool isSet; - - //aka MPI-communicator - void * communicationGroup; - -}; - -} // namespace DistributedMeshes -} // namespace Meshes -} // namespace TNL - -#include <TNL/Meshes/DistributedMeshes/DistributedGrid_Base.hpp> diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_Base.hpp b/src/TNL/Meshes/DistributedMeshes/DistributedGrid_Base.hpp deleted file mode 100644 index 25208cdff6abb2ca6faf307c26e9a4fcc3a25830..0000000000000000000000000000000000000000 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_Base.hpp +++ /dev/null @@ -1,505 +0,0 @@ -/*************************************************************************** - DistributedGrid_Base.hpp - description - ------------------- - begin : July 07, 2018 - copyright : (C) 2018 by Tomas Oberhuber - email : tomas.oberhuber@fjfi.cvut.cz - ***************************************************************************/ - -/* See Copyright Notice in tnl/Copyright */ - -#pragma once - -#include <cstdlib> -#include <TNL/StaticVectorFor.h> -#include <TNL/Communicators/MpiCommunicator.h> -#include <TNL/Exceptions/UnsupportedDimension.h> - -#include <iostream> - -#include "DistributedGrid_Base.h" - -namespace TNL { - namespace Meshes { - namespace DistributedMeshes { - - -template<int Dimension, typename RealType, typename Device, typename Index > -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -DistributedGrid_Base() - : domainDecomposition( 0 ), isSet( false ) {} - -template<int Dimension, typename RealType, typename Device, typename Index > -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -~DistributedGrid_Base() -{ - if(isSet && this->communicationGroup!=nullptr) - std::free(this->communicationGroup); -} - -template< int Dimension, typename RealType, typename Device, typename Index > -void -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -setDomainDecomposition( const CoordinatesType& domainDecomposition ) -{ - this->domainDecomposition = domainDecomposition; -} - -template< int Dimension, typename RealType, typename Device, typename Index > -const typename DistributedGrid_Base< Dimension, RealType, Device, Index >::CoordinatesType& -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getDomainDecomposition() const -{ - return this->domainDecomposition; -} - - -template< int Dimension, typename RealType, typename Device, typename Index > -template< typename CommunicatorType > -void -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -setGlobalGrid( const GridType &globalGrid ) -{ - if(this->isSet && this->communicationGroup != nullptr) - std::free(this->communicationGroup); - this->communicationGroup= std::malloc(sizeof(typename CommunicatorType::CommunicationGroup)); - - *((typename CommunicatorType::CommunicationGroup *)this->communicationGroup) = CommunicatorType::AllGroup; - auto group=*((typename CommunicatorType::CommunicationGroup *)this->communicationGroup); - - this->globalGrid = globalGrid; - this->isSet=true; - this->overlap.setValue( 1 ); // TODO: Remove this - its only for compatibility with old code - this->lowerOverlap.setValue( 0 ); - this->upperOverlap.setValue( 0 ); - - for( int i = 0; i < getNeighborsCount(); i++ ) - this->neighbors[ i ] = -1; - - this->Dimensions= GridType::getMeshDimension(); - this->spaceSteps=globalGrid.getSpaceSteps(); - this->distributed=false; - - if( CommunicatorType::IsInitialized() ) - { - this->rank=CommunicatorType::GetRank(group); - this->nproc=CommunicatorType::GetSize(group); - //use MPI only if have more than one process - if(this->nproc>1) - { - this->distributed=true; - } - } - - if( !this->distributed ) - { - this->subdomainCoordinates.setValue( 0 ); - this->domainDecomposition.setValue( 0 ); - this->localOrigin=globalGrid.getOrigin(); - this->localGridSize=globalGrid.getDimensions(); - this->localSize=globalGrid.getDimensions(); - this->globalBegin=CoordinatesType(0); - this->localBegin.setValue( 0 ); - return; - } - else - { - CoordinatesType numberOfLarger; - //compute node distribution - int dims[ Dimension ]; - for( int i = 0; i < Dimension; i++ ) - dims[ i ]= this->domainDecomposition[ i ]; - - - CommunicatorType::DimsCreate( this->nproc, Dimension, dims ); - for( int i = 0; i < Dimension; i++ ) - this->domainDecomposition[ i ] = dims[ i ]; - - // TODO: Make one formula for arbitraty dimension - switch( Dimension ) - { - case 1: - this->subdomainCoordinates[ 0 ] = this->rank; - break; - case 2: - this->subdomainCoordinates[ 0 ] = this->rank % this->domainDecomposition[ 0 ]; - this->subdomainCoordinates[ 1 ] = this->rank / this->domainDecomposition[ 0 ]; - break; - case 3: - this->subdomainCoordinates[ 2 ] = this->rank / ( this->domainDecomposition[0] * this->domainDecomposition[1] ); - this->subdomainCoordinates[ 1 ] = ( this->rank % ( this->domainDecomposition[0] * this->domainDecomposition[1] ) ) / this->domainDecomposition[0]; - this->subdomainCoordinates[ 0 ] = ( this->rank % ( this->domainDecomposition[0] * this->domainDecomposition[1] ) ) % this->domainDecomposition[0]; - break; - default: - throw Exceptions::UnsupportedDimension( Dimension ); - } - - for( int i = 0; i < Dimension; i++ ) - { - numberOfLarger[ i ] = globalGrid.getDimensions().x() % this->domainDecomposition[ i ]; - - this->localSize[ i ] = globalGrid.getDimensions()[ i ] / this->domainDecomposition[ i ]; - - if( numberOfLarger[ i ] > this->subdomainCoordinates[ i ] ) - this->localSize[ i ] += 1; - - if( numberOfLarger[ i ] > this->subdomainCoordinates[ i ] ) - this->globalBegin[ i ] = this->subdomainCoordinates[ i ] * this->localSize[ i ]; - else - this->globalBegin[ i ] = numberOfLarger[ i ] * ( this->localSize[ i ] + 1 ) + - ( this->subdomainCoordinates[ i ] - numberOfLarger[ i ] ) * this->localSize[ i ]; - } - - this->localGridSize = this->localSize; - this->setupNeighbors(); - } -} - -template< int Dimension, typename RealType, typename Device, typename Index > -void -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -setOverlaps( const SubdomainOverlapsType& lower, - const SubdomainOverlapsType& upper ) -{ - this->lowerOverlap = lower; - this->upperOverlap = upper; - - for( int i = 0; i < Dimension; i++ ) - { - this->localOrigin[ i ] = this->globalGrid.getOrigin()[ i ] + - this->globalGrid.getSpaceSteps()[ i ] * - ( this->globalBegin[ i ] - this->lowerOverlap[ i ] ); - - } - - this->localBegin = this->lowerOverlap; - this->localGridSize = this->localSize + this->lowerOverlap + this->upperOverlap; - //this->print( std::cerr ); -} - - -template< int Dimension, typename RealType, typename Device, typename Index > -const typename DistributedGrid_Base< Dimension, RealType, Device, Index >::CoordinatesType& -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getSubdomainCoordinates() const -{ - return this->subdomainCoordinates; -} - -template< int Dimension, typename RealType, typename Device, typename Index > -const typename DistributedGrid_Base< Dimension, RealType, Device, Index >::PointType& -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getLocalOrigin() const -{ - return this->localOrigin; -} - -template< int Dimension, typename RealType, typename Device, typename Index > -const typename DistributedGrid_Base< Dimension, RealType, Device, Index >::PointType& -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getSpaceSteps() const -{ - return this->spaceSteps; -} - -template< int Dimension, typename RealType, typename Device, typename Index > -bool -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -isDistributed() const -{ - return this->distributed; -}; - -template< int Dimension, typename RealType, typename Device, typename Index > -const typename DistributedGrid_Base< Dimension, RealType, Device, Index >::CoordinatesType& -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getOverlap() const -{ - return this->overlap; -}; - -template< int Dimension, typename RealType, typename Device, typename Index > -const typename DistributedGrid_Base< Dimension, RealType, Device, Index >::CoordinatesType& -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getLocalSize() const -{ - return this->localSize; -} - -template< int Dimension, typename RealType, typename Device, typename Index > -const typename DistributedGrid_Base< Dimension, RealType, Device, Index >::CoordinatesType& -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getGlobalSize() const -{ - return this->globalGrid.getDimensions(); -} - -template< int Dimension, typename RealType, typename Device, typename Index > -const typename DistributedGrid_Base< Dimension, RealType, Device, Index >::GridType& -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getGlobalGrid() const -{ - return this->globalGrid; -} - -template< int Dimension, typename RealType, typename Device, typename Index > -const typename DistributedGrid_Base< Dimension, RealType, Device, Index >::CoordinatesType& -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getGlobalBegin() const -{ - return this->globalBegin; -} - -template< int Dimension, typename RealType, typename Device, typename Index > -const typename DistributedGrid_Base< Dimension, RealType, Device, Index >::CoordinatesType& -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getLocalGridSize() const -{ - return this->localGridSize; -} - -template< int Dimension, typename RealType, typename Device, typename Index > -const typename DistributedGrid_Base< Dimension, RealType, Device, Index >::CoordinatesType& -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getLocalBegin() const -{ - return this->localBegin; -} - -template< int Dimension, typename RealType, typename Device, typename Index > - template< int EntityDimension > -Index -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getEntitiesCount() const -{ - return this->globalGrid. template getEntitiesCount< EntityDimension >(); -} - -template< int Dimension, typename RealType, typename Device, typename Index > - template< typename Entity > -Index -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getEntitiesCount() const -{ - return this->globalGrid. template getEntitiesCount< Entity >(); -} - -template< int Dimension, typename RealType, typename Device, typename Index > -void -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -setCommunicationGroup(void * group) -{ - this->communicationGroup=group; -} - -template< int Dimension, typename RealType, typename Device, typename Index > -void * -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getCommunicationGroup() const -{ - return this->communicationGroup; -} - -template< int Dimension, typename RealType, typename Device, typename Index > -int -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getRankOfProcCoord(const CoordinatesType &nodeCoordinates) const -{ - int DimensionOffset=1; - int ret=0; - for(int i=0;i<Dimension;i++) - { - ret += DimensionOffset*nodeCoordinates[i]; - DimensionOffset *= this->domainDecomposition[i]; - } - return ret; -} - -template< int Dimension, typename RealType, typename Device, typename Index > -bool -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -isThereNeighbor(const CoordinatesType &direction) const -{ - bool res=true; - for(int i=0;i<Dimension;i++) - { - if(direction[i]==-1) - res&= this->subdomainCoordinates[i]>0; - - if(direction[i]==1) - res&= this->subdomainCoordinates[i]<this->domainDecomposition[i]-1; - } - return res; - -} - -template< int Dimension, typename RealType, typename Device, typename Index > -void -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -setupNeighbors() -{ - int *neighbors = this->neighbors; - - for( int i = 0; i < getNeighborsCount(); i++ ) - { - auto direction = Directions::template getXYZ< Dimension >( i ); - auto coordinates = this->subdomainCoordinates+direction; - if( this->isThereNeighbor( direction ) ) - this->neighbors[ i ] = this->getRankOfProcCoord( coordinates ); - else - this->neighbors[ i ] = -1; - - // Handling periodic neighbors - for( int d = 0; d < Dimension; d++ ) - { - if( coordinates[ d ] == -1 ) - coordinates[ d ] = this->domainDecomposition[ d ] - 1; - if( coordinates[ d ] == this->domainDecomposition[ d ] ) - coordinates[ d ] = 0; - this->periodicNeighbors[ i ] = this->getRankOfProcCoord( coordinates ); - } - - //std::cout << "Setting i-th neighbour to " << neighbors[ i ] << " and " << periodicNeighbors[ i ] << std::endl; - } -} - -template< int Dimension, typename RealType, typename Device, typename Index > -const int* -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getNeighbors() const -{ - TNL_ASSERT_TRUE(this->isSet,"DistributedGrid is not set, but used by getNeighbors"); - return this->neighbors; -} - -template< int Dimension, typename RealType, typename Device, typename Index > -const int* -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -getPeriodicNeighbors() const -{ - TNL_ASSERT_TRUE(this->isSet,"DistributedGrid is not set, but used by getNeighbors"); - return this->periodicNeighbors; -} - -template< int Dimension, typename RealType, typename Device, typename Index > - template<typename CommunicatorType, typename DistributedGridType > -bool -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -SetupByCut(DistributedGridType &inputDistributedGrid, - Containers::StaticVector<Dimension, int> savedDimensions, - Containers::StaticVector<DistributedGridType::getMeshDimension()-Dimension,int> reducedDimensions, - Containers::StaticVector<DistributedGridType::getMeshDimension()-Dimension,IndexType> fixedIndexs) -{ - - int coDimensionension=DistributedGridType::getMeshDimension()-Dimension; - - bool isInCut=true; - for(int i=0;i<coDimensionension; i++) - { - auto begin=inputDistributedGrid.getGlobalBegin(); - auto size= inputDistributedGrid.getLocalSize(); - isInCut &= fixedIndexs[i]>begin[reducedDimensions[i]] && fixedIndexs[i]< (begin[reducedDimensions[i]]+size[reducedDimensions[i]]); - } - - //create new group with used nodes - typename CommunicatorType::CommunicationGroup *oldGroup=(typename CommunicatorType::CommunicationGroup *)(inputDistributedGrid.getCommunicationGroup()); - if(this->isSet && this->communicationGroup != nullptr) - free(this->communicationGroup); - this->communicationGroup = std::malloc(sizeof(typename CommunicatorType::CommunicationGroup)); - - if(isInCut) - { - this->isSet=true; - - auto fromGlobalMesh=inputDistributedGrid.getGlobalGrid(); - //set global grid - typename GridType::PointType outOrigin; - typename GridType::PointType outProportions; - typename GridType::CoordinatesType outDimensions; - - for(int i=0; i<Dimension;i++) - { - outOrigin[i]=fromGlobalMesh.getOrigin()[savedDimensions[i]]; - outProportions[i]=fromGlobalMesh.getProportions()[savedDimensions[i]]; - outDimensions[i]=fromGlobalMesh.getDimensions()[savedDimensions[i]]; - - this->domainDecomposition[i]=inputDistributedGrid.getDomainDecomposition()[savedDimensions[i]]; - this->subdomainCoordinates[i]=inputDistributedGrid.getSubdomainCoordinates()[savedDimensions[i]]; - - this->overlap[i]=inputDistributedGrid.getOverlap()[savedDimensions[i]]; - this->localSize[i]=inputDistributedGrid.getLocalSize()[savedDimensions[i]]; - this->globalBegin[i]=inputDistributedGrid.getGlobalBegin()[savedDimensions[i]]; - this->localGridSize[i]=inputDistributedGrid.getLocalGridSize()[savedDimensions[i]]; - this->localBegin[i]=inputDistributedGrid.getLocalBegin()[savedDimensions[i]]; - - this->localOrigin[i]=inputDistributedGrid.getLocalOrigin()[savedDimensions[i]]; - this->spaceSteps[i]=inputDistributedGrid.getSpaceSteps()[savedDimensions[i]]; - } - - int newRank= getRankOfProcCoord(this->subdomainCoordinates); - - CommunicatorType::CreateNewGroup(isInCut,newRank,*oldGroup ,*((typename CommunicatorType::CommunicationGroup*) this->communicationGroup)); - - setupNeighbors(); - - - - bool isDistributed=false; - for(int i=0;i<Dimension; i++) - { - isDistributed|=(domainDecomposition[i]>1); - } - - this->distributed=isDistributed; - - this->globalGrid.setDimensions(outDimensions); - this->globalGrid.setDomain(outOrigin,outProportions); - - return true; - } - else - { - CommunicatorType::CreateNewGroup(isInCut,0,*oldGroup ,*((typename CommunicatorType::CommunicationGroup*) this->communicationGroup)); - } - - return false; -} - -template< int Dimension, typename RealType, typename Device, typename Index > -void -DistributedGrid_Base< Dimension, RealType, Device, Index >:: -print( ostream& str ) const -{ - using Communicator = Communicators::MpiCommunicator; - for( int j = 0; j < Communicator::GetSize( Communicator::AllGroup ); j++ ) - { - if( j == Communicator::GetRank( Communicator::AllGroup ) ) - { - str << "Node : " << Communicator::GetRank( Communicator::AllGroup ) << std::endl - << " localOrigin : " << localOrigin << std::endl - << " localBegin : " << localBegin << std::endl - << " localSize : " << localSize << std::endl - << " localGridSize : " << localGridSize << std::endl - << " overlap : " << overlap << std::endl - << " globalBegin : " << globalBegin << std::endl - << " spaceSteps : " << spaceSteps << std::endl - << " lowerOverlap : " << lowerOverlap << std::endl - << " upperOverlap : " << upperOverlap << std::endl - << " domainDecomposition : " << domainDecomposition << std::endl - << " subdomainCoordinates : " << subdomainCoordinates << std::endl - << " neighbors : "; - for( int i = 0; i < getNeighborsCount(); i++ ) - str << " " << neighbors[ i ]; - str << std::endl; - str << " periodicNeighbours : "; - for( int i = 0; i < getNeighborsCount(); i++ ) - str << " " << periodicNeighbors[ i ]; - str << std::endl; - } - Communicator::Barrier( Communicator::AllGroup ); - } -} - - } //namespace DistributedMeshes - } // namespace Meshes -} // namespace TNL diff --git a/src/TNL/Solvers/IterativeSolver_impl.h b/src/TNL/Solvers/IterativeSolver_impl.h index e7430bb1051167ae2ce260a8179350a16b3fcbe4..f96b7c2353b7a4beba08931ff266ab0bde90cb94 100644 --- a/src/TNL/Solvers/IterativeSolver_impl.h +++ b/src/TNL/Solvers/IterativeSolver_impl.h @@ -111,12 +111,7 @@ bool IterativeSolver< Real, Index> :: checkNextIteration() { this->refreshSolverMonitor(); - if( -#ifndef HAVE_CUDA - std::isnan( this->getResidue() ) || - // TODO: Fix this !!!! - // this does not work (at least) with nvcc 8.0 and g++ 5.4 -#endif + if( std::isnan( this->getResidue() ) || this->getIterations() > this->getMaxIterations() || ( this->getResidue() > this->getDivergenceResidue() && this->getIterations() >= this->getMinIterations() ) || ( this->getResidue() < this->getConvergenceResidue() && this->getIterations() >= this->getMinIterations() ) ) diff --git a/src/TNL/Solvers/Linear/tnlJacobiSolver.h b/src/TNL/Solvers/Linear/tnlJacobiSolver.h deleted file mode 100644 index 56f3259518351ce97b601279435906884e811ddb..0000000000000000000000000000000000000000 --- a/src/TNL/Solvers/Linear/tnlJacobiSolver.h +++ /dev/null @@ -1,126 +0,0 @@ -/*************************************************************************** - Jacobi.h - description - ------------------- - begin : Jul 30, 2007 - copyright : (C) 2007 by Tomas Oberhuber - email : tomas.oberhuber@fjfi.cvut.cz - ***************************************************************************/ - -/*************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ - -#pragma once - -#include <math.h> -#include <TNL/Object.h> -#include <TNL/Solvers/Preconditioners/Dummy.h> -#include <TNL/Solvers/IterativeSolver.h> -#include <TNL/Solvers/Linear/LinearResidueGetter.h> - -namespace TNL { - namespace Solvers { - namespace Linear { - -template< typename Matrix, - typename Preconditioner = DummyPreconditioner< typename Matrix :: RealType, - typename Matrix :: DeviceType, - typename Matrix :: IndexType> > -class Jacobi : public Object, - public IterativeSolver< typename Matrix :: RealType, - typename Matrix :: IndexType > -{ - public: - - typedef typename Matrix::RealType RealType; - typedef typename Matrix::IndexType IndexType; - typedef typename Matrix::DeviceType DeviceType; - typedef Matrix MatrixType; - typedef Preconditioner PreconditionerType; - - - Jacobi():omega(0){} - - String getType() const - { - return String( "Jacobi< " ) + this -> matrix -> getType() + ", " + this -> preconditioner -> getType() + " >"; - } - - static void configSetup( tnlConfigDescription& config, - const String& prefix = "" ) - { - config.addEntry< double >( prefix + "jacobi-omega", "Relaxation parameter of the weighted/damped Jacobi method.", 1.0 ); - } - - bool setup( const Config::ParameterContainer& parameters, - const String& prefix = "" ) - { - IterativeSolver< RealType, IndexType >::setup( parameters, prefix ); - this->setOmega( parameters.getParameter< double >( prefix + "jacobi-omega" ) ); - if( this->omega <= 0.0 || this->omega > 2.0 ) - { - cerr << "Warning: The Jacobi method parameter omega is out of interval (0,2). The value is " << this->omega << " the method will not converge." << endl; - } - return true; - } - - void setOmega( const RealType& omega ) - { - this->omega = omega; - } - - const RealType& getOmega() const - { - return omega; - } - - void setMatrix( const MatrixType& matrix ) - { - this -> matrix = &matrix; - } - - void setPreconditioner( const Preconditioner& preconditioner ) - { - this -> preconditioner = &preconditioner; - } - - template< typename Vector, - typename ResidueGetter = LinearResidueGetter< Matrix, Vector > > - bool solve( const Vector& b, Vector& x, Vector& aux) - { - const IndexType size = matrix -> getRows(); - - this -> resetIterations(); - this -> setResidue( this -> getConvergenceResidue() + 1.0 ); - - RealType bNorm = b. lpNorm( ( RealType ) 2.0 ); - aux = x; - while( this->nextIteration() ) - { - for( IndexType row = 0; row < size; row ++ ) - matrix->performJacobiIteration( b, row, x, aux, this->getOmega() ); - for( IndexType row = 0; row < size; row ++ ) - matrix->performJacobiIteration( b, row, aux, x, this->getOmega() ); - this -> setResidue( ResidueGetter :: getResidue( *matrix, x, b, bNorm ) ); - } - this -> setResidue( ResidueGetter :: getResidue( *matrix, x, b, bNorm ) ); - this -> refreshSolverMonitor(); - return this->checkConvergence(); - } - - protected: - - RealType omega; - const MatrixType* matrix; - const PreconditionerType* preconditioner; - -}; - - } // namespace Linear - } // namespace Solvers -} // namespace TNL diff --git a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp index 6075f721ad2ca3311b45757362e81db8eeda32be..38276dd5436d83200b75f69a5b6b5221f23e1126 100644 --- a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp +++ b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp @@ -297,88 +297,6 @@ void check_Overlap_2D(int rank, GridType &grid, DofType &dof, typename DofType:: } } -/*Expect 9 process - */ -template<typename DofType,typename GridType> -void checkNeighbor_2D(int rank, GridType &grid, DofType &dof) -{ - if(rank==0)//Up Left - { - checkRightEdge(grid,dof,true,false,1); - checkDownEdge(grid,dof,true,false,3); - checkConner(grid,dof,false,false,4); - - } - - if(rank==1)//Up Center - { - checkLeftEdge(grid,dof,true,false,0); - checkRightEdge(grid,dof,true,false,2); - checkConner(grid,dof,false,true,3); - checkDownEdge(grid,dof,false,false,4); - checkConner(grid,dof,false,false,5); - } - - if(rank==2)//Up Right - { - checkLeftEdge(grid,dof,true,false,1); - checkConner(grid,dof,false,true,4); - checkDownEdge(grid,dof,false,true,5); - } - - if(rank==3)//Center Left - { - checkUpEdge(grid,dof,true,false,0); - checkConner(grid,dof,true,false,1); - checkRightEdge(grid,dof,false,false,4); - checkDownEdge(grid,dof,true,false,6); - checkConner(grid,dof,false,false,7); - } - - if(rank==4)//Center Center - { - checkConner(grid,dof,true,true,0); - checkUpEdge(grid,dof,false,false,1); - checkConner(grid,dof,true,false,2); - checkLeftEdge(grid,dof,false,false,3); - checkRightEdge(grid,dof,false,false,5); - checkConner(grid,dof,false,true,6); - checkDownEdge(grid,dof,false,false,7); - checkConner(grid,dof,false,false,8); - } - - if(rank==5)//Center Right - { - checkConner(grid,dof,true,true,1); - checkUpEdge(grid,dof,false,true,2); - checkLeftEdge(grid,dof,false,false,4); - checkConner(grid,dof,false,true,7); - checkDownEdge(grid,dof,false,true,8); - } - - if(rank==6)//Down Left - { - checkUpEdge(grid,dof,true,false,3); - checkConner(grid,dof,true,false,4); - checkRightEdge(grid,dof,false,true,7); - } - - if(rank==7) //Down Center - { - checkConner(grid,dof,true,true,3); - checkUpEdge(grid,dof,false,false,4); - checkConner(grid,dof,true,false,5); - checkLeftEdge(grid,dof,false,true,6); - checkRightEdge(grid,dof,false,true,8); - } - - if(rank==8) //Down Right - { - checkConner(grid,dof,true,true,4); - checkUpEdge(grid,dof,false,true,5); - checkLeftEdge(grid,dof,false,true,7); - } -} template<typename DofType,typename GridType>