diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 48e304c9b736a0587cefc313ba0a3d4b35438232..50697564b0d47c4327776fa48d1702adf6dd9812 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -41,7 +41,7 @@ stages:
-DWITH_COVERAGE=${WITH_COVERAGE}
-DWITH_EXAMPLES=${WITH_EXAMPLES}
- make
- - make test
+ - make test CTEST_OUTPUT_ON_FAILURE=1
- make install
- popd
variables:
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c8ec69cdd89b9c0801416001d7134b8c45fdbfb7..a35e94e07823e22d0405525ed08a5a2a7a387188 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -50,7 +50,7 @@ endif()
# set Debug/Release options
set( CMAKE_CXX_FLAGS "-std=c++11 -pthread -Wall -Wno-unused-local-typedefs -Wno-unused-variable" )
-set( CMAKE_CXX_FLAGS_DEBUG "-g" )
+set( CMAKE_CXX_FLAGS_DEBUG "-g -rdynamic -ftemplate-backtrace-limit=0" )
set( CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native -mtune=native -DNDEBUG" )
#set( CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native -mtune=native -DNDEBUG -ftree-vectorizer-verbose=1 -ftree-vectorize -fopt-info-vec-missed -funroll-loops" )
# pass -rdynamic only in Debug mode
@@ -63,7 +63,7 @@ if( CXX_COMPILER_NAME MATCHES "icpc" )
message( "Intel compiler detected..." )
set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_ICPC -wd2568 -wd2571 -wd2570")
#####
- # Ckeck for MIC
+ # Check for MIC
#
if( WITH_MIC STREQUAL "yes" )
message( "Compile MIC support..." )
@@ -86,6 +86,27 @@ if( DEFINED ENV{CI_JOB_NAME} )
endif()
endif()
+#####
+# Check for MPI -- poznej podle vraperu compileru -- da se testovat preklad bez MPI
+#
+if( ${CXX_COMPILER_NAME} STREQUAL "mpic++" )
+ message( "MPI compiler detected." )
+ set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_MPI" )
+ set( CUDA_HOST_COMPILER "mpic++" )
+
+endif()
+
+####
+# Check for MPI -- not working
+#
+#find_package( MPI )
+#if( MPI_CXX_FOUND )
+ # set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_MPI" )
+ # message( "MPI headers found -- ${MPI_CXX_INCLUDE_PATH}")
+ # message( "MPI link flags -- ${MPI_CXX_LINK_FLAGS}")
+ # message( "MPI libraries-- ${MPI_CXX_LIBRARIES}")
+#endif()
+
#####
# Check for CUDA
#
@@ -181,7 +202,7 @@ if( WITH_CUDA STREQUAL "yes" )
if( NOT WITH_CUSPARSE STREQUAL "no" )
find_path( CUSPARSE_INCLUDE_DIR cusparse.h
/usr/local/cuda/include
- ${CUDA_INCLUDE_DIR}
+ ${CUDA_INCLUDE_DIRS}
DOC "CUSPARSE headers." )
if( ${CUSPARSE_INCLUDE_DIR} STREQUAL "CUSPARSE_INCLUDE_DIR-NOTFOUND" )
message( "CUSPARSE not found." )
@@ -205,15 +226,6 @@ if( OPENMP_FOUND AND WITH_OPENMP STREQUAL "yes" )
set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_OPENMP ${OpenMP_CXX_FLAGS}" )
endif()
-####
-# Check for MPI
-#
-#find_package( MPI )
-#if( MPI_CXX_FOUND )
-# set( BUILD_MPI TRUE )
-# message( "MPI headers found -- ${MPI_CXX_INCLUDE_PATH}")
-#endif()
-
find_package( DCMTK )
if( DCMTK_FOUND )
set( HAVE_DCMTK_H "#define HAVE_DCMTK_H 1" )
diff --git a/build b/build
index 305a7e5d1ab69130e561afe16046e6329736c45e..328a3fe1f251aa3fa62f04ffa4b508a7b8f739ff 100755
--- a/build
+++ b/build
@@ -162,7 +162,7 @@ fi
if test ${WITH_TESTS} = "yes";
then
- make test
+ make test CTEST_OUTPUT_ON_FAILURE=1
if test $? != 0; then
echo "Error: Some test did not pass successfuly."
fi
diff --git a/examples/heat-equation/tnl-heat-equation-eoc.h b/examples/heat-equation/tnl-heat-equation-eoc.h
index 57ec6536fee8c43add54c4122a1c328ca489b508..593d90b710cf9e2c7e1a5b2e27b145bba22ec1e8 100644
--- a/examples/heat-equation/tnl-heat-equation-eoc.h
+++ b/examples/heat-equation/tnl-heat-equation-eoc.h
@@ -44,7 +44,8 @@ template< typename Real,
typename Index,
typename MeshType,
typename MeshConfig,
- typename SolverStarter >
+ typename SolverStarter,
+ typename CommunicatorType >
class heatEquationSetter
{
public:
@@ -64,7 +65,7 @@ class heatEquationSetter
typedef HeatEquationEocRhs< ExactOperator, TestFunction > RightHandSide;
typedef Containers::StaticVector < MeshType::getMeshDimension(), Real > Point;
typedef Operators::DirichletBoundaryConditions< MeshType, TestFunction, Dimension, Real, Index > BoundaryConditions;
- typedef HeatEquationEocProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Solver;
+ typedef HeatEquationEocProblem< MeshType, BoundaryConditions, RightHandSide, CommunicatorType, ApproximateOperator > Solver;
SolverStarter solverStarter;
return solverStarter.template run< Solver >( parameters );
};
diff --git a/examples/heat-equation/tnl-heat-equation.h b/examples/heat-equation/tnl-heat-equation.h
index 048abb409ac0d4567a22d87817edca057120b328..dea5fce6ac2f1c1d1fa7bf9e18bcdd37b58728cf 100644
--- a/examples/heat-equation/tnl-heat-equation.h
+++ b/examples/heat-equation/tnl-heat-equation.h
@@ -56,7 +56,8 @@ template< typename Real,
typename Index,
typename MeshType,
typename MeshConfig,
- typename SolverStarter >
+ typename SolverStarter,
+ typename CommunicatorType >
class heatEquationSetter
{
public:
@@ -78,12 +79,12 @@ class heatEquationSetter
if( boundaryConditionsType == "dirichlet" )
{
typedef Operators::DirichletBoundaryConditions< MeshType, Constant > BoundaryConditions;
- typedef HeatEquationProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;
+ typedef HeatEquationProblem< MeshType, BoundaryConditions, RightHandSide,CommunicatorType, ApproximateOperator > Problem;
SolverStarter solverStarter;
return solverStarter.template run< Problem >( parameters );
}
typedef Operators::NeumannBoundaryConditions< MeshType, Constant, Real, Index > BoundaryConditions;
- typedef HeatEquationProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;
+ typedef HeatEquationProblem< MeshType, BoundaryConditions, RightHandSide, CommunicatorType, ApproximateOperator > Problem;
SolverStarter solverStarter;
return solverStarter.template run< Problem >( parameters );
}
@@ -91,12 +92,12 @@ class heatEquationSetter
if( boundaryConditionsType == "dirichlet" )
{
typedef Operators::DirichletBoundaryConditions< MeshType, MeshFunction > BoundaryConditions;
- typedef HeatEquationProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;
+ typedef HeatEquationProblem< MeshType, BoundaryConditions, RightHandSide,CommunicatorType, ApproximateOperator > Problem;
SolverStarter solverStarter;
return solverStarter.template run< Problem >( parameters );
}
typedef Operators::NeumannBoundaryConditions< MeshType, MeshFunction, Real, Index > BoundaryConditions;
- typedef HeatEquationProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;
+ typedef HeatEquationProblem< MeshType, BoundaryConditions, RightHandSide,CommunicatorType, ApproximateOperator > Problem;
SolverStarter solverStarter;
return solverStarter.template run< Problem >( parameters );
};
diff --git a/examples/inviscid-flow/euler.h b/examples/inviscid-flow/euler.h
index 162cff09e9db0855a9a0c59e649d04f62850ed11..67bd825ff700fb5ef9e851607133047a5357dc13 100644
--- a/examples/inviscid-flow/euler.h
+++ b/examples/inviscid-flow/euler.h
@@ -51,7 +51,8 @@ template< typename Real,
typename Index,
typename MeshType,
typename ConfigTag,
- typename SolverStarter >
+ typename SolverStarter,
+ typename CommunicatorType >
class eulerSetter
{
public:
@@ -79,12 +80,12 @@ class eulerSetter
if( boundaryConditionsType == "dirichlet" )
{
typedef Operators::DirichletBoundaryConditions< MeshType, Constant, MeshType::getMeshDimension(), Real, Index > BoundaryConditions;
- typedef eulerProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;
+ typedef eulerProblem< MeshType, BoundaryConditions, RightHandSide, CommunicatorType, ApproximateOperator > Problem;
SolverStarter solverStarter;
return solverStarter.template run< Problem >( parameters );
}
typedef Operators::NeumannBoundaryConditions< MeshType, Constant, Real, Index > BoundaryConditions;
- typedef eulerProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;
+ typedef eulerProblem< MeshType, BoundaryConditions, RightHandSide, CommunicatorType, ApproximateOperator > Problem;
SolverStarter solverStarter;
return solverStarter.template run< Problem >( parameters );
}
@@ -92,12 +93,12 @@ class eulerSetter
if( boundaryConditionsType == "dirichlet" )
{
typedef Operators::DirichletBoundaryConditions< MeshType, MeshFunction, MeshType::getMeshDimension(), Real, Index > BoundaryConditions;
- typedef eulerProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;
+ typedef eulerProblem< MeshType, BoundaryConditions, RightHandSide, CommunicatorType, ApproximateOperator> Problem;
SolverStarter solverStarter;
return solverStarter.template run< Problem >( parameters );
}
typedef Operators::NeumannBoundaryConditions< MeshType, MeshFunction, Real, Index > BoundaryConditions;
- typedef eulerProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;
+ typedef eulerProblem< MeshType, BoundaryConditions, RightHandSide, CommunicatorType ,ApproximateOperator > Problem;
SolverStarter solverStarter;
return solverStarter.template run< Problem >( parameters );
}
diff --git a/examples/inviscid-flow/eulerProblem.h b/examples/inviscid-flow/eulerProblem.h
index ea9e3155641613eb0cb20712fbc1416d883e1de5..21b831096610708ac3e9e70f22616f664c63af0b 100644
--- a/examples/inviscid-flow/eulerProblem.h
+++ b/examples/inviscid-flow/eulerProblem.h
@@ -22,6 +22,7 @@ namespace TNL {
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename InviscidOperators >
class eulerProblem:
public PDEProblem< Mesh,
@@ -35,6 +36,8 @@ class eulerProblem:
typedef typename Mesh::DeviceType DeviceType;
typedef typename InviscidOperators::IndexType IndexType;
typedef PDEProblem< Mesh, RealType, DeviceType, IndexType > BaseType;
+
+ typedef CommType CommunicatorType;
using typename BaseType::MeshType;
using typename BaseType::MeshPointer;
diff --git a/examples/inviscid-flow/eulerProblem_impl.h b/examples/inviscid-flow/eulerProblem_impl.h
index f0ab4e553aba1672a2649061c1d51e9d71ccb201..e2a6023d3145af7d23ce72f81e5e8dfcc39e3ee6 100644
--- a/examples/inviscid-flow/eulerProblem_impl.h
+++ b/examples/inviscid-flow/eulerProblem_impl.h
@@ -33,9 +33,10 @@ namespace TNL {
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename InviscidOperators >
String
-eulerProblem< Mesh, BoundaryCondition, RightHandSide, InviscidOperators >::
+eulerProblem< Mesh, BoundaryCondition, RightHandSide, CommType, InviscidOperators >::
getTypeStatic()
{
return String( "eulerProblem< " ) + Mesh :: getTypeStatic() + " >";
@@ -44,9 +45,10 @@ getTypeStatic()
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename InviscidOperators >
String
-eulerProblem< Mesh, BoundaryCondition, RightHandSide, InviscidOperators >::
+eulerProblem< Mesh, BoundaryCondition, RightHandSide, CommType, InviscidOperators >::
getPrologHeader() const
{
return String( "Inviscid flow solver" );
@@ -55,9 +57,10 @@ getPrologHeader() const
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename InviscidOperators >
void
-eulerProblem< Mesh, BoundaryCondition, RightHandSide, InviscidOperators >::
+eulerProblem< Mesh, BoundaryCondition, RightHandSide, CommType, InviscidOperators >::
writeProlog( Logger& logger, const Config::ParameterContainer& parameters ) const
{
/****
@@ -69,9 +72,10 @@ writeProlog( Logger& logger, const Config::ParameterContainer& parameters ) cons
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename InviscidOperators >
bool
-eulerProblem< Mesh, BoundaryCondition, RightHandSide, InviscidOperators >::
+eulerProblem< Mesh, BoundaryCondition, RightHandSide, CommType, InviscidOperators >::
setup( const MeshPointer& meshPointer,
const Config::ParameterContainer& parameters,
const String& prefix )
@@ -89,9 +93,10 @@ setup( const MeshPointer& meshPointer,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename InviscidOperators >
-typename eulerProblem< Mesh, BoundaryCondition, RightHandSide, InviscidOperators >::IndexType
-eulerProblem< Mesh, BoundaryCondition, RightHandSide, InviscidOperators >::
+typename eulerProblem< Mesh, BoundaryCondition, RightHandSide, CommType, InviscidOperators >::IndexType
+eulerProblem< Mesh, BoundaryCondition, RightHandSide, CommType, InviscidOperators >::
getDofs( const MeshPointer& mesh ) const
{
/****
@@ -104,9 +109,10 @@ getDofs( const MeshPointer& mesh ) const
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename InviscidOperators >
void
-eulerProblem< Mesh, BoundaryCondition, RightHandSide, InviscidOperators >::
+eulerProblem< Mesh, BoundaryCondition, RightHandSide, CommType, InviscidOperators >::
bindDofs( const MeshPointer& mesh,
DofVectorPointer& dofVector )
{
@@ -116,9 +122,10 @@ bindDofs( const MeshPointer& mesh,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename InviscidOperators >
bool
-eulerProblem< Mesh, BoundaryCondition, RightHandSide, InviscidOperators >::
+eulerProblem< Mesh, BoundaryCondition, RightHandSide, CommType, InviscidOperators >::
setInitialCondition( const Config::ParameterContainer& parameters,
const MeshPointer& mesh,
DofVectorPointer& dofs,
@@ -142,10 +149,11 @@ setInitialCondition( const Config::ParameterContainer& parameters,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename InviscidOperators >
template< typename Matrix >
bool
-eulerProblem< Mesh, BoundaryCondition, RightHandSide, InviscidOperators >::
+eulerProblem< Mesh, BoundaryCondition, RightHandSide, CommType, InviscidOperators >::
setupLinearSystem( const MeshPointer& mesh,
Matrix& matrix )
{
@@ -168,9 +176,10 @@ setupLinearSystem( const MeshPointer& mesh,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename InviscidOperators >
bool
-eulerProblem< Mesh, BoundaryCondition, RightHandSide, InviscidOperators >::
+eulerProblem< Mesh, BoundaryCondition, RightHandSide, CommType, InviscidOperators >::
makeSnapshot( const RealType& time,
const IndexType& step,
const MeshPointer& mesh,
@@ -209,9 +218,10 @@ makeSnapshot( const RealType& time,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename InviscidOperators >
void
-eulerProblem< Mesh, BoundaryCondition, RightHandSide, InviscidOperators >::
+eulerProblem< Mesh, BoundaryCondition, RightHandSide, CommType, InviscidOperators >::
getExplicitUpdate( const RealType& time,
const RealType& tau,
const MeshPointer& mesh,
@@ -256,7 +266,7 @@ getExplicitUpdate( const RealType& time,
explicitUpdaterContinuity.setDifferentialOperator( this->inviscidOperatorsPointer->getContinuityOperator() );
explicitUpdaterContinuity.setBoundaryConditions( this->boundaryConditionPointer );
explicitUpdaterContinuity.setRightHandSide( this->rightHandSidePointer );
- explicitUpdaterContinuity.template update< typename Mesh::Cell >( time, tau, mesh,
+ explicitUpdaterContinuity.template update< typename Mesh::Cell, CommunicatorType >( time, tau, mesh,
this->conservativeVariables->getDensity(),
this->conservativeVariablesRHS->getDensity() );
@@ -267,7 +277,7 @@ getExplicitUpdate( const RealType& time,
explicitUpdaterMomentumX.setDifferentialOperator( this->inviscidOperatorsPointer->getMomentumXOperator() );
explicitUpdaterMomentumX.setBoundaryConditions( this->boundaryConditionPointer );
explicitUpdaterMomentumX.setRightHandSide( this->rightHandSidePointer );
- explicitUpdaterMomentumX.template update< typename Mesh::Cell >( time, tau, mesh,
+ explicitUpdaterMomentumX.template update< typename Mesh::Cell, CommunicatorType >( time, tau, mesh,
( *this->conservativeVariables->getMomentum() )[ 0 ], // uRhoVelocityX,
( *this->conservativeVariablesRHS->getMomentum() )[ 0 ] ); //, fuRhoVelocityX );
@@ -277,7 +287,7 @@ getExplicitUpdate( const RealType& time,
explicitUpdaterMomentumY.setDifferentialOperator( this->inviscidOperatorsPointer->getMomentumYOperator() );
explicitUpdaterMomentumY.setBoundaryConditions( this->boundaryConditionPointer );
explicitUpdaterMomentumY.setRightHandSide( this->rightHandSidePointer );
- explicitUpdaterMomentumY.template update< typename Mesh::Cell >( time, tau, mesh,
+ explicitUpdaterMomentumY.template update< typename Mesh::Cell, CommunicatorType >( time, tau, mesh,
( *this->conservativeVariables->getMomentum() )[ 1 ], // uRhoVelocityX,
( *this->conservativeVariablesRHS->getMomentum() )[ 1 ] ); //, fuRhoVelocityX );
}
@@ -288,7 +298,7 @@ getExplicitUpdate( const RealType& time,
explicitUpdaterMomentumZ.setDifferentialOperator( this->inviscidOperatorsPointer->getMomentumZOperator() );
explicitUpdaterMomentumZ.setBoundaryConditions( this->boundaryConditionPointer );
explicitUpdaterMomentumZ.setRightHandSide( this->rightHandSidePointer );
- explicitUpdaterMomentumZ.template update< typename Mesh::Cell >( time, tau, mesh,
+ explicitUpdaterMomentumZ.template update< typename Mesh::Cell, CommunicatorType >( time, tau, mesh,
( *this->conservativeVariables->getMomentum() )[ 2 ], // uRhoVelocityX,
( *this->conservativeVariablesRHS->getMomentum() )[ 2 ] ); //, fuRhoVelocityX );
}
@@ -301,7 +311,7 @@ getExplicitUpdate( const RealType& time,
explicitUpdaterEnergy.setDifferentialOperator( this->inviscidOperatorsPointer->getEnergyOperator() );
explicitUpdaterEnergy.setBoundaryConditions( this->boundaryConditionPointer );
explicitUpdaterEnergy.setRightHandSide( this->rightHandSidePointer );
- explicitUpdaterEnergy.template update< typename Mesh::Cell >( time, tau, mesh,
+ explicitUpdaterEnergy.template update< typename Mesh::Cell, CommunicatorType >( time, tau, mesh,
this->conservativeVariables->getEnergy(), // uRhoVelocityX,
this->conservativeVariablesRHS->getEnergy() ); //, fuRhoVelocityX );
@@ -315,10 +325,11 @@ getExplicitUpdate( const RealType& time,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename InviscidOperators >
template< typename Matrix >
void
-eulerProblem< Mesh, BoundaryCondition, RightHandSide, InviscidOperators >::
+eulerProblem< Mesh, BoundaryCondition, RightHandSide, CommType, InviscidOperators >::
assemblyLinearSystem( const RealType& time,
const RealType& tau,
const MeshPointer& mesh,
@@ -351,9 +362,10 @@ assemblyLinearSystem( const RealType& time,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename InviscidOperators >
bool
-eulerProblem< Mesh, BoundaryCondition, RightHandSide, InviscidOperators >::
+eulerProblem< Mesh, BoundaryCondition, RightHandSide, CommType, InviscidOperators >::
postIterate( const RealType& time,
const RealType& tau,
const MeshPointer& mesh,
diff --git a/examples/transport-equation/tnl-transport-equation-eoc.h b/examples/transport-equation/tnl-transport-equation-eoc.h
index 98a38425290f06eef37fd6606a2619fb898c6908..fc2eda33336e8c93d196a3aad5643697c80863ef 100644
--- a/examples/transport-equation/tnl-transport-equation-eoc.h
+++ b/examples/transport-equation/tnl-transport-equation-eoc.h
@@ -72,7 +72,8 @@ template< typename Real,
typename Index,
typename MeshType,
typename ConfigTag,
- typename SolverStarter >
+ typename SolverStarter,
+ typename CommunicatorType >
class advectionSetter
{
public:
@@ -98,13 +99,13 @@ class advectionSetter
if( boundaryConditionsType == "dirichlet" )
{
typedef Operators::DirichletBoundaryConditions< MeshType, ConstantFunctionType, MeshType::getMeshDimension(), Real, Index > BoundaryConditions;
- typedef transportEquationProblemEoc< MeshType, BoundaryConditions, ConstantFunctionType, DifferentialOperatorType > Problem;
+ typedef transportEquationProblemEoc< MeshType, BoundaryConditions, ConstantFunctionType, CommunicatorType, DifferentialOperatorType > Problem;
return callSolverStarter< Problem >( parameters );
}
if( boundaryConditionsType == "neumann" )
{
typedef Operators::DirichletBoundaryConditions< MeshType, ConstantFunctionType, MeshType::getMeshDimension(), Real, Index > BoundaryConditions;
- typedef transportEquationProblemEoc< MeshType, BoundaryConditions, ConstantFunctionType, DifferentialOperatorType > Problem;
+ typedef transportEquationProblemEoc< MeshType, BoundaryConditions, ConstantFunctionType, CommunicatorType, DifferentialOperatorType > Problem;
return callSolverStarter< Problem >( parameters );
}
std::cerr << "Unknown boundary conditions type: " << boundaryConditionsType << "." << std::endl;
diff --git a/examples/transport-equation/tnl-transport-equation.h b/examples/transport-equation/tnl-transport-equation.h
index be0083153b91c87142888d4a5174c331f06b8e8b..c52555b46dab233b57b34ee76f21557ca9f039f2 100644
--- a/examples/transport-equation/tnl-transport-equation.h
+++ b/examples/transport-equation/tnl-transport-equation.h
@@ -59,7 +59,8 @@ template< typename Real,
typename Index,
typename MeshType,
typename ConfigTag,
- typename SolverStarter >
+ typename SolverStarter,
+ typename CommunicatorType >
class advectionSetter
{
public:
@@ -85,13 +86,13 @@ class advectionSetter
if( boundaryConditionsType == "dirichlet" )
{
typedef Operators::DirichletBoundaryConditions< MeshType, ConstantFunctionType, MeshType::getMeshDimension(), Real, Index > BoundaryConditions;
- typedef transportEquationProblem< MeshType, BoundaryConditions, ConstantFunctionType, DifferentialOperatorType > Problem;
+ typedef transportEquationProblem< MeshType, BoundaryConditions, ConstantFunctionType, CommunicatorType, DifferentialOperatorType > Problem;
return callSolverStarter< Problem >( parameters );
}
if( boundaryConditionsType == "neumann" )
{
typedef Operators::DirichletBoundaryConditions< MeshType, ConstantFunctionType, MeshType::getMeshDimension(), Real, Index > BoundaryConditions;
- typedef transportEquationProblem< MeshType, BoundaryConditions, ConstantFunctionType, DifferentialOperatorType > Problem;
+ typedef transportEquationProblem< MeshType, BoundaryConditions, ConstantFunctionType, CommunicatorType, DifferentialOperatorType > Problem;
return callSolverStarter< Problem >( parameters );
}
std::cerr << "Unknown boundary conditions type: " << boundaryConditionsType << "." << std::endl;
diff --git a/examples/transport-equation/transportEquationProblem.h b/examples/transport-equation/transportEquationProblem.h
index a9fc1482075afd51b4e5490dae2c67c3572e451e..1de9750e4dd6ce25331afe241386ee82781a316c 100644
--- a/examples/transport-equation/transportEquationProblem.h
+++ b/examples/transport-equation/transportEquationProblem.h
@@ -21,6 +21,7 @@ namespace TNL {
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
class transportEquationProblem:
public PDEProblem< Mesh,
@@ -42,6 +43,8 @@ public PDEProblem< Mesh,
typedef typename DifferentialOperator::VelocityFieldType VelocityFieldType;
typedef SharedPointer< VelocityFieldType, DeviceType > VelocityFieldPointer;
+ typedef CommType CommunicatorType;
+
using typename BaseType::MeshType;
using typename BaseType::MeshPointer;
using typename BaseType::DofVectorType;
diff --git a/examples/transport-equation/transportEquationProblemEoc.h b/examples/transport-equation/transportEquationProblemEoc.h
index cd851bd3005ea5eac669ad6779f56fdccb9dbe11..2bf30f508c4a24d10a3f041bb339709f14216ec5 100644
--- a/examples/transport-equation/transportEquationProblemEoc.h
+++ b/examples/transport-equation/transportEquationProblemEoc.h
@@ -22,9 +22,10 @@ namespace TNL {
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
class transportEquationProblemEoc:
-public transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >
+public transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >
{
public:
@@ -32,7 +33,7 @@ public transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, Differe
typedef typename Mesh::DeviceType DeviceType;
typedef typename DifferentialOperator::IndexType IndexType;
typedef Functions::MeshFunction< Mesh > MeshFunctionType;
- typedef transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator > BaseType;
+ typedef transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator > BaseType;
typedef SharedPointer< MeshFunctionType, DeviceType > MeshFunctionPointer;
typedef SharedPointer< DifferentialOperator > DifferentialOperatorPointer;
typedef SharedPointer< BoundaryCondition > BoundaryConditionPointer;
@@ -40,7 +41,8 @@ public transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, Differe
typedef typename DifferentialOperator::VelocityFieldType VelocityFieldType;
typedef SharedPointer< VelocityFieldType, DeviceType > VelocityFieldPointer;
-
+
+ typedef CommType CommunicatorType;
using typename BaseType::MeshType;
using typename BaseType::MeshPointer;
using typename BaseType::DofVectorType;
diff --git a/examples/transport-equation/transportEquationProblemEoc_impl.h b/examples/transport-equation/transportEquationProblemEoc_impl.h
index 7e2accc3a1413d43b6bdd993916660f54e857b83..c3f4f6e68d9fe1a9f6791bc8d6f0f97a025a3f7a 100644
--- a/examples/transport-equation/transportEquationProblemEoc_impl.h
+++ b/examples/transport-equation/transportEquationProblemEoc_impl.h
@@ -26,9 +26,10 @@ namespace TNL {
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
String
-transportEquationProblemEoc< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+transportEquationProblemEoc< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getTypeStatic()
{
return String( "transportEquationProblemEoc< " ) + Mesh :: getTypeStatic() + " >";
@@ -37,9 +38,10 @@ getTypeStatic()
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
String
-transportEquationProblemEoc< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+transportEquationProblemEoc< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getPrologHeader() const
{
return String( "Transport Equation EOC" );
@@ -49,9 +51,10 @@ getPrologHeader() const
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
bool
-transportEquationProblemEoc< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+transportEquationProblemEoc< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
setup( const MeshPointer& meshPointer,
const Config::ParameterContainer& parameters,
const String& prefix )
@@ -126,9 +129,10 @@ setup( const MeshPointer& meshPointer,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
bool
-transportEquationProblemEoc< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+transportEquationProblemEoc< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
setInitialCondition( const Config::ParameterContainer& parameters,
const MeshPointer& meshPointer,
DofVectorPointer& dofs,
diff --git a/examples/transport-equation/transportEquationProblem_impl.h b/examples/transport-equation/transportEquationProblem_impl.h
index 698a7bc687dc4c3b4df49a48fef2f5a73566ecbc..cf5864477c6a79deea1fe579877656669d39512a 100644
--- a/examples/transport-equation/transportEquationProblem_impl.h
+++ b/examples/transport-equation/transportEquationProblem_impl.h
@@ -23,9 +23,10 @@ namespace TNL {
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
String
-transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getTypeStatic()
{
return String( "transportEquationProblem< " ) + Mesh :: getTypeStatic() + " >";
@@ -34,9 +35,10 @@ getTypeStatic()
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
String
-transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getPrologHeader() const
{
return String( "Transport Equation" );
@@ -45,9 +47,10 @@ getPrologHeader() const
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
void
-transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
writeProlog( Logger& logger, const Config::ParameterContainer& parameters ) const
{
/****
@@ -59,9 +62,10 @@ writeProlog( Logger& logger, const Config::ParameterContainer& parameters ) cons
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
bool
-transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
setup( const MeshPointer& meshPointer,
const Config::ParameterContainer& parameters,
const String& prefix )
@@ -76,9 +80,10 @@ setup( const MeshPointer& meshPointer,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
-typename transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::IndexType
-transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+typename transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::IndexType
+transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getDofs( const MeshPointer& mesh ) const
{
/****
@@ -91,9 +96,10 @@ getDofs( const MeshPointer& mesh ) const
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
void
-transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
bindDofs( const MeshPointer& meshPointer,
DofVectorPointer& dofVector )
{
@@ -104,9 +110,10 @@ bindDofs( const MeshPointer& meshPointer,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
bool
-transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
setInitialCondition( const Config::ParameterContainer& parameters,
const MeshPointer& meshPointer,
DofVectorPointer& dofs,
@@ -125,10 +132,11 @@ setInitialCondition( const Config::ParameterContainer& parameters,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
template< typename Matrix >
bool
-transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
setupLinearSystem( const MeshPointer& mesh,
Matrix& matrix )
{
@@ -151,9 +159,10 @@ setupLinearSystem( const MeshPointer& mesh,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
bool
-transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
makeSnapshot( const RealType& time,
const IndexType& step,
const MeshPointer& mesh,
@@ -174,9 +183,10 @@ makeSnapshot( const RealType& time,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
void
-transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getExplicitUpdate( const RealType& time,
const RealType& tau,
const MeshPointer& mesh,
@@ -203,16 +213,17 @@ getExplicitUpdate( const RealType& time,
explicitUpdater.setDifferentialOperator( this->differentialOperatorPointer );
explicitUpdater.setBoundaryConditions( this->boundaryConditionPointer );
explicitUpdater.setRightHandSide( this->rightHandSidePointer );
- explicitUpdater.template update< typename Mesh::Cell >( time, tau, mesh, u, fu );
+ explicitUpdater.template update< typename Mesh::Cell, CommType >( time, tau, mesh, u, fu );
}
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
template< typename Matrix >
void
-transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+transportEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
assemblyLinearSystem( const RealType& time,
const RealType& tau,
const MeshPointer& mesh,
diff --git a/src/Python/pytnl/tnl_indexing.h b/src/Python/pytnl/tnl_indexing.h
index 29663d66b553d8095ca0a67f980b346fac04e712..795226537dd4caf98f64ec0e478e511b814ad16e 100644
--- a/src/Python/pytnl/tnl_indexing.h
+++ b/src/Python/pytnl/tnl_indexing.h
@@ -68,7 +68,7 @@ void tnl_slice_indexing( Scope & scope )
if (!slice.compute(a.getSize(), &start, &stop, &step, &slicelength))
throw py::error_already_set();
- if (slicelength != value.getSize())
+ if (slicelength != (size_t) value.getSize())
throw std::runtime_error("Left and right hand size of slice assignment have different sizes!");
for (size_t i = 0; i < slicelength; ++i) {
diff --git a/src/TNL/Assert.h b/src/TNL/Assert.h
index f3befbaa0edfaddab1c11c33a6833718f7b25956..ae1758a19bf3e6250335e448b223453354401cb2 100644
--- a/src/TNL/Assert.h
+++ b/src/TNL/Assert.h
@@ -50,6 +50,7 @@
#include <stdio.h>
#include <TNL/Devices/CudaCallable.h>
+#include <TNL/Debugging/StackBacktrace.h>
namespace TNL {
namespace Assert {
@@ -88,6 +89,9 @@ printDiagnosticsHost( const char* assertion,
<< "Function: " << function << "\n"
<< "Line: " << line << "\n"
<< "Diagnostics:\n" << diagnostics << std::endl;
+
+ PrintStackBacktrace;
+
throw AssertionError( str.str() );
}
@@ -108,6 +112,8 @@ printDiagnosticsHost( const char* assertion,
<< "Function: " << function << "\n"
<< "Line: " << line << "\n"
<< "Diagnostics:\n" << diagnostics << std::endl;
+
+ PrintStackBacktrace;
}
#endif // TNL_THROW_ASSERTION_ERROR
diff --git a/src/TNL/CMakeLists.txt b/src/TNL/CMakeLists.txt
index 54551e7d8bc8dc2fe1b4fd6b5217826932e2c9ac..13113f9c3fa6b04c3e485eb836e149f48a1c91d7 100644
--- a/src/TNL/CMakeLists.txt
+++ b/src/TNL/CMakeLists.txt
@@ -1,5 +1,6 @@
ADD_SUBDIRECTORY( Config )
ADD_SUBDIRECTORY( Containers )
+ADD_SUBDIRECTORY( Communicators )
ADD_SUBDIRECTORY( Debugging )
ADD_SUBDIRECTORY( Devices )
ADD_SUBDIRECTORY( Exceptions )
@@ -31,7 +32,6 @@ set( headers
Logger.h
Logger_impl.h
Math.h
- mpi-supp.h
ParallelFor.h
param-types.h
SharedPointer.h
diff --git a/src/TNL/Communicators/CMakeLists.txt b/src/TNL/Communicators/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..271f5a2d64003a51f47d877e70c0701f129d6242
--- /dev/null
+++ b/src/TNL/Communicators/CMakeLists.txt
@@ -0,0 +1,4 @@
+SET( headers MpiCommunicator.h
+ NoDistrCommunicator.h )
+
+INSTALL( FILES ${headers} DESTINATION include/tnl-${tnlVersion}/TNL/Communicators )
diff --git a/src/TNL/Communicators/MpiCommunicator.h b/src/TNL/Communicators/MpiCommunicator.h
new file mode 100644
index 0000000000000000000000000000000000000000..421c9e2a6fd36b6e73bb1024ca98737fb987c315
--- /dev/null
+++ b/src/TNL/Communicators/MpiCommunicator.h
@@ -0,0 +1,192 @@
+/***************************************************************************
+ MpiCommunicator.h - description
+ -------------------
+ begin : 2005/04/23
+ copyright : (C) 2005 by Tomas Oberhuber
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+/* See Copyright Notice in tnl/Copyright */
+
+#pragma once
+
+
+
+#ifdef HAVE_MPI
+
+#include <mpi.h>
+#include <TNL/String.h>
+#include <iostream>
+#include <fstream>
+
+namespace TNL {
+namespace Communicators {
+
+ class MpiCommunicator
+ {
+
+ private:
+ inline static MPI_Datatype MPIDataType( const signed char* ) { return MPI_CHAR; };
+ inline static MPI_Datatype MPIDataType( const signed short int* ) { return MPI_SHORT; };
+ inline static MPI_Datatype MPIDataType( const signed int* ) { return MPI_INT; };
+ inline static MPI_Datatype MPIDataType( const signed long int* ) { return MPI_LONG; };
+ inline static MPI_Datatype MPIDataType( const unsigned char *) { return MPI_UNSIGNED_CHAR; };
+ inline static MPI_Datatype MPIDataType( const unsigned short int* ) { return MPI_UNSIGNED_SHORT; };
+ inline static MPI_Datatype MPIDataType( const unsigned int* ) { return MPI_UNSIGNED; };
+ inline static MPI_Datatype MPIDataType( const unsigned long int* ) { return MPI_UNSIGNED_LONG; };
+ inline static MPI_Datatype MPIDataType( const float* ) { return MPI_FLOAT; };
+ inline static MPI_Datatype MPIDataType( const double* ) { return MPI_DOUBLE; };
+ inline static MPI_Datatype MPIDataType( const long double* ) { return MPI_LONG_DOUBLE; };
+
+ public:
+
+ //can be call before init
+ static bool isAvailable()
+ {
+ return true;
+ }
+
+ //can be called only after init
+ static bool isDistributed()
+ {
+ return GetSize()>1;
+ };
+
+ typedef MPI::Request Request;
+ static MPI::Request NullRequest;
+ static std::streambuf *psbuf;
+ static std::streambuf *backup;
+ static std::ofstream filestr;
+
+ static void Init(int argc, char **argv,bool redirect=false)
+ {
+ MPI::Init(argc,argv);
+ NullRequest=MPI::REQUEST_NULL;
+
+ if(isDistributed() && redirect)
+ {
+ //redirect all stdout to files, only 0 take to go to console
+ backup=std::cout.rdbuf();
+
+ //redirect output to files...
+ if(MPI::COMM_WORLD.Get_rank()!=0)
+ {
+ std::cout<< GetRank() <<": Redirecting std::out to file" <<std::endl;
+ String stdoutfile;
+ stdoutfile=String( "./stdout-")+convertToString(MPI::COMM_WORLD.Get_rank())+String(".txt");
+ filestr.open (stdoutfile.getString());
+ psbuf = filestr.rdbuf();
+ std::cout.rdbuf(psbuf);
+ }
+ }
+ };
+
+ static void Finalize()
+ {
+ if(isDistributed())
+ {
+ if(MPI::COMM_WORLD.Get_rank()!=0)
+ {
+ std::cout.rdbuf(backup);
+ filestr.close();
+ }
+ }
+ MPI::Finalize();
+ };
+
+ static bool IsInitialized()
+ {
+ return MPI::Is_initialized();
+ };
+
+ static int GetRank()
+ {
+ return MPI::COMM_WORLD.Get_rank();
+ };
+
+ static int GetSize()
+ {
+ return MPI::COMM_WORLD.Get_size();
+ };
+
+ //dim-number of dimesions, distr array of guess distr - 0 for computation
+ //distr array will be filled by computed distribution
+ //more information in MPI documentation
+ static void DimsCreate(int nproc, int dim, int *distr)
+ {
+ /***HACK for linear distribution***/
+ int sum=0;
+ for(int i=0;i<dim;i++)
+ sum+=distr[i];
+ if(sum==0) //uživatel neovlivňuje distribuci
+ {
+ std::cout << "vynucuji distribuci" <<std::endl;
+ for(int i=0;i<dim-1;i++)
+ {
+ distr[i]=1;
+ }
+ distr[dim-1]=0;
+ }
+ /***END OF HACK***/
+
+ MPI_Dims_create(nproc, dim, distr);
+ };
+
+ static void Barrier()
+ {
+ MPI::COMM_WORLD.Barrier();
+ };
+
+ template <typename T>
+ static Request ISend( const T *data, int count, int dest)
+ {
+ return MPI::COMM_WORLD.Isend((void*) data, count, MPIDataType(data) , dest, 0);
+ }
+
+ template <typename T>
+ static Request IRecv( const T *data, int count, int src)
+ {
+ return MPI::COMM_WORLD.Irecv((void*) data, count, MPIDataType(data) , src, 0);
+ }
+
+ static void WaitAll(Request *reqs, int length)
+ {
+ MPI::Request::Waitall(length, reqs);
+ };
+
+ template< typename T >
+ static void Bcast( T& data, int count, int root)
+ {
+ MPI::COMM_WORLD.Bcast((void*) &data, count, MPIDataType(data), root);
+ }
+
+ /* template< typename T >
+ static void Allreduce( T& data,
+ T& reduced_data,
+ int count,
+ const MPI_Op &op)
+ {
+ MPI::COMM_WORLD.Allreduce((void*) &data, (void*) &reduced_data,count,MPIDataType(data),op);
+ };
+
+ template< typename T >
+ static void Reduce( T& data,
+ T& reduced_data,
+ int count,
+ MPI_Op &op,
+ int root)
+ {
+ MPI::COMM_WORLD.Reduce((void*) &data, (void*) &reduced_data,count,MPIDataType(data),op,root);
+ };*/
+ };
+
+ MPI::Request MpiCommunicator::NullRequest;
+ std::streambuf *MpiCommunicator::psbuf;
+ std::streambuf *MpiCommunicator::backup;
+ std::ofstream MpiCommunicator::filestr;
+
+}//namespace Communicators
+} // namespace TNL
+#endif
+
+
diff --git a/src/TNL/Communicators/NoDistrCommunicator.h b/src/TNL/Communicators/NoDistrCommunicator.h
new file mode 100644
index 0000000000000000000000000000000000000000..480be330fd50506fd06ede7608083f97cb5622ce
--- /dev/null
+++ b/src/TNL/Communicators/NoDistrCommunicator.h
@@ -0,0 +1,117 @@
+/***************************************************************************
+ NoDistrCommunicator.h - description
+ -------------------
+ begin : 2018/01/09
+ copyright : (C) 2018 by Tomas Oberhuber
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+/* See Copyright Notice in tnl/Copyright */
+
+#pragma once
+
+namespace TNL {
+namespace Communicators {
+
+ class NoDistrCommunicator
+ {
+
+
+ public:
+
+ typedef int Request;
+ static Request NullRequest;
+
+ static bool isAvailable()
+ {
+ return true;
+ }
+
+ static void Init(int argc, char **argv, bool redirect=false)
+ {
+ NullRequest=-1;
+ };
+
+ static void Finalize()
+ {
+ };
+
+ static bool IsInitialized()
+ {
+ return true;
+ };
+
+ static bool isDistributed()
+ {
+ return false;
+ };
+
+ static int GetRank()
+ {
+ return 0;
+ };
+
+ static int GetSize()
+ {
+ return 1;
+ };
+
+ static void DimsCreate(int nproc, int dim, int *distr)
+ {
+ for(int i=0;i<dim;i++)
+ {
+ distr[i]=1;
+ }
+ };
+
+ static void Barrier()
+ {
+ };
+
+ template <typename T>
+ static Request ISend( const T *data, int count, int dest)
+ {
+ return 1;
+ };
+
+ template <typename T>
+ static Request IRecv( const T *data, int count, int src)
+ {
+ return 1;
+ };
+
+ static void WaitAll(Request *reqs, int length)
+ {
+ };
+
+ template< typename T >
+ static void Bcast( T& data, int count, int root)
+ {
+ };
+
+ /* template< typename T >
+ static void Allreduce( T& data,
+ T& reduced_data,
+ int count,
+ const MPI_Op &op)
+ {
+ MPI::COMM_WORLD.Allreduce((void*) &data, (void*) &reduced_data,count,MPIDataType(data),op);
+ };
+
+ template< typename T >
+ static void Reduce( T& data,
+ T& reduced_data,
+ int count,
+ MPI_Op &op,
+ int root)
+ {
+ MPI::COMM_WORLD.Reduce((void*) &data, (void*) &reduced_data,count,MPIDataType(data),op,root);
+ };*/
+ };
+
+ int NoDistrCommunicator::NullRequest;
+
+} // namespace Communicators
+} // namespace TNL
+
+
diff --git a/src/TNL/Config/ParameterContainer.cpp b/src/TNL/Config/ParameterContainer.cpp
index 11f3a816152123c7df77c819a9263bc5b918c46a..e1b451bd651c32457947b293ce9e426252331456 100644
--- a/src/TNL/Config/ParameterContainer.cpp
+++ b/src/TNL/Config/ParameterContainer.cpp
@@ -93,9 +93,9 @@ ParameterContainer::
parameters. DeepEraseAll();
}
-void ParameterContainer::MPIBcast( int root, MPI_Comm mpi_comm )
+/*void ParameterContainer::MPIBcast( int root, MPI_Comm mpi_comm )
{
-#ifdef HAVE_MPI
+#ifdef USE_MPI
int i;
int size = parameters. getSize();
:: MPIBcast( size, 1, root, mpi_comm );
@@ -161,7 +161,7 @@ void ParameterContainer::MPIBcast( int root, MPI_Comm mpi_comm )
}
#endif
}
-
+*/
bool
parseCommandLine( int argc, char* argv[],
const Config::ConfigDescription& config_description,
diff --git a/src/TNL/Config/ParameterContainer.h b/src/TNL/Config/ParameterContainer.h
index 0fe07d424a5a60f03f9f162995e6f9b7295bee01..0cb866a0e1cf9d0e95d419e7e29fcd9cd68fcbdf 100644
--- a/src/TNL/Config/ParameterContainer.h
+++ b/src/TNL/Config/ParameterContainer.h
@@ -12,7 +12,6 @@
#include <TNL/Containers/List.h>
#include <TNL/Config/ConfigDescription.h>
-#include <TNL/mpi-supp.h>
#include <TNL/param-types.h>
//#include <TNL/Debugging/StackBacktrace.h>
@@ -94,7 +93,7 @@ class ParameterContainer
}
//! Broadcast to other nodes in MPI cluster
- void MPIBcast( int root, MPI_Comm mpi_comm = MPI_COMM_WORLD );
+ // void MPIBcast( int root, MPI_Comm mpi_comm = MPI_COMM_WORLD );
~ParameterContainer();
diff --git a/src/TNL/Containers/Algorithms/ArrayOperationsHost_impl.h b/src/TNL/Containers/Algorithms/ArrayOperationsHost_impl.h
index 7390cfdb0e317a036f45c1db7b58d09a18afac0f..02eab36d7147c290f6417076fb11008450b7d7d6 100644
--- a/src/TNL/Containers/Algorithms/ArrayOperationsHost_impl.h
+++ b/src/TNL/Containers/Algorithms/ArrayOperationsHost_impl.h
@@ -101,7 +101,18 @@ copyMemory( DestinationElement* destination,
if( std::is_same< DestinationElement, SourceElement >::value &&
( std::is_fundamental< DestinationElement >::value ||
std::is_pointer< DestinationElement >::value ) )
+ {
+ // GCC 8.1 complains that we bypass a non-trivial copy-constructor
+ // (in C++17 we could use constexpr if to avoid compiling this branch in that case)
+ #if defined(__GNUC__) && !defined(__clang__) && !defined(__NVCC__)
+ #pragma GCC diagnostic push
+ #pragma GCC diagnostic ignored "-Wclass-memaccess"
+ #endif
memcpy( destination, source, size * sizeof( DestinationElement ) );
+ #if defined(__GNUC__) && !defined(__clang__) && !defined(__NVCC__)
+ #pragma GCC diagnostic pop
+ #endif
+ }
else
for( Index i = 0; i < size; i ++ )
destination[ i ] = ( DestinationElement ) source[ i ];
diff --git a/src/TNL/Containers/Array_impl.h b/src/TNL/Containers/Array_impl.h
index 568df2f5136e0b4a0a1258f7254518152f7d71c7..11c546c664fbe2468485f11755d2709ed0d6c7f2 100644
--- a/src/TNL/Containers/Array_impl.h
+++ b/src/TNL/Containers/Array_impl.h
@@ -547,6 +547,7 @@ boundLoad( File& file )
return true;
}
+
template< typename Element,
typename Device,
typename Index >
diff --git a/src/TNL/Containers/StaticVector.h b/src/TNL/Containers/StaticVector.h
index 9d5fa6471dbf26d6c95dd9c34ec86dd02df3641b..b6c89eefaf88e72ea1586aec469b43e95b1c1b45 100644
--- a/src/TNL/Containers/StaticVector.h
+++ b/src/TNL/Containers/StaticVector.h
@@ -477,6 +477,39 @@ Real tnlScalarProduct( const StaticVector< 3, Real >& u,
return u[ 0 ] * v[ 0 ] + u[ 1 ] * v[ 1 ] + u[ 2 ] * v[ 2 ];
}
+template< typename T1,
+ typename T2>
+StaticVector<1, T1> tnlDotProduct( const StaticVector< 1, T1 >& u,
+ const StaticVector< 1, T2 >& v )
+{
+ StaticVector<1, T1> ret;
+ ret[0]=u[0]*v[0];
+ return ret;
+}
+
+template< typename T1,
+ typename T2>
+StaticVector<2, T1> tnlDotProduct( const StaticVector< 2, T1 >& u,
+ const StaticVector< 2, T2 >& v )
+{
+ StaticVector<2, T1> ret;
+ ret[0]=u[0]*v[0];
+ ret[1]=u[1]*v[1];
+ return ret;
+}
+
+template< typename T1,
+ typename T2>
+StaticVector<3, T1> tnlDotProduct( const StaticVector< 3, T1 >& u,
+ const StaticVector< 3, T2 >& v )
+{
+ StaticVector<3, T1> ret;
+ ret[0]=u[0]*v[0];
+ ret[1]=u[1]*v[1];
+ ret[2]=u[2]*v[2];
+ return ret;
+}
+
template< typename Real >
Real tnlTriangleArea( const StaticVector< 2, Real >& a,
const StaticVector< 2, Real >& b,
diff --git a/src/TNL/Experimental/Hamilton-Jacobi/Solvers/hamilton-jacobi/tnl-direct-eikonal-solver.h b/src/TNL/Experimental/Hamilton-Jacobi/Solvers/hamilton-jacobi/tnl-direct-eikonal-solver.h
index c4055d95df7afe453337e592de0f9ef224226b52..6eca7bc281e3b18c361ad96279df23d19e6e2509 100644
--- a/src/TNL/Experimental/Hamilton-Jacobi/Solvers/hamilton-jacobi/tnl-direct-eikonal-solver.h
+++ b/src/TNL/Experimental/Hamilton-Jacobi/Solvers/hamilton-jacobi/tnl-direct-eikonal-solver.h
@@ -42,7 +42,8 @@ template< typename Real,
typename Index,
typename MeshType,
typename MeshConfig,
- typename SolverStarter >
+ typename SolverStarter,
+ typename CommunicatorType>
class tnlDirectEikonalSolverSetter
{
public:
diff --git a/src/TNL/Functions/MeshFunction.h b/src/TNL/Functions/MeshFunction.h
index 2c8d41a564c3af16ad406fbf9dca5cf95f3e98e5..adcde3fc48a2cd89548ca7191abed096b992794f 100644
--- a/src/TNL/Functions/MeshFunction.h
+++ b/src/TNL/Functions/MeshFunction.h
@@ -13,6 +13,8 @@
#include <TNL/Functions/MeshFunctionGnuplotWriter.h>
#include <TNL/Functions/MeshFunctionVTKWriter.h>
#include <TNL/SharedPointer.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMeshSynchronizer.h>
#pragma once
@@ -37,6 +39,8 @@ class MeshFunction :
typedef Real RealType;
typedef Containers::Vector< RealType, DeviceType, IndexType > VectorType;
typedef Functions::MeshFunction< Mesh, MeshEntityDimension, Real > ThisType;
+ typedef Meshes::DistributedMeshes::DistributedMesh<MeshType> DistributedMeshType;
+ typedef Meshes::DistributedMeshes::DistributedMeshSynchronizer<ThisType> DistributedMeshSynchronizerType;
static constexpr int getEntitiesDimension() { return MeshEntityDimension; }
@@ -155,14 +159,24 @@ class MeshFunction :
using Object::load;
using Object::boundLoad;
+
+ template< typename CommunicatorType>
+ void Synchronize();
+
protected:
+
+ DistributedMeshSynchronizerType synchronizer;
MeshPointer meshPointer;
VectorType data;
template< typename, typename > friend class MeshFunctionEvaluator;
+
+ private:
+ void SetupSynchronizer(DistributedMeshType *distrMesh);
+
};
} // namespace Functions
diff --git a/src/TNL/Functions/MeshFunction_impl.h b/src/TNL/Functions/MeshFunction_impl.h
index d87cbadb30953137cedc4008d78e2b54d58dc59d..037d014dede9cbcaffbc7ecec2d85c9e399e6c74 100644
--- a/src/TNL/Functions/MeshFunction_impl.h
+++ b/src/TNL/Functions/MeshFunction_impl.h
@@ -34,8 +34,11 @@ template< typename Mesh,
typename Real >
MeshFunction< Mesh, MeshEntityDimension, Real >::
MeshFunction( const MeshPointer& meshPointer )
-: meshPointer( meshPointer )
{
+
+ SetupSynchronizer(meshPointer->GetDistMesh());
+
+ this->meshPointer=meshPointer;
this->data.setSize( getMesh().template getEntitiesCount< typename Mesh::template EntityType< MeshEntityDimension > >() );
TNL_ASSERT( this->data.getSize() == this->getMesh().template getEntitiesCount< typename MeshType::template EntityType< MeshEntityDimension > >(),
std::cerr << "this->data.getSize() = " << this->data.getSize() << std::endl
@@ -47,8 +50,11 @@ template< typename Mesh,
typename Real >
MeshFunction< Mesh, MeshEntityDimension, Real >::
MeshFunction( const ThisType& meshFunction )
-: meshPointer( meshFunction.meshPointer )
{
+
+ SetupSynchronizer(meshFunction.meshPointer->GetDistMesh());
+
+ this->meshPointer=meshFunction.meshPointer;
this->data.bind( meshFunction.getData() );
}
@@ -60,10 +66,17 @@ MeshFunction< Mesh, MeshEntityDimension, Real >::
MeshFunction( const MeshPointer& meshPointer,
Vector& data,
const IndexType& offset )
-: meshPointer( meshPointer )
+//: meshPointer( meshPointer )
{
+<<<<<<< HEAD
TNL_ASSERT_GE( data.getSize(), meshPointer->template getEntitiesCount< typename MeshType::template EntityType< MeshEntityDimension > >(),
"The input vector is not large enough for binding to the mesh function." );
+=======
+
+ SetupSynchronizer(meshPointer->GetDistMesh());
+
+ this->meshPointer=meshPointer;
+>>>>>>> origin/mpi-explosive
this->data.bind( data, offset, getMesh().template getEntitiesCount< typename Mesh::template EntityType< MeshEntityDimension > >() );
}
@@ -76,10 +89,17 @@ MeshFunction< Mesh, MeshEntityDimension, Real >::
MeshFunction( const MeshPointer& meshPointer,
SharedPointer< Vector >& data,
const IndexType& offset )
-: meshPointer( meshPointer )
+//: meshPointer( meshPointer )
{
+<<<<<<< HEAD
TNL_ASSERT_GE( data->getSize(), offset + meshPointer->template getEntitiesCount< typename MeshType::template EntityType< MeshEntityDimension > >(),
"The input vector is not large enough for binding to the mesh function." );
+=======
+
+ SetupSynchronizer(meshPointer->GetDistMesh());
+
+ this->meshPointer=meshPointer;
+>>>>>>> origin/mpi-explosive
this->data.bind( *data, offset, getMesh().template getEntitiesCount< typename Mesh::template EntityType< MeshEntityDimension > >() );
}
@@ -174,7 +194,10 @@ void
MeshFunction< Mesh, MeshEntityDimension, Real >::
bind( ThisType& meshFunction )
{
- this->meshPointer = meshFunction.getMeshPointer();
+
+ SetupSynchronizer(meshFunction.meshPointer->GetDistMesh());
+
+ this->meshPointer=meshFunction.meshPointer;
this->data.bind( meshFunction.getData() );
}
@@ -188,9 +211,16 @@ bind( const MeshPointer& meshPointer,
const Vector& data,
const IndexType& offset )
{
+<<<<<<< HEAD
TNL_ASSERT_GE( data.getSize(), offset + meshPointer->template getEntitiesCount< typename MeshType::template EntityType< MeshEntityDimension > >(),
"The input vector is not large enough for binding to the mesh function." );
this->meshPointer = meshPointer;
+=======
+
+ SetupSynchronizer(meshPointer->GetDistMesh());
+
+ this->meshPointer=meshPointer;
+>>>>>>> origin/mpi-explosive
this->data.bind( data, offset, getMesh().template getEntitiesCount< typename Mesh::template EntityType< MeshEntityDimension > >() );
}
@@ -204,9 +234,16 @@ bind( const MeshPointer& meshPointer,
const SharedPointer< Vector >& data,
const IndexType& offset )
{
+<<<<<<< HEAD
TNL_ASSERT_GE( data->getSize(), offset + meshPointer->template getEntitiesCount< typename MeshType::template EntityType< MeshEntityDimension > >(),
"The input vector is not large enough for binding to the mesh function." );
this->meshPointer = meshPointer;
+=======
+
+ SetupSynchronizer(meshPointer->GetDistMesh());
+
+ this->meshPointer=meshPointer;
+>>>>>>> origin/mpi-explosive
this->data.bind( *data, offset, getMesh().template getEntitiesCount< typename Mesh::template EntityType< MeshEntityDimension > >() );
}
@@ -218,7 +255,10 @@ void
MeshFunction< Mesh, MeshEntityDimension, Real >::
setMesh( const MeshPointer& meshPointer )
{
- this->meshPointer = meshPointer;
+
+ SetupSynchronizer(meshPointer->GetDistMesh());
+
+ this->meshPointer=meshPointer;
this->data.setSize( getMesh().template getEntitiesCount< typename Mesh::template EntityType< MeshEntityDimension > >() );
}
@@ -444,7 +484,7 @@ save( File& file ) const
std::cerr << "this->data.getSize() = " << this->data.getSize() << std::endl
<< "this->getMesh().template getEntitiesCount< typename MeshType::template EntityType< MeshEntityDimension > >() = " << this->getMesh().template getEntitiesCount< typename MeshType::template EntityType< MeshEntityDimension > >() );
if( ! Object::save( file ) )
- return false;
+ return false;
return this->data.save( file );
}
@@ -456,7 +496,7 @@ MeshFunction< Mesh, MeshEntityDimension, Real >::
load( File& file )
{
if( ! Object::load( file ) )
- return false;
+ return false;
if( ! this->data.load( file ) )
return false;
const IndexType meshSize = this->getMesh().template getEntitiesCount< typename MeshType::template EntityType< MeshEntityDimension > >();
@@ -506,6 +546,38 @@ write( const String& fileName,
}
return true;
}
+
+template< typename Mesh,
+ int MeshEntityDimension,
+ typename Real >
+template< typename CommunicatorType>
+void
+MeshFunction< Mesh, MeshEntityDimension, Real >::
+Synchronize()
+{
+ auto distrMesh = this->getMesh().GetDistMesh();
+ if(distrMesh != NULL && distrMesh->IsDistributed())
+ {
+ this->synchronizer.template Synchronize<CommunicatorType>(*this);
+ }
+}
+
+template< typename Mesh,
+ int MeshEntityDimension,
+ typename Real >
+void
+MeshFunction< Mesh, MeshEntityDimension, Real >::
+SetupSynchronizer(DistributedMeshType *distrMesh )
+{
+ if(distrMesh)//pokud sÃÅ¥ kterou se snažà uživatel nastavit je distruibuovaná
+ {
+ if(this->getMesh().GetDistMesh()!=distrMesh)//pokud má nová sÃÅ¥ jinou distribuovanou sÃÅ¥ než pÅ™edchozÃ
+ {
+ this->synchronizer.SetDistributedGrid(distrMesh);
+ }
+ }
+}
+
} // namespace Functions
} // namespace TNL
diff --git a/src/TNL/Meshes/CMakeLists.txt b/src/TNL/Meshes/CMakeLists.txt
index 2f3f3f634dc0b80c16312b9d703c08bd1975b94a..9dd51e9b858050cb095d3a3cb68b672a3ba263ae 100644
--- a/src/TNL/Meshes/CMakeLists.txt
+++ b/src/TNL/Meshes/CMakeLists.txt
@@ -1,6 +1,7 @@
ADD_SUBDIRECTORY( Geometry )
ADD_SUBDIRECTORY( GridDetails )
ADD_SUBDIRECTORY( MeshDetails )
+ADD_SUBDIRECTORY( DistributedMeshes )
ADD_SUBDIRECTORY( Readers )
ADD_SUBDIRECTORY( Topologies )
ADD_SUBDIRECTORY( TypeResolver )
@@ -24,3 +25,4 @@ SET( tnl_mesh_SOURCES ${tnl_mesh_griddetails_SOURCES}
PARENT_SCOPE )
INSTALL( FILES ${headers} DESTINATION ${TNL_TARGET_INCLUDE_DIRECTORY}/Meshes )
+
diff --git a/src/TNL/Meshes/DistributedMeshes/BufferEntitiesHelper.h b/src/TNL/Meshes/DistributedMeshes/BufferEntitiesHelper.h
new file mode 100644
index 0000000000000000000000000000000000000000..7655bc67937d99645370e3380782f1957bf3a4a8
--- /dev/null
+++ b/src/TNL/Meshes/DistributedMeshes/BufferEntitiesHelper.h
@@ -0,0 +1,125 @@
+/***************************************************************************
+ BufferEntittiesHelper.h - description
+ -------------------
+ begin : March 1, 2018
+ copyright : (C) 2018 by Tomas Oberhuber
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+/* See Copyright Notice in tnl/Copyright */
+
+#pragma once
+
+#include <TNL/Devices/Host.h>
+#include <TNL/Devices/Cuda.h>
+#include <TNL/ParallelFor.h>
+
+namespace TNL {
+namespace Meshes {
+namespace DistributedMeshes {
+
+
+template < typename MeshFunctionType,
+ int dim,
+ typename RealType=typename MeshFunctionType::MeshType::RealType,
+ typename Device=typename MeshFunctionType::MeshType::DeviceType,
+ typename Index=typename MeshFunctionType::MeshType::GlobalIndexType >
+class BufferEntitiesHelper
+{
+};
+
+//======================================== 1D ====================================================
+
+//host
+template < typename MeshFunctionType, typename RealType, typename Device, typename Index >
+class BufferEntitiesHelper<MeshFunctionType,1,RealType,Device,Index>
+{
+ public:
+ static void BufferEntities(MeshFunctionType meshFunction, RealType * buffer, Index beginx, Index sizex, bool tobuffer)
+ {
+ auto mesh = meshFunction.getMesh();
+ RealType* meshFunctionData = meshFunction.getData().getData();
+ auto kernel = [tobuffer, mesh, buffer, meshFunctionData, beginx] __cuda_callable__ ( Index j )
+ {
+ typename MeshFunctionType::MeshType::Cell entity(mesh);
+ entity.getCoordinates().x()=beginx+j;
+ entity.refresh();
+ if(tobuffer)
+ buffer[j]=meshFunctionData[entity.getIndex()];
+ else
+ meshFunctionData[entity.getIndex()]=buffer[j];
+ };
+ ParallelFor< Device >::exec( 0, sizex, kernel );
+ };
+};
+
+
+//======================================== 2D ====================================================
+template <typename MeshFunctionType, typename RealType, typename Device, typename Index >
+class BufferEntitiesHelper<MeshFunctionType,2,RealType,Device,Index>
+{
+ public:
+ static void BufferEntities(MeshFunctionType meshFunction, RealType * buffer, Index beginx, Index beginy, Index sizex, Index sizey,bool tobuffer)
+ {
+ auto mesh=meshFunction.getMesh();
+ RealType *meshFunctionData=meshFunction.getData().getData();
+ auto kernel = [tobuffer, mesh, buffer, meshFunctionData, beginx, sizex, beginy] __cuda_callable__ ( Index i, Index j )
+ {
+ typename MeshFunctionType::MeshType::Cell entity(mesh);
+ entity.getCoordinates().x()=beginx+j;
+ entity.getCoordinates().y()=beginy+i;
+ entity.refresh();
+ if(tobuffer)
+ buffer[i*sizex+j]=meshFunctionData[entity.getIndex()];
+ else
+ meshFunctionData[entity.getIndex()]=buffer[i*sizex+j];
+ };
+
+ ParallelFor2D< Device >::exec( 0, 0, sizey, sizex, kernel );
+
+ };
+};
+
+
+//======================================== 3D ====================================================
+template <typename MeshFunctionType, typename RealType, typename Device, typename Index >
+class BufferEntitiesHelper<MeshFunctionType,3,RealType,Device,Index>
+{
+ public:
+ static void BufferEntities(MeshFunctionType meshFunction, RealType * buffer, Index beginx, Index beginy, Index beginz, Index sizex, Index sizey, Index sizez, bool tobuffer)
+ {
+
+ auto mesh=meshFunction.getMesh();
+ RealType * meshFunctionData=meshFunction.getData().getData();
+ auto kernel = [tobuffer, mesh, buffer, meshFunctionData, beginx, sizex, beginy, sizey, beginz] __cuda_callable__ ( Index k, Index i, Index j )
+ {
+ typename MeshFunctionType::MeshType::Cell entity(mesh);
+ entity.getCoordinates().x()=beginx+j;
+ entity.getCoordinates().z()=beginz+k;
+ entity.getCoordinates().y()=beginy+i;
+ entity.refresh();
+ if(tobuffer)
+ buffer[k*sizex*sizey+i*sizex+j]=meshFunctionData[entity.getIndex()];
+ else
+ meshFunctionData[entity.getIndex()]=buffer[k*sizex*sizey+i*sizex+j];
+ };
+
+ ParallelFor3D< Device >::exec( 0, 0, 0, sizez, sizey, sizex, kernel );
+
+ /*for(int k=0;k<sizez;k++)
+ {
+ for(int i=0;i<sizey;i++)
+ {
+ for(int j=0;j<sizex;j++)
+ {
+ kernel(k,i,j);
+ }
+ }
+ }*/
+ };
+};
+
+
+} // namespace DistributedMeshes
+} // namespace Meshes
+} // namespace TNL
diff --git a/src/TNL/Meshes/DistributedMeshes/CMakeLists.txt b/src/TNL/Meshes/DistributedMeshes/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..abe91d0d467a222ea7868740d3726115842d2572
--- /dev/null
+++ b/src/TNL/Meshes/DistributedMeshes/CMakeLists.txt
@@ -0,0 +1,10 @@
+SET( headers DistributedMesh.h
+ DistributedMeshSynchronizer.h
+ DistributedGrid_1D.h
+ DistributedGrid_2D.h
+ DistributedGrid_3D.h
+ DistributedGridSynchronizer.h
+ DistributedGridIO.h
+ )
+
+INSTALL( FILES ${headers} DESTINATION ${TNL_TARGET_INCLUDE_DIRECTORY}/Meshes/DistributedMesh )
diff --git a/src/TNL/Meshes/DistributedMeshes/CopyEntitiesHelper.h b/src/TNL/Meshes/DistributedMeshes/CopyEntitiesHelper.h
new file mode 100644
index 0000000000000000000000000000000000000000..fe2f82cffe1a54fb03f51e3aa6b85e8952397718
--- /dev/null
+++ b/src/TNL/Meshes/DistributedMeshes/CopyEntitiesHelper.h
@@ -0,0 +1,136 @@
+/***************************************************************************
+ CopyEntitiesHelper.h - description
+ -------------------
+ begin : March 8, 2018
+ copyright : (C) 2018 by Tomas Oberhuber
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+/* See Copyright Notice in tnl/Copyright */
+
+#pragma once
+
+#include <TNL/Devices/Host.h>
+#include <TNL/Devices/Cuda.h>
+#include <TNL/ParallelFor.h>
+
+namespace TNL {
+namespace Meshes {
+namespace DistributedMeshes {
+
+template<typename MeshFunctionType,
+ int dim=MeshFunctionType::getMeshDimension()>
+class CopyEntitiesHelper
+{
+ public:
+ typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;
+ static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)
+ {
+ }
+
+};
+
+
+template<typename MeshFunctionType>
+class CopyEntitiesHelper<MeshFunctionType, 1>
+{
+ public:
+ typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;
+ typedef typename MeshFunctionType::MeshType::Cell Cell;
+ typedef typename MeshFunctionType::MeshType::GlobalIndexType Index;
+
+ static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)
+ {
+ auto toData=to.getData().getData();
+ auto fromData=from.getData().getData();
+ auto fromMesh=from.getMesh();
+ auto toMesh=to.getMesh();
+ auto kernel = [fromData,toData, fromMesh, toMesh, fromBegin, toBegin] __cuda_callable__ ( Index i )
+ {
+ Cell fromEntity(fromMesh);
+ Cell toEntity(toMesh);
+ toEntity.getCoordinates().x()=toBegin.x()+i;
+ toEntity.refresh();
+ fromEntity.getCoordinates().x()=fromBegin.x()+i;
+ fromEntity.refresh();
+ toData[toEntity.getIndex()]=fromData[fromEntity.getIndex()];
+ };
+ ParallelFor< typename MeshFunctionType::MeshType::DeviceType >::exec( (Index)0, (Index)size.x(), kernel );
+
+ }
+
+};
+
+
+template<typename MeshFunctionType>
+
+class CopyEntitiesHelper<MeshFunctionType,2>
+{
+ public:
+ typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;
+ typedef typename MeshFunctionType::MeshType::Cell Cell;
+ typedef typename MeshFunctionType::MeshType::GlobalIndexType Index;
+
+ static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)
+ {
+ auto toData=to.getData().getData();
+ auto fromData=from.getData().getData();
+ auto fromMesh=from.getMesh();
+ auto toMesh=to.getMesh();
+ auto kernel = [fromData,toData, fromMesh, toMesh, fromBegin, toBegin] __cuda_callable__ ( Index j, Index i )
+ {
+ Cell fromEntity(fromMesh);
+ Cell toEntity(toMesh);
+ toEntity.getCoordinates().x()=toBegin.x()+i;
+ toEntity.getCoordinates().y()=toBegin.y()+j;
+ toEntity.refresh();
+ fromEntity.getCoordinates().x()=fromBegin.x()+i;
+ fromEntity.getCoordinates().y()=fromBegin.y()+j;
+ fromEntity.refresh();
+ toData[toEntity.getIndex()]=fromData[fromEntity.getIndex()];
+ };
+ ParallelFor2D< typename MeshFunctionType::MeshType::DeviceType >::exec( (Index)0,(Index)0,(Index)size.y(), (Index)size.x(), kernel );
+
+ }
+
+};
+
+
+template<typename MeshFunctionType>
+class CopyEntitiesHelper<MeshFunctionType,3>
+{
+ public:
+ typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;
+ typedef typename MeshFunctionType::MeshType::Cell Cell;
+ typedef typename MeshFunctionType::MeshType::GlobalIndexType Index;
+
+ static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)
+ {
+ auto toData=to.getData().getData();
+ auto fromData=from.getData().getData();
+ auto fromMesh=from.getMesh();
+ auto toMesh=to.getMesh();
+ auto kernel = [fromData,toData, fromMesh, toMesh, fromBegin, toBegin] __cuda_callable__ ( Index k, Index j, Index i )
+ {
+ Cell fromEntity(fromMesh);
+ Cell toEntity(toMesh);
+ toEntity.getCoordinates().x()=toBegin.x()+i;
+ toEntity.getCoordinates().y()=toBegin.y()+j;
+ toEntity.getCoordinates().z()=toBegin.z()+k;
+ toEntity.refresh();
+ fromEntity.getCoordinates().x()=fromBegin.x()+i;
+ fromEntity.getCoordinates().y()=fromBegin.y()+j;
+ fromEntity.getCoordinates().z()=fromBegin.z()+k;
+ fromEntity.refresh();
+ toData[toEntity.getIndex()]=fromData[fromEntity.getIndex()];
+ };
+ ParallelFor3D< typename MeshFunctionType::MeshType::DeviceType >::exec( (Index)0,(Index)0,(Index)0,(Index)size.z() ,(Index)size.y(), (Index)size.x(), kernel );
+ }
+};
+
+
+
+
+} // namespace DistributedMeshes
+} // namespace Meshes
+} // namespace TNL
diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridIO.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridIO.h
new file mode 100644
index 0000000000000000000000000000000000000000..931252d53f857163b71afb03ae76b8066443f846
--- /dev/null
+++ b/src/TNL/Meshes/DistributedMeshes/DistributedGridIO.h
@@ -0,0 +1,401 @@
+/***************************************************************************
+ DistributedGrid.h - description
+ -------------------
+ begin : October 5, 2017
+ copyright : (C) 2017 by Tomas Oberhuber
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+/* See Copyright Notice in tnl/Copyright */
+
+#pragma once
+
+#include <TNL/File.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+#include <TNL/Meshes/DistributedMeshes/CopyEntitiesHelper.h>
+#include <TNL/Functions/MeshFunction.h>
+
+
+#include <iostream>
+
+#ifdef MPIIO
+#include <TNL/Communicators/MpiCommunicator.h>
+#endif
+
+namespace TNL {
+namespace Meshes {
+namespace DistributedMeshes {
+
+enum DistrGridIOTypes { Dummy = 0 , LocalCopy = 1, MpiIO=2 };
+
+template<typename MeshFunctionType,
+ DistrGridIOTypes type = LocalCopy>
+class DistributedGridIO
+{
+};
+
+template<typename MeshFunctionType>
+class DistributedGridIO<MeshFunctionType,Dummy>
+{
+ bool save(const String& fileName, MeshFunctionType &meshFunction)
+ {
+ return true;
+ };
+
+ bool load(const String& fileName, MeshFunctionType &meshFunction)
+ {
+ return true;
+ };
+};
+
+
+/*
+ * This variant cerate copy of MeshFunction but smaler, reduced to local entites, without overlap.
+ * It slow and has high RAM consupation
+ */
+template<typename MeshFunctionType>
+class DistributedGridIO<MeshFunctionType,LocalCopy>
+{
+
+ public:
+
+ typedef typename MeshFunctionType::MeshType MeshType;
+ typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;
+ typedef typename MeshFunctionType::MeshType::PointType PointType;
+ typedef typename MeshFunctionType::VectorType VectorType;
+ //typedef DistributedGrid< MeshType,MeshFunctionType::getMeshDimension()> DistributedGridType;
+
+ static bool save(const String& fileName, MeshFunctionType &meshFunction)
+ {
+ auto *distrGrid=meshFunction.getMesh().GetDistMesh();
+
+ if(distrGrid==NULL) //not distributed
+ {
+ return meshFunction.save(fileName);
+ }
+
+ MeshType mesh=meshFunction.getMesh();
+
+ PointType spaceSteps=mesh.getSpaceSteps();
+ PointType origin=mesh.getOrigin();
+
+ CoordinatesType localSize=distrGrid->getLocalSize();
+ CoordinatesType localBegin=distrGrid->getLocalBegin();
+
+ SharedPointer<MeshType> newMesh;
+ newMesh->setDimensions(localSize);
+ newMesh->setSpaceSteps(spaceSteps);
+ newMesh->setOrigin(origin+TNL::Containers::tnlDotProduct(spaceSteps,localBegin));
+
+ File meshFile;
+ meshFile.open( fileName+String("-mesh-")+distrGrid->printProcessCoords()+String(".tnl"),IOMode::write);
+ newMesh->save( meshFile );
+ meshFile.close();
+
+ VectorType newDof(newMesh-> template getEntitiesCount< typename MeshType::Cell >());
+
+ MeshFunctionType newMeshFunction;
+ newMeshFunction.bind(newMesh,newDof);
+
+ CoordinatesType zeroCoord;
+ zeroCoord.setValue(0);
+
+ CopyEntitiesHelper<MeshFunctionType>::Copy(meshFunction,newMeshFunction,localBegin,zeroCoord,localSize);
+
+ File file;
+ file.open( fileName+String("-")+distrGrid->printProcessCoords()+String(".tnl"), IOMode::write );
+ bool ret=newMeshFunction.save(file);
+ file.close();
+
+ return ret;
+
+ };
+
+ static bool load(const String& fileName,MeshFunctionType &meshFunction)
+ {
+ auto *distrGrid=meshFunction.getMesh().GetDistMesh();
+ if(distrGrid==NULL) //not distributed
+ {
+ return meshFunction.boundLoad(fileName);
+ }
+
+ MeshType mesh=meshFunction.getMesh();
+
+ PointType spaceSteps=mesh.getSpaceSteps();
+ PointType origin=mesh.getOrigin();
+
+ CoordinatesType localSize=distrGrid->getLocalSize();
+ CoordinatesType localBegin=distrGrid->getLocalBegin();
+
+ SharedPointer<MeshType> newMesh;
+ newMesh->setDimensions(localSize);
+ newMesh->setSpaceSteps(spaceSteps);
+ newMesh->setOrigin(origin+TNL::Containers::tnlDotProduct(spaceSteps,localBegin));
+
+ VectorType newDof(newMesh-> template getEntitiesCount< typename MeshType::Cell >());
+ MeshFunctionType newMeshFunction;
+ newMeshFunction.bind(newMesh,newDof);
+
+ CoordinatesType zeroCoord;
+ zeroCoord.setValue(0);
+
+ File file;
+ file.open( fileName+String("-")+distrGrid->printProcessCoords()+String(".tnl"), IOMode::read );
+ bool result=newMeshFunction.boundLoad(file);
+ file.close();
+ CopyEntitiesHelper<MeshFunctionType>::Copy(newMeshFunction,meshFunction,zeroCoord,localBegin,localSize);
+
+ return result;
+ };
+
+};
+
+/*
+ * Save distributed data into single file without overlaps using MPIIO and MPI datatypes,
+ * EXPLOSIVE: works with only Grids and MPI
+ * BAD IMPLEMENTTION creating MPI-Types at every save! -- I dont want contamine more places by MPI..
+ */
+
+#ifdef MPIIO
+template<typename MeshFunctionType>
+class DistributedGridIO<MeshFunctionType,MpiIO>
+{
+
+ public:
+
+ typedef typename MeshFunctionType::MeshType MeshType;
+ typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;
+ typedef typename MeshFunctionType::MeshType::PointType PointType;
+ typedef typename MeshFunctionType::VectorType VectorType;
+ //typedef DistributedGrid< MeshType,MeshFunctionType::getMeshDimension()> DistributedGridType;
+
+ static bool save(const String& fileName, MeshFunctionType &meshFunction)
+ {
+ auto *distrGrid=meshFunction.getMesh().GetDistMesh();
+
+ if(distrGrid==NULL) //not distributed
+ {
+ return meshFunction.save(fileName);
+ }
+
+ MPI_Datatype ftype;
+ MPI_Datatype atype;
+ int dataCount=CreateDataTypes(distrGrid,&ftype,&atype);
+
+ double * data=meshFunction.getData().getData();//TYP
+
+ //write
+ MPI_File file;
+ MPI_File_open(MPI_COMM_WORLD,fileName.getString(),
+ MPI_MODE_CREATE|MPI_MODE_WRONLY,
+ MPI_INFO_NULL, &file);
+
+ int headerSize=0;
+
+ if(Communicators::MpiCommunicator::GetRank()==0)
+ {
+ headerSize=writeMeshFunctionHeader(file,meshFunction,dataCount);
+ }
+ MPI_Bcast(&headerSize, 1, MPI_INT,0, MPI_COMM_WORLD);
+
+ MPI_File_set_view(file,headerSize,MPI_DOUBLE,ftype,"native",MPI_INFO_NULL);//TYP
+ MPI_Status wstatus;
+
+ MPI_File_write(file,data,1,atype,&wstatus);
+
+ MPI_File_close(&file);
+
+ MPI_Type_free(&atype);
+ MPI_Type_free(&ftype);
+
+ return true;
+ };
+
+ template<typename DitsributedGridType>
+ static int CreateDataTypes(DitsributedGridType *distrGrid,MPI_Datatype *ftype,MPI_Datatype *atype)
+ {
+ int dim=distrGrid->getMeshDimension();
+
+ int fstarts[dim];
+ int flsize[dim];
+ int fgsize[dim];
+
+ hackArray(dim,fstarts,distrGrid->getGlobalBegin().getData());
+ hackArray(dim,flsize,distrGrid->getLocalSize().getData());
+ hackArray(dim,fgsize,distrGrid->getGlobalSize().getData());
+
+ MPI_Type_create_subarray(dim,
+ fgsize,flsize,fstarts,
+ MPI_ORDER_C,MPI_DOUBLE,ftype); //TYP
+ MPI_Type_commit(ftype);
+
+ int agsize[dim];
+ int alsize[dim];
+ int astarts[dim];
+
+ hackArray(dim,astarts,distrGrid->getLocalBegin().getData());
+ hackArray(dim,alsize,distrGrid->getLocalSize().getData());
+ hackArray(dim,agsize,distrGrid->getLocalGridSize().getData());
+
+ MPI_Type_create_subarray(dim,
+ agsize,alsize,astarts,
+ MPI_ORDER_C,MPI_DOUBLE,atype); //TYP
+ MPI_Type_commit(atype);
+
+ int dataCount=1;
+ for(int i=0;i<dim;i++)
+ dataCount*=fgsize[i];
+
+ return dataCount;
+
+ }
+
+ template<typename Index>
+ static void hackArray(int dim, int* out, Index* in)
+ {
+ if(dim==1)
+ {
+ out[0]=in[0];
+ }
+
+ if(dim==2)
+ {
+ out[1]=in[0];
+ out[0]=in[1];
+ }
+
+ if(dim==3)
+ {
+ out[0]=in[2];
+ out[1]=in[1];
+ out[2]=in[0];
+ }
+ }
+
+ static unsigned int writeMeshFunctionHeader(MPI_File file,MeshFunctionType &meshFunction, int length )
+ {
+
+ unsigned int size=0;
+ int count;
+ MPI_Status wstatus;
+
+ //Magic
+ MPI_File_write(file,"TNLMN",5,MPI_CHAR,&wstatus);
+ MPI_Get_count(&wstatus,MPI_CHAR,&count);
+ size+=count*sizeof(char);
+
+ //Meshfunction type
+ String meshFunctionSerializationType=meshFunction.getSerializationTypeVirtual();
+ int meshFunctionSerializationTypeLength=meshFunctionSerializationType.getLength();
+ MPI_File_write(file,&meshFunctionSerializationTypeLength,1,MPI_INT,&wstatus);
+ MPI_Get_count(&wstatus,MPI_INT,&count);
+ size+=count*sizeof(int);
+ MPI_File_write(file,meshFunctionSerializationType.getString(),meshFunctionSerializationType.getLength(),MPI_CHAR,&wstatus);
+ MPI_Get_count(&wstatus,MPI_CHAR,&count);
+ size+=count*sizeof(char);
+
+ //Magic
+ MPI_File_write(file,"TNLMN",5,MPI_CHAR,&wstatus);
+ MPI_Get_count(&wstatus,MPI_CHAR,&count);
+ size+=count*sizeof(char);
+ //Vector Type
+ String dataSerializationType=meshFunction.getData().getSerializationTypeVirtual();
+ int dataSerializationTypeLength=dataSerializationType.getLength();
+ MPI_File_write(file,&dataSerializationTypeLength,1,MPI_INT,&wstatus);
+ MPI_Get_count(&wstatus,MPI_INT,&count);
+ size+=count*sizeof(int);
+ MPI_File_write(file,dataSerializationType.getString(),dataSerializationType.getLength(),MPI_CHAR,&wstatus);
+ MPI_Get_count(&wstatus,MPI_CHAR,&count);
+ size+=count*sizeof(char);
+ //Data count
+ MPI_File_write(file,&(length),1,MPI_INT,&wstatus);
+ MPI_Get_count(&wstatus,MPI_INT,&count);
+ size+=count*sizeof(int);
+
+ return size;
+ };
+
+ /* Funky bomb - no checks - only dirty load */
+ static bool load(const String& fileName,MeshFunctionType &meshFunction)
+ {
+ auto *distrGrid=meshFunction.getMesh().GetDistMesh();
+ if(distrGrid==NULL) //not distributed
+ {
+ return meshFunction.boundLoad(fileName);
+ }
+
+ MPI_Datatype ftype;
+ MPI_Datatype atype;
+ int dataCount=CreateDataTypes(distrGrid,&ftype,&atype);
+
+ double * data=meshFunction.getData().getData();//TYP
+
+ //write
+ MPI_File file;
+ MPI_File_open(MPI_COMM_WORLD,fileName.getString(),
+ MPI_MODE_RDONLY,
+ MPI_INFO_NULL, &file);
+
+ int headerSize=0;
+
+ if(Communicators::MpiCommunicator::GetRank()==0)
+ {
+ headerSize=readMeshFunctionHeader(file,meshFunction,dataCount);
+ }
+ MPI_Bcast(&headerSize, 1, MPI_INT,0, MPI_COMM_WORLD);
+
+ if(headerSize<0)
+ return false;
+
+ MPI_File_set_view(file,headerSize,MPI_DOUBLE,ftype,"native",MPI_INFO_NULL);//TYP
+ MPI_Status wstatus;
+ MPI_File_read(file,(void*)data,1,atype,&wstatus);
+ MPI_File_close(&file);
+
+ MPI_Type_free(&atype);
+ MPI_Type_free(&ftype);
+
+ return true;
+ };
+
+ //tak mohlo by to něco kontrolovat...ale nic to nekontroluje
+ static int readMeshFunctionHeader(MPI_File file,MeshFunctionType &meshFunction, int length)
+ {
+ MPI_Status rstatus;
+ char buffer[255];
+ int size=0;
+ int count=0;
+
+ MPI_File_read(file, (void *)buffer,5, MPI_CHAR, &rstatus);//MAGIC
+ size+=5*sizeof(char);
+ MPI_File_read(file, (void *)&count,1, MPI_INT, &rstatus);//SIZE OF DATATYPE
+ size+=1*sizeof(int);
+ MPI_File_read(file, (void *)&buffer,count, MPI_CHAR, &rstatus);//DATATYPE
+ size+=count*sizeof(char);
+
+ MPI_File_read(file, (void *)buffer,5, MPI_CHAR, &rstatus);//MAGIC
+ size+=5*sizeof(char);
+ MPI_File_read(file, (void *)&count,1, MPI_INT, &rstatus);//SIZE OF DATATYPE
+ size+=1*sizeof(int);
+ MPI_File_read(file, (void *)&buffer,count, MPI_CHAR, &rstatus);//DATATYPE
+ size+=count*sizeof(char);
+ MPI_File_read(file, (void *)&count,1, MPI_INT, &rstatus);//DATACOUNT
+ size+=1*sizeof(int);
+
+ if(count!=length)
+ {
+ std::cerr<<"Chyba naÄÃtánà MeshFunction, délka dat v souboru neodpovÃdá oÄekávané délce" << std::endl;
+ size=-1;
+ }
+
+ return size;
+ };
+
+};
+
+#endif
+
+
+}
+}
+}
diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h
new file mode 100644
index 0000000000000000000000000000000000000000..fcf4c35ec772f46a61e5c4661aa424d5aa5c6082
--- /dev/null
+++ b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h
@@ -0,0 +1,542 @@
+/***************************************************************************
+ DistributedGrid.h - description
+ -------------------
+ begin : March 17, 2017
+ copyright : (C) 2017 by Tomas Oberhuber
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+/* See Copyright Notice in tnl/Copyright */
+
+#pragma once
+
+#include <TNL/Meshes/Grid.h>
+#include <TNL/Containers/Array.h>
+#include <TNL/Meshes/DistributedMeshes/BufferEntitiesHelper.h>
+
+
+namespace TNL {
+namespace Functions{
+template< typename Mesh,
+ int MeshEntityDimension,
+ typename Real >
+class MeshFunction;
+}//Functions
+}//TNL
+
+namespace TNL {
+namespace Meshes {
+namespace DistributedMeshes {
+
+
+//=============================================1D==================================
+
+template <typename RealType,
+ int EntityDimension,
+ typename Index,
+ typename Device,
+ typename GridReal>
+class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 1, GridReal, Device, Index >,EntityDimension, RealType>>
+{
+
+public:
+ typedef typename Grid< 1, GridReal, Device, Index >::Cell Cell;
+ typedef typename Functions::MeshFunction< Grid< 1, GridReal, Device, Index >,EntityDimension, RealType> MeshFunctionType;
+ typedef typename Grid< 1, GridReal, Device, Index >::DistributedMeshType DistributedGridType;
+ typedef RealType Real;
+
+
+private:
+ Containers::Array<RealType, Device> sendbuffs[2];
+ Containers::Array<RealType, Device> rcvbuffs[2];
+ int overlapSize;
+
+ DistributedGridType *distributedgrid;
+
+ bool isSet;
+
+
+ public:
+ DistributedMeshSynchronizer()
+ {
+ isSet=false;
+ };
+
+ DistributedMeshSynchronizer(DistributedGridType *distrgrid)
+ {
+ isSet=false;
+ SetDistributedGrid(distrgrid);
+ };
+
+ void SetDistributedGrid(DistributedGridType *distrgrid)
+ {
+ isSet=true;
+
+ this->distributedgrid=distrgrid;
+
+ overlapSize = distributedgrid->getOverlap().x();
+
+ sendbuffs[0].setSize(overlapSize);
+ sendbuffs[1].setSize(overlapSize);
+ rcvbuffs[0].setSize(overlapSize);
+ rcvbuffs[1].setSize(overlapSize);
+ };
+
+ template<typename CommunicatorType>
+ void Synchronize(MeshFunctionType &meshfunction)
+ {
+ TNL_ASSERT_TRUE(isSet,"Synchronizer is not set, but used to Synchronize");
+
+ if(!distributedgrid->IsDistributed())
+ return;
+
+ int leftN=distributedgrid->getLeft();
+ int rightN=distributedgrid->getRight();
+
+ int totalSize = meshfunction.getMesh().getDimensions().x();
+
+ CopyBuffers(meshfunction, sendbuffs, true,
+ overlapSize, totalSize-2*overlapSize, overlapSize,
+ leftN, rightN);
+
+ //async send
+ typename CommunicatorType::Request req[4];
+
+ //send everithing, recieve everything
+ if(leftN!=-1)
+ {
+ req[0]=CommunicatorType::ISend(sendbuffs[Left].getData(), overlapSize, leftN);
+ req[2]=CommunicatorType::IRecv(rcvbuffs[Left].getData(), overlapSize, leftN);
+ }
+ else
+ {
+ req[0]=CommunicatorType::NullRequest;
+ req[2]=CommunicatorType::NullRequest;
+ }
+
+ if(rightN!=-1)
+ {
+ req[1]=CommunicatorType::ISend(sendbuffs[Right].getData(), overlapSize, rightN);
+ req[3]=CommunicatorType::IRecv(rcvbuffs[Right].getData(), overlapSize, rightN);
+ }
+ else
+ {
+ req[1]=CommunicatorType::NullRequest;
+ req[3]=CommunicatorType::NullRequest;
+ }
+
+ //wait until send and recv is done
+ CommunicatorType::WaitAll(req, 4);
+
+ CopyBuffers(meshfunction, rcvbuffs, false,
+ 0, totalSize-overlapSize, overlapSize,
+ leftN, rightN);
+ }
+
+ private:
+ template <typename Real>
+ void CopyBuffers(MeshFunctionType meshfunction, TNL::Containers::Array<Real,Device> * buffers, bool toBuffer,
+ int left, int right,
+ int size,
+ int leftNeighbor, int rightNeighbor)
+ {
+ if(leftNeighbor!=-1)
+ BufferEntitiesHelper<MeshFunctionType,1,Real,Device>::BufferEntities(meshfunction,buffers[Left].getData(),left,size,toBuffer);
+ if(rightNeighbor!=-1)
+ BufferEntitiesHelper<MeshFunctionType,1,Real,Device>::BufferEntities(meshfunction,buffers[Right].getData(),right,size,toBuffer);
+ }
+
+};
+
+//=========================2D=================================================
+template <typename RealType,
+ int EntityDimension,
+ typename Index,
+ typename Device,
+ typename GridReal>
+class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 2, GridReal, Device, Index >,EntityDimension, RealType>>
+{
+
+ public:
+ typedef typename Grid< 2, GridReal, Device, Index >::Cell Cell;
+ typedef typename Functions::MeshFunction< Grid< 2, GridReal, Device, Index >,EntityDimension, RealType> MeshFunctionType;
+ typedef typename Grid< 2, GridReal, Device, Index >::DistributedMeshType DistributedGridType;
+ typedef typename MeshFunctionType::RealType Real;
+ typedef typename DistributedGridType::CoordinatesType CoordinatesType;
+
+ private:
+ DistributedGridType *distributedgrid;
+
+ Containers::Array<RealType, Device, Index> sendbuffs[8];
+ Containers::Array<RealType, Device, Index> rcvbuffs[8];
+ int sizes[8];
+
+ int leftSrc;
+ int rightSrc;
+ int upSrc;
+ int downSrc;
+ int xcenter;
+ int ycenter;
+ int leftDst;
+ int rightDst;
+ int upDst;
+ int downDst;
+
+ CoordinatesType overlap;
+ CoordinatesType localsize;
+
+ bool isSet;
+
+ public:
+ DistributedMeshSynchronizer()
+ {
+ isSet=false;
+ };
+
+ DistributedMeshSynchronizer(DistributedGridType *distrgrid)
+ {
+ isSet=false;
+ SetDistributedGrid(distrgrid);
+ };
+
+ void SetDistributedGrid(DistributedGridType *distrgrid)
+ {
+ isSet=true;
+
+ this->distributedgrid=distrgrid;
+
+ overlap = distributedgrid->getOverlap();
+ localsize = distributedgrid->getLocalSize();
+
+ CoordinatesType localgridsize = this->distributedgrid->getLocalGridSize();
+ CoordinatesType localbegin=this->distributedgrid->getLocalBegin();
+
+ int updownsize=localsize.x()*overlap.y();
+ int leftrightsize=localsize.y()*overlap.x();
+ int connersize=overlap.x()*overlap.y();
+
+ sizes[Left]=leftrightsize;
+ sizes[Right]=leftrightsize;
+ sizes[Up]=updownsize;
+ sizes[Down]=updownsize;
+ sizes[UpLeft]=connersize;
+ sizes[DownLeft]=connersize;
+ sizes[UpRight]=connersize;
+ sizes[DownRight]=connersize;
+
+ for(int i=0;i<8;i++)
+ {
+ sendbuffs[i].setSize(sizes[i]);
+ rcvbuffs[i].setSize(sizes[i]);
+ }
+
+ leftSrc=localbegin.x();
+ rightSrc=localgridsize.x()-2*overlap.x();
+ upSrc=localbegin.y();
+ downSrc=localgridsize.y()-2*overlap.y();
+
+ xcenter=localbegin.x();
+ ycenter=localbegin.y();
+
+ leftDst=0;
+ rightDst=localgridsize.x()-overlap.x();
+ upDst=0;
+ downDst=localgridsize.y()-overlap.y();
+ }
+
+ template<typename CommunicatorType>
+ void Synchronize( MeshFunctionType &meshfunction)
+ {
+
+ TNL_ASSERT_TRUE(isSet,"Synchronizer is not set, but used to Synchronize");
+
+ if(!distributedgrid->IsDistributed())
+ return;
+
+ int *neighbor=distributedgrid->getNeighbors();
+
+ CopyBuffers(meshfunction, sendbuffs, true,
+ leftSrc, rightSrc, upSrc, downSrc,
+ xcenter, ycenter,
+ overlap,localsize,
+ neighbor);
+
+ //async send and rcv
+ typename CommunicatorType::Request req[16];
+
+ //send everithing, recieve everything
+ for(int i=0;i<8;i++)
+ if(neighbor[i]!=-1)
+ {
+ req[i]=CommunicatorType::ISend(sendbuffs[i].getData(), sizes[i], neighbor[i]);
+ req[8+i]=CommunicatorType::IRecv(rcvbuffs[i].getData(), sizes[i], neighbor[i]);
+ }
+ else
+ {
+ req[i]=CommunicatorType::NullRequest;
+ req[8+i]=CommunicatorType::NullRequest;
+ }
+
+ //wait until send is done
+ CommunicatorType::WaitAll(req,16);
+
+ //copy data form rcv buffers
+ CopyBuffers(meshfunction, rcvbuffs, false,
+ leftDst, rightDst, upDst, downDst,
+ xcenter, ycenter,
+ overlap,localsize,
+ neighbor);
+ }
+
+ private:
+ template <typename Real>
+ void CopyBuffers(MeshFunctionType meshfunction, Containers::Array<Real, Device, Index> * buffers, bool toBuffer,
+ int left, int right, int up, int down,
+ int xcenter, int ycenter,
+ CoordinatesType shortDim, CoordinatesType longDim,
+ int *neighbor)
+ {
+ if(neighbor[Left]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,2,Real,Device>::BufferEntities(meshfunction,buffers[Left].getData(),left,ycenter,shortDim.x(),longDim.y(),toBuffer);
+ if(neighbor[Right]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,2,Real,Device>::BufferEntities(meshfunction,buffers[Right].getData(),right,ycenter,shortDim.x(),longDim.y(),toBuffer);
+ if(neighbor[Up]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,2,Real,Device>::BufferEntities(meshfunction,buffers[Up].getData(),xcenter,up,longDim.x(),shortDim.y(),toBuffer);
+ if(neighbor[Down]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,2,Real,Device>::BufferEntities(meshfunction,buffers[Down].getData(),xcenter,down,longDim.x(),shortDim.y(),toBuffer);
+ if(neighbor[UpLeft]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,2,Real,Device>::BufferEntities(meshfunction,buffers[UpLeft].getData(),left,up,shortDim.x(),shortDim.y(),toBuffer);
+ if(neighbor[UpRight]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,2,Real,Device>::BufferEntities(meshfunction,buffers[UpRight].getData(),right,up,shortDim.x(),shortDim.y(),toBuffer);
+ if(neighbor[DownLeft]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,2,Real,Device>::BufferEntities(meshfunction,buffers[DownLeft].getData(),left,down,shortDim.x(),shortDim.y(),toBuffer);
+ if(neighbor[DownRight]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,2,Real,Device>::BufferEntities(meshfunction,buffers[DownRight].getData(),right,down,shortDim.x(),shortDim.y(),toBuffer);
+ }
+};
+
+
+//=========================3D=================================================
+template <typename RealType,
+ int EntityDimension,
+ typename Index,
+ typename Device,
+ typename GridReal>
+class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 3, GridReal, Device, Index >,EntityDimension, RealType>>
+{
+
+ public:
+ typedef typename Grid< 3, GridReal, Device, Index >::Cell Cell;
+ typedef typename Functions::MeshFunction< Grid< 3, GridReal, Device, Index >,EntityDimension, RealType> MeshFunctionType;
+ typedef typename Grid< 3, GridReal, Device, Index >::DistributedMeshType DistributedGridType;
+ typedef typename MeshFunctionType::RealType Real;
+ typedef typename DistributedGridType::CoordinatesType CoordinatesType;
+
+ private:
+ Containers::Array<RealType, Device, Index> sendbuffs[26];
+ Containers::Array<RealType, Device, Index> rcvbuffs[26];
+ int sizes[26];
+ DistributedGridType *distributedgrid;
+
+ int westSrc;
+ int eastSrc;
+ int nordSrc;
+ int southSrc;
+ int bottomSrc;
+ int topSrc;
+ int xcenter;
+ int ycenter;
+ int zcenter;
+ int westDst;
+ int eastDst;
+ int nordDst;
+ int southDst;
+ int bottomDst;
+ int topDst;
+
+ CoordinatesType overlap;
+ CoordinatesType localsize;
+
+ bool isSet;
+
+ public:
+
+ DistributedMeshSynchronizer()
+ {
+ isSet=false;
+ };
+
+ DistributedMeshSynchronizer(DistributedGridType *distrgrid)
+ {
+ isSet=false;
+ SetDistributedGrid(distrgrid);
+ };
+
+ void SetDistributedGrid(DistributedGridType *distrgrid)
+ {
+ isSet=true;
+
+ this->distributedgrid=distrgrid;
+ overlap = this->distributedgrid->getOverlap();
+ localsize = this->distributedgrid->getLocalSize();
+
+ CoordinatesType localbegin=this->distributedgrid->getLocalBegin();
+ CoordinatesType localgridsize = this->distributedgrid->getLocalGridSize();
+
+ sizes[West]=sizes[East]=localsize.y()*localsize.z()*overlap.x();
+ sizes[Nord]=sizes[South]=localsize.x()*localsize.z()*overlap.y();
+ sizes[Bottom]=sizes[Top]=localsize.x()*localsize.y()*overlap.z();
+
+ sizes[NordWest]=sizes[NordEast]=sizes[SouthWest]=sizes[SouthEast]=localsize.z()*overlap.x()*overlap.y();
+ sizes[BottomWest]=sizes[BottomEast]=sizes[TopWest]=sizes[TopEast]=localsize.y()*overlap.x()*overlap.z();
+ sizes[BottomNord]=sizes[BottomSouth]=sizes[TopNord]=sizes[TopSouth]=localsize.x()*overlap.y()*overlap.z();
+
+ sizes[BottomNordWest]=sizes[BottomNordEast]=sizes[BottomSouthWest]=sizes[BottomSouthEast]=
+ sizes[TopNordWest]=sizes[TopNordEast]=sizes[TopSouthWest]=sizes[TopSouthEast]=
+ overlap.x()*overlap.y()*overlap.z();
+
+ for(int i=0;i<26;i++)
+ {
+ sendbuffs[i].setSize(sizes[i]);
+ rcvbuffs[i].setSize(sizes[i]);
+ }
+
+ westSrc=localbegin.x();
+ eastSrc=localgridsize.x()-2*overlap.x();
+ nordSrc=localbegin.y();
+ southSrc=localgridsize.y()-2*overlap.y();
+ bottomSrc=localbegin.z();
+ topSrc=localgridsize.z()-2*overlap.z();
+
+ xcenter=localbegin.x();
+ ycenter=localbegin.y();
+ zcenter=localbegin.z();
+
+ westDst=0;
+ eastDst=localgridsize.x()-overlap.x();
+ nordDst=0;
+ southDst=localgridsize.y()-overlap.y();
+ bottomDst=0;
+ topDst=localgridsize.z()-overlap.z();
+
+ }
+
+ template<typename CommunicatorType>
+ void Synchronize(MeshFunctionType &meshfunction)
+ {
+
+ TNL_ASSERT_TRUE(isSet,"Synchronizer is not set, but used to Synchronize");
+
+ if(!distributedgrid->IsDistributed())
+ return;
+
+ int *neighbor=distributedgrid->getNeighbors();
+
+ //fill send buffers
+ CopyBuffers(meshfunction, sendbuffs, true,
+ westSrc, eastSrc, nordSrc, southSrc, bottomSrc, topSrc,
+ xcenter, ycenter, zcenter,
+ overlap, localsize,
+ neighbor);
+
+ //async send and rcv
+ typename CommunicatorType::Request req[52];
+
+ //send everithing, recieve everything
+ for(int i=0;i<26;i++)
+ if(neighbor[i]!=-1)
+ {
+ req[i]=CommunicatorType::ISend(sendbuffs[i].getData(), sizes[i], neighbor[i]);
+ req[26+i]=CommunicatorType::IRecv(rcvbuffs[i].getData(), sizes[i], neighbor[i]);
+ }
+ else
+ {
+ req[i]=CommunicatorType::NullRequest;
+ req[26+i]=CommunicatorType::NullRequest;
+ }
+
+ //wait until send is done
+ CommunicatorType::WaitAll(req,52);
+
+ //copy data form rcv buffers
+ CopyBuffers(meshfunction, rcvbuffs, false,
+ westDst, eastDst, nordDst, southDst, bottomDst, topDst,
+ xcenter, ycenter, zcenter,
+ overlap, localsize,
+ neighbor);
+
+ }
+
+ private:
+ template <typename Real>
+ void CopyBuffers(MeshFunctionType meshfunction, Containers::Array<Real, Device, Index> * buffers, bool toBuffer,
+ int west, int east, int nord, int south, int bottom, int top,
+ int xcenter, int ycenter, int zcenter,
+ CoordinatesType shortDim, CoordinatesType longDim,
+ int *neighbor)
+ {
+ //X-Y-Z
+ if(neighbor[West]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[West].getData(),west,ycenter,zcenter,shortDim.x(),longDim.y(),longDim.z(),toBuffer);
+ if(neighbor[East]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[East].getData(),east,ycenter,zcenter,shortDim.x(),longDim.y(),longDim.z(),toBuffer);
+ if(neighbor[Nord]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[Nord].getData(),xcenter,nord,zcenter,longDim.x(),shortDim.y(),longDim.z(),toBuffer);
+ if(neighbor[South]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[South].getData(),xcenter,south,zcenter,longDim.x(),shortDim.y(),longDim.z(),toBuffer);
+ if(neighbor[Bottom]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[Bottom].getData(),xcenter,ycenter,bottom,longDim.x(),longDim.y(),shortDim.z(),toBuffer);
+ if(neighbor[Top]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[Top].getData(),xcenter,ycenter,top,longDim.x(),longDim.y(),shortDim.z(),toBuffer);
+ //XY
+ if(neighbor[NordWest]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[NordWest].getData(),west,nord,zcenter,shortDim.x(),shortDim.y(),longDim.z(),toBuffer);
+ if(neighbor[NordEast]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[NordEast].getData(),east,nord,zcenter,shortDim.x(),shortDim.y(),longDim.z(),toBuffer);
+ if(neighbor[SouthWest]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[SouthWest].getData(),west,south,zcenter,shortDim.x(),shortDim.y(),longDim.z(),toBuffer);
+ if(neighbor[SouthEast]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[SouthEast].getData(),east,south,zcenter,shortDim.x(),shortDim.y(),longDim.z(),toBuffer);
+ //XZ
+ if(neighbor[BottomWest]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[BottomWest].getData(),west,ycenter,bottom,shortDim.x(),longDim.y(),shortDim.z(),toBuffer);
+ if(neighbor[BottomEast]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[BottomEast].getData(),east,ycenter,bottom,shortDim.x(),longDim.y(),shortDim.z(),toBuffer);
+ if(neighbor[TopWest]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[TopWest].getData(),west,ycenter,top,shortDim.x(),longDim.y(),shortDim.z(),toBuffer);
+ if(neighbor[TopEast]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[TopEast].getData(),east,ycenter,top,shortDim.x(),longDim.y(),shortDim.z(),toBuffer);
+ //YZ
+ if(neighbor[BottomNord]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[BottomNord].getData(),xcenter,nord,bottom,longDim.x(),shortDim.y(),shortDim.z(),toBuffer);
+ if(neighbor[BottomSouth]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[BottomSouth].getData(),xcenter,south,bottom,longDim.x(),shortDim.y(),shortDim.z(),toBuffer);
+ if(neighbor[TopNord]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[TopNord].getData(),xcenter,nord,top,longDim.x(),shortDim.y(),shortDim.z(),toBuffer);
+ if(neighbor[TopSouth]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[TopSouth].getData(),xcenter,south,top,longDim.x(),shortDim.y(),shortDim.z(),toBuffer);
+ //XYZ
+ if(neighbor[BottomNordWest]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[BottomNordWest].getData(),west,nord,bottom,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
+ if(neighbor[BottomNordEast]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[BottomNordEast].getData(),east,nord,bottom,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
+ if(neighbor[BottomSouthWest]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[BottomSouthWest].getData(),west,south,bottom,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
+ if(neighbor[BottomSouthEast]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[BottomSouthEast].getData(),east,south,bottom,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
+ if(neighbor[TopNordWest]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[TopNordWest].getData(),west,nord,top,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
+ if(neighbor[TopNordEast]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[TopNordEast].getData(),east,nord,top,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
+ if(neighbor[TopSouthWest]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[TopSouthWest].getData(),west,south,top,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
+ if(neighbor[TopSouthEast]!=-1)
+ BufferEntitiesHelper<MeshFunctionType,3,Real,Device>::BufferEntities(meshfunction,buffers[TopSouthEast].getData(),east,south,top,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
+
+ }
+};
+
+
+} // namespace DistributedMeshes
+} // namespace Meshes
+} // namespace TNL
diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_1D.h b/src/TNL/Meshes/DistributedMeshes/DistributedGrid_1D.h
new file mode 100644
index 0000000000000000000000000000000000000000..874955452346be5429fa5948d8fb209d5357051c
--- /dev/null
+++ b/src/TNL/Meshes/DistributedMeshes/DistributedGrid_1D.h
@@ -0,0 +1,232 @@
+/***************************************************************************
+ DistributedGrid_1D.h - description
+ -------------------
+ begin : January 09, 2018
+ copyright : (C) 2018 by Tomas Oberhuber
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+/* See Copyright Notice in tnl/Copyright */
+
+#pragma once
+
+#include <TNL/Meshes/Grid.h>
+
+namespace TNL {
+namespace Meshes {
+namespace DistributedMeshes {
+
+template<typename RealType, typename Device, typename Index >
+class DistributedMesh<Grid< 1, RealType, Device, Index >>
+{
+
+ public:
+
+ typedef Index IndexType;
+ typedef Grid< 1, RealType, Device, IndexType > GridType;
+ typedef typename GridType::PointType PointType;
+ typedef Containers::StaticVector< 1, IndexType > CoordinatesType;
+
+ static constexpr int getMeshDimension() { return 1; };
+
+ private :
+
+ PointType localorigin;
+ CoordinatesType localbegin;
+ CoordinatesType localsize;
+ CoordinatesType localgridsize;
+ CoordinatesType overlap;
+ CoordinatesType globalsize;
+ CoordinatesType globalbegin;
+ PointType spaceSteps;
+
+
+ IndexType Dimensions;
+ bool isDistributed;
+
+ int rank;
+ int nproc;
+
+ int numberoflarger;
+
+ int left;
+ int right;
+
+ bool isSet;
+
+
+ public:
+
+ DistributedMesh()
+ {
+ isSet=false;
+ };
+
+ //compute everithing
+ template<typename CommunicatorType>
+ void setGlobalGrid(GridType globalGrid, CoordinatesType overlap, int *distribution=NULL)
+ {
+ isSet=true;
+
+ this->overlap=overlap;
+
+ left=-1;
+ right=-1;
+
+ Dimensions= GridType::getMeshDimension();
+ spaceSteps=globalGrid.getSpaceSteps();
+
+ isDistributed=false;
+ if(CommunicatorType::IsInitialized())
+ {
+ rank=CommunicatorType::GetRank();
+ this->nproc=CommunicatorType::GetSize();
+ //use only if have more than one process
+ if(this->nproc>1)
+ {
+ isDistributed=true;
+ }
+ }
+
+ if(!isDistributed)
+ {
+ //Without distribution
+
+ std::cout <<"BEZ Distribuce"<<std::endl;
+ rank=0;
+ localorigin=globalGrid.getOrigin();
+ localsize=globalGrid.getDimensions();
+ localgridsize=globalGrid.getDimensions();
+ globalsize=globalGrid.getDimensions();
+ globalbegin=CoordinatesType(0);
+
+ localbegin=CoordinatesType(0);
+ return;
+ }
+ else
+ {
+ //nearnodes
+ if(rank!=0)
+ left=rank-1;
+ if(rank!=nproc-1)
+ right=rank+1;
+
+ globalsize=globalGrid.getDimensions();
+
+ //compute local mesh size
+ numberoflarger=globalGrid.getDimensions().x()%nproc;
+
+ localsize.x()=(globalGrid.getDimensions().x()/nproc);
+ if(numberoflarger>rank)
+ localsize.x()+=1;
+
+ if(numberoflarger>rank)
+ {
+ globalbegin.x()=rank*localsize.x();
+ localorigin.x()=globalGrid.getOrigin().x()
+ +(globalbegin.x()-overlap.x())*globalGrid.getSpaceSteps().x();
+ }
+ else
+ {
+ globalbegin.x()=numberoflarger*(localsize.x()+1)+(rank-numberoflarger)*localsize.x();
+ localorigin.x()=(globalGrid.getOrigin().x()-overlap.x())
+ +globalbegin.x()*globalGrid.getSpaceSteps().x();
+ }
+
+ localbegin=overlap;
+
+ //vlevo neni prekryv
+ if(left==-1)
+ {
+ localorigin.x()+=overlap.x()*globalGrid.getSpaceSteps().x();
+ localbegin.x()=0;
+ }
+
+ localgridsize=localsize;
+ //add overlaps
+ if(left==-1||right==-1)
+ localgridsize.x()+=overlap.x();
+ else
+ localgridsize.x()+=2*overlap.x();
+
+ }
+ }
+
+ void SetupGrid( GridType& grid)
+ {
+ TNL_ASSERT_TRUE(isSet,"DistributedGrid is not set, but used by SetupGrid");
+ grid.setOrigin(localorigin);
+ grid.setDimensions(localgridsize);
+ //compute local proporions by sideefect
+ grid.setSpaceSteps(spaceSteps);
+ grid.SetDistMesh(this);
+ };
+
+ String printProcessCoords()
+ {
+ return convertToString(rank);
+ };
+
+ String printProcessDistr()
+ {
+ return convertToString(nproc);
+ };
+
+ bool IsDistributed(void)
+ {
+ return this->isDistributed;
+ };
+
+ int getLeft()
+ {
+ TNL_ASSERT_TRUE(isSet,"DistributedGrid is not set, but used by getLeft");
+ return this->left;
+ };
+
+ int getRight()
+ {
+ TNL_ASSERT_TRUE(isSet,"DistributedGrid is not set, but used by getRight");
+ return this->right;
+ };
+
+ CoordinatesType getOverlap()
+ {
+ return this->overlap;
+ };
+
+ //number of elemnts of local sub domain WITHOUT overlap
+ CoordinatesType getLocalSize()
+ {
+ return this->localsize;
+ }
+
+ //number of elements of global grid
+ CoordinatesType getGlobalSize()
+ {
+ return this->globalsize;
+ }
+
+ //coordinates of begin of local subdomain without overlaps in global grid
+ CoordinatesType getGlobalBegin()
+ {
+ return this->globalbegin;
+ }
+
+ //number of elemnts of local sub domain WITH overlap
+ CoordinatesType getLocalGridSize()
+ {
+ return this->localgridsize;
+ }
+
+ //coordinates of begin of local subdomain without overlaps in local grid
+ CoordinatesType getLocalBegin()
+ {
+ return this->localbegin;
+ }
+};
+
+
+
+} // namespace DistributedMeshes
+} // namespace Meshes
+} // namespace TNL
diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_2D.h b/src/TNL/Meshes/DistributedMeshes/DistributedGrid_2D.h
new file mode 100644
index 0000000000000000000000000000000000000000..0c0dcf845c899fa744f40678883f6242184548a8
--- /dev/null
+++ b/src/TNL/Meshes/DistributedMeshes/DistributedGrid_2D.h
@@ -0,0 +1,282 @@
+/***************************************************************************
+ DistributedGrid_2D.h - description
+ -------------------
+ begin : January 09, 2018
+ copyright : (C) 2018 by Tomas Oberhuber
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+/* See Copyright Notice in tnl/Copyright */
+
+#pragma once
+
+#include <TNL/Meshes/Grid.h>
+
+namespace TNL {
+namespace Meshes {
+namespace DistributedMeshes {
+
+enum Directions2D { Left = 0 , Right = 1 , Up = 2, Down=3, UpLeft =4, UpRight=5, DownLeft=6, DownRight=7 };
+
+template<typename RealType, typename Device, typename Index >
+class DistributedMesh<Grid< 2, RealType, Device, Index >>
+{
+ public:
+
+ typedef Index IndexType;
+ typedef Grid< 2, RealType, Device, IndexType > GridType;
+ typedef typename GridType::PointType PointType;
+ typedef Containers::StaticVector< 2, IndexType > CoordinatesType;
+
+ static constexpr int getMeshDimension() { return 2; };
+
+ private :
+
+ PointType spaceSteps;
+ PointType localorigin;
+ CoordinatesType localsize;//velikost gridu zpracovavane danym uzlem bez prekryvu
+ CoordinatesType localbegin;//souradnice zacatku zpracovavane vypoctove oblasi
+ CoordinatesType localgridsize;//velikost lokálnÃho gridu vÄetnÄ› pÅ™ekryvů
+ CoordinatesType overlap;
+ CoordinatesType globalsize;//velikost celé sÃtÄ›
+ CoordinatesType globalbegin;
+
+
+ IndexType Dimensions;
+ bool isDistributed;
+
+ int rank;
+ int nproc;
+
+ int procsdistr[2];
+ int myproccoord[2];
+ int numberoflarger[2];
+
+ int neighbors[8];
+
+ bool isSet;
+
+ public:
+ DistributedMesh()
+ {
+ isSet=false;
+ };
+
+ //compute everithing
+ template<typename CommunicatorType>
+ void setGlobalGrid(GridType &globalGrid, CoordinatesType overlap, int *distribution=NULL)
+ {
+ isSet=true;
+
+ this->overlap=overlap;
+
+ for (int i=0;i<8;i++)
+ neighbors[i]=-1;
+
+ Dimensions= GridType::getMeshDimension();
+ spaceSteps=globalGrid.getSpaceSteps();
+ //Detect MPI and number of process
+ isDistributed=false;
+
+
+
+ if(CommunicatorType::IsInitialized())
+ {
+ rank=CommunicatorType::GetRank();
+ this->nproc=CommunicatorType::GetSize();
+ //use MPI only if have more than one process
+ if(this->nproc>1)
+ {
+ isDistributed=true;
+ }
+ }
+
+ if(!isDistributed)
+ {
+ //Without MPI
+ myproccoord[0]=0;
+ myproccoord[1]=0;
+ procsdistr[0]=1;
+ procsdistr[1]=1;
+ localorigin=globalGrid.getOrigin();
+ localgridsize=globalGrid.getDimensions();
+ localsize=globalGrid.getDimensions();
+ globalsize=globalGrid.getDimensions();
+ globalbegin=CoordinatesType(0);
+ localbegin.x()=0;
+ localbegin.y()=0;
+
+ return;
+ }
+ else
+ {
+ //With MPI
+ //compute node distribution
+ if(distribution!=NULL)
+ {
+ procsdistr[0]=distribution[0];
+ procsdistr[1]=distribution[1];
+ }
+ else
+ {
+ procsdistr[0]=0;
+ procsdistr[1]=0;
+ }
+ CommunicatorType::DimsCreate(nproc, 2, procsdistr);
+
+ myproccoord[0]=rank%procsdistr[0];
+ myproccoord[1]=rank/procsdistr[0];
+
+ //compute local mesh size
+ globalsize=globalGrid.getDimensions();
+ numberoflarger[0]=globalGrid.getDimensions().x()%procsdistr[0];
+ numberoflarger[1]=globalGrid.getDimensions().y()%procsdistr[1];
+
+ localsize.x()=(globalGrid.getDimensions().x()/procsdistr[0]);
+ localsize.y()=(globalGrid.getDimensions().y()/procsdistr[1]);
+
+ if(numberoflarger[0]>myproccoord[0])
+ localsize.x()+=1;
+ if(numberoflarger[1]>myproccoord[1])
+ localsize.y()+=1;
+
+ if(numberoflarger[0]>myproccoord[0])
+ globalbegin.x()=myproccoord[0]*localsize.x();
+ else
+ globalbegin.x()=numberoflarger[0]*(localsize.x()+1)+(myproccoord[0]-numberoflarger[0])*localsize.x();
+
+ if(numberoflarger[1]>myproccoord[1])
+ globalbegin.y()=myproccoord[1]*localsize.y();
+
+ else
+ globalbegin.y()=numberoflarger[1]*(localsize.y()+1)+(myproccoord[1]-numberoflarger[1])*localsize.y();
+
+ localorigin=globalGrid.getOrigin()+TNL::Containers::tnlDotProduct(globalGrid.getSpaceSteps(),globalbegin-overlap);
+
+ //nearnodes
+ if(myproccoord[0]>0)
+ neighbors[Left]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]);
+ if(myproccoord[0]<procsdistr[0]-1)
+ neighbors[Right]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]);
+ if(myproccoord[1]>0)
+ neighbors[Up]=getRankOfProcCoord(myproccoord[0],myproccoord[1]-1);
+ if(myproccoord[1]<procsdistr[1]-1)
+ neighbors[Down]=getRankOfProcCoord(myproccoord[0],myproccoord[1]+1);
+ if(myproccoord[0]>0 && myproccoord[1]>0)
+ neighbors[UpLeft]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]-1);
+ if(myproccoord[0]>0 && myproccoord[1]<procsdistr[1]-1)
+ neighbors[DownLeft]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]+1);
+ if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]>0)
+ neighbors[UpRight]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]-1);
+ if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]<procsdistr[1]-1)
+ neighbors[DownRight]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]+1);
+
+ localbegin=overlap;
+
+ if(neighbors[Left]==-1)
+ {
+ localorigin.x()+=overlap.x()*globalGrid.getSpaceSteps().x();
+ localbegin.x()=0;
+ }
+
+ if(neighbors[Up]==-1)
+ {
+ localorigin.y()+=overlap.y()*globalGrid.getSpaceSteps().y();
+ localbegin.y()=0;
+ }
+
+ localgridsize=localsize;
+ //Add Overlaps
+ if(neighbors[Left]!=-1)
+ localgridsize.x()+=overlap.x();
+ if(neighbors[Right]!=-1)
+ localgridsize.x()+=overlap.x();
+
+ if(neighbors[Up]!=-1)
+ localgridsize.y()+=overlap.y();
+ if(neighbors[Down]!=-1)
+ localgridsize.y()+=overlap.y();
+ }
+
+
+
+ }
+
+ void SetupGrid( GridType& grid)
+ {
+ TNL_ASSERT_TRUE(isSet,"DistributedGrid is not set, but used by SetupGrid");
+ grid.setOrigin(localorigin);
+ grid.setDimensions(localgridsize);
+ //compute local proporions by sideefect
+ grid.setSpaceSteps(spaceSteps);
+ grid.SetDistMesh(this);
+ };
+
+ String printProcessCoords()
+ {
+ return convertToString(myproccoord[0])+String("-")+convertToString(myproccoord[1]);
+ };
+
+ String printProcessDistr()
+ {
+ return convertToString(procsdistr[0])+String("-")+convertToString(procsdistr[1]);
+ };
+
+ bool IsDistributed(void)
+ {
+ return this->isDistributed;
+ };
+
+ CoordinatesType getOverlap()
+ {
+ return this->overlap;
+ };
+
+ int * getNeighbors()
+ {
+ TNL_ASSERT_TRUE(isSet,"DistributedGrid is not set, but used by getNeighbors");
+ return this->neighbors;
+ }
+
+ CoordinatesType getLocalSize()
+ {
+ return this->localsize;
+ }
+
+ //number of elements of global grid
+ CoordinatesType getGlobalSize()
+ {
+ return this->globalsize;
+ }
+
+ //coordinates of begin of local subdomain without overlaps in global grid
+ CoordinatesType getGlobalBegin()
+ {
+ return this->globalbegin;
+ }
+
+ CoordinatesType getLocalGridSize()
+ {
+ return this->localgridsize;
+ }
+
+
+ CoordinatesType getLocalBegin()
+ {
+ return this->localbegin;
+ }
+
+ private:
+
+ int getRankOfProcCoord(int x, int y)
+ {
+ return y*procsdistr[0]+x;
+ }
+
+
+};
+
+} // namespace DistributedMeshes
+} // namespace Meshes
+} // namespace TNL
+
diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid_3D.h b/src/TNL/Meshes/DistributedMeshes/DistributedGrid_3D.h
new file mode 100644
index 0000000000000000000000000000000000000000..f12950178a7efcbc3017c0a288024096573e1aa9
--- /dev/null
+++ b/src/TNL/Meshes/DistributedMeshes/DistributedGrid_3D.h
@@ -0,0 +1,349 @@
+/***************************************************************************
+ DistributedGrid_3D.h - description
+ -------------------
+ begin : January 15, 2018
+ copyright : (C) 2018 by Tomas Oberhuber
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+/* See Copyright Notice in tnl/Copyright */
+
+#pragma once
+
+#include <TNL/Meshes/Grid.h>
+
+namespace TNL {
+namespace Meshes {
+namespace DistributedMeshes {
+
+enum Directions3D { West = 0 , East = 1 , Nord = 2, South=3, Top =4, Bottom=5,
+ NordWest=6, NordEast=7, SouthWest=8, SouthEast=9,
+ BottomWest=10,BottomEast=11,BottomNord=12,BottomSouth=13,
+ TopWest=14,TopEast=15,TopNord=16,TopSouth=17,
+ BottomNordWest=18,BottomNordEast=19,BottomSouthWest=20,BottomSouthEast=21,
+ TopNordWest=22,TopNordEast=23,TopSouthWest=24,TopSouthEast=25
+ };
+
+
+template<typename RealType, typename Device, typename Index >
+class DistributedMesh<Grid< 3, RealType, Device, Index >>
+{
+
+ public:
+
+ typedef Index IndexType;
+ typedef Grid< 3, RealType, Device, IndexType > GridType;
+ typedef typename GridType::PointType PointType;
+ typedef Containers::StaticVector< 3, IndexType > CoordinatesType;
+
+ static constexpr int getMeshDimension() { return 3; };
+
+ private :
+
+ PointType spaceSteps;
+ PointType localorigin;
+ CoordinatesType localsize;
+ CoordinatesType localgridsize;
+ CoordinatesType localbegin;
+ CoordinatesType overlap;
+ CoordinatesType globalsize;
+ CoordinatesType globalbegin;
+
+ IndexType Dimensions;
+ bool isDistributed;
+
+ int rank;
+ int nproc;
+
+ int procsdistr[3];
+ int myproccoord[3];
+ int numberoflarger[3];
+
+ int neighbors[26];
+
+ bool isSet;
+
+ public:
+ DistributedMesh()
+ {
+ isSet=false;
+ };
+
+ //compute everithing
+ template<typename CommunicatorType>
+ void setGlobalGrid(GridType &globalGrid, CoordinatesType overlap, int *distribution=NULL)
+ {
+
+ isSet=true;
+
+ this->overlap=overlap;
+
+ for (int i=0;i<26;i++)
+ neighbors[i]=-1;
+
+ Dimensions= GridType::getMeshDimension();
+ spaceSteps=globalGrid.getSpaceSteps();
+
+ isDistributed=false;
+
+
+
+ if(CommunicatorType::IsInitialized())
+ {
+ rank=CommunicatorType::GetRank();
+ this->nproc=CommunicatorType::GetSize();
+ //use MPI only if have more than one process
+ if(this->nproc>1)
+ {
+ isDistributed=true;
+ }
+ }
+
+ if(!isDistributed)
+ {
+ //Without MPI
+ myproccoord[0]=0;
+ myproccoord[1]=0;
+ myproccoord[2]=0;
+
+ procsdistr[0]=1;
+ procsdistr[1]=1;
+ procsdistr[2]=1;
+
+ localorigin=globalGrid.getOrigin();
+ localsize=globalGrid.getDimensions();
+ globalsize=globalGrid.getDimensions();
+ localgridsize=localsize;
+ globalbegin=CoordinatesType(0);
+ return;
+ }
+ else
+ {
+ //With MPI
+ //compute node distribution
+ if(distribution!=NULL)
+ {
+ procsdistr[0]=distribution[0];
+ procsdistr[1]=distribution[1];
+ procsdistr[2]=distribution[2];
+ }
+ else
+ {
+ procsdistr[0]=0;
+ procsdistr[1]=0;
+ procsdistr[2]=0;
+ }
+ CommunicatorType::DimsCreate(nproc, 3, procsdistr);
+ myproccoord[2]=rank/(procsdistr[0]*procsdistr[1]);
+ myproccoord[1]=(rank%(procsdistr[0]*procsdistr[1]))/procsdistr[0];
+ myproccoord[0]=(rank%(procsdistr[0]*procsdistr[1]))%procsdistr[0];
+
+ //compute local mesh size
+ globalsize=globalGrid.getDimensions();
+ numberoflarger[0]=globalGrid.getDimensions().x()%procsdistr[0];
+ numberoflarger[1]=globalGrid.getDimensions().y()%procsdistr[1];
+ numberoflarger[2]=globalGrid.getDimensions().z()%procsdistr[2];
+
+ localsize.x()=(globalGrid.getDimensions().x()/procsdistr[0]);
+ localsize.y()=(globalGrid.getDimensions().y()/procsdistr[1]);
+ localsize.z()=(globalGrid.getDimensions().z()/procsdistr[2]);
+
+ if(numberoflarger[0]>myproccoord[0])
+ localsize.x()+=1;
+ if(numberoflarger[1]>myproccoord[1])
+ localsize.y()+=1;
+ if(numberoflarger[2]>myproccoord[2])
+ localsize.z()+=1;
+
+ if(numberoflarger[0]>myproccoord[0])
+ globalbegin.x()=myproccoord[0]*localsize.x();
+ else
+ globalbegin.x()=numberoflarger[0]*(localsize.x()+1)+(myproccoord[0]-numberoflarger[0])*localsize.x();
+
+ if(numberoflarger[1]>myproccoord[1])
+ globalbegin.y()=myproccoord[1]*localsize.y();
+ else
+ globalbegin.y()=numberoflarger[1]*(localsize.y()+1)+(myproccoord[1]-numberoflarger[1])*localsize.y();
+
+ if(numberoflarger[2]>myproccoord[2])
+ globalbegin.z()=myproccoord[2]*localsize.z();
+ else
+ globalbegin.z()=numberoflarger[2]*(localsize.z()+1)+(myproccoord[2]-numberoflarger[2])*localsize.z();
+
+ localorigin=globalGrid.getOrigin()+TNL::Containers::tnlDotProduct(globalGrid.getSpaceSteps(),globalbegin-overlap);
+
+ //nearnodes
+ //X Y Z
+ if(myproccoord[0]>0)
+ neighbors[West]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1],myproccoord[2]);
+ if(myproccoord[0]<procsdistr[0]-1)
+ neighbors[East]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1],myproccoord[2]);
+ if(myproccoord[1]>0)
+ neighbors[Nord]=getRankOfProcCoord(myproccoord[0],myproccoord[1]-1,myproccoord[2]);
+ if(myproccoord[1]<procsdistr[1]-1)
+ neighbors[South]=getRankOfProcCoord(myproccoord[0],myproccoord[1]+1,myproccoord[2]);
+ if(myproccoord[2]>0)
+ neighbors[Bottom]=getRankOfProcCoord(myproccoord[0],myproccoord[1],myproccoord[2]-1);
+ if(myproccoord[2]<procsdistr[2]-1)
+ neighbors[Top]=getRankOfProcCoord(myproccoord[0],myproccoord[1],myproccoord[2]+1);
+
+ //XY
+ if(myproccoord[0]>0 && myproccoord[1]>0)
+ neighbors[NordWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]-1,myproccoord[2]);
+ if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]>0)
+ neighbors[NordEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]-1,myproccoord[2]);
+ if(myproccoord[0]>0 && myproccoord[1]<procsdistr[1]-1)
+ neighbors[SouthWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]+1,myproccoord[2]);
+ if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]<procsdistr[1]-1)
+ neighbors[SouthEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]+1,myproccoord[2]);
+ //XZ
+ if(myproccoord[0]>0 && myproccoord[2]>0)
+ neighbors[BottomWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1],myproccoord[2]-1);
+ if(myproccoord[0]<procsdistr[0]-1 && myproccoord[2]>0)
+ neighbors[BottomEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1],myproccoord[2]-1);
+ if(myproccoord[0]>0 && myproccoord[2]<procsdistr[2]-1)
+ neighbors[TopWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1],myproccoord[2]+1);
+ if(myproccoord[0]<procsdistr[0]-1 && myproccoord[2]<procsdistr[2]-1)
+ neighbors[TopEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1],myproccoord[2]+1);
+ //YZ
+ if(myproccoord[1]>0 && myproccoord[2]>0)
+ neighbors[BottomNord]=getRankOfProcCoord(myproccoord[0],myproccoord[1]-1,myproccoord[2]-1);
+ if(myproccoord[1]<procsdistr[1]-1 && myproccoord[2]>0)
+ neighbors[BottomSouth]=getRankOfProcCoord(myproccoord[0],myproccoord[1]+1,myproccoord[2]-1);
+ if(myproccoord[1]>0 && myproccoord[2]<procsdistr[2]-1)
+ neighbors[TopNord]=getRankOfProcCoord(myproccoord[0],myproccoord[1]-1,myproccoord[2]+1);
+ if(myproccoord[1]<procsdistr[1]-1 && myproccoord[2]<procsdistr[2]-1)
+ neighbors[TopSouth]=getRankOfProcCoord(myproccoord[0],myproccoord[1]+1,myproccoord[2]+1);
+ //XYZ
+ if(myproccoord[0]>0 && myproccoord[1]>0 && myproccoord[2]>0 )
+ neighbors[BottomNordWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]-1,myproccoord[2]-1);
+ if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]>0 && myproccoord[2]>0 )
+ neighbors[BottomNordEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]-1,myproccoord[2]-1);
+ if(myproccoord[0]>0 && myproccoord[1]<procsdistr[1]-1 && myproccoord[2]>0 )
+ neighbors[BottomSouthWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]+1,myproccoord[2]-1);
+ if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]<procsdistr[1]-1 && myproccoord[2]>0 )
+ neighbors[BottomSouthEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]+1,myproccoord[2]-1);
+ if(myproccoord[0]>0 && myproccoord[1]>0 && myproccoord[2]<procsdistr[2]-1 )
+ neighbors[TopNordWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]-1,myproccoord[2]+1);
+ if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]>0 && myproccoord[2]<procsdistr[2]-1 )
+ neighbors[TopNordEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]-1,myproccoord[2]+1);
+ if(myproccoord[0]>0 && myproccoord[1]<procsdistr[1]-1 && myproccoord[2]<procsdistr[2]-1 )
+ neighbors[TopSouthWest]=getRankOfProcCoord(myproccoord[0]-1,myproccoord[1]+1,myproccoord[2]+1);
+ if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]<procsdistr[1]-1 && myproccoord[2]<procsdistr[2]-1 )
+ neighbors[TopSouthEast]=getRankOfProcCoord(myproccoord[0]+1,myproccoord[1]+1,myproccoord[2]+1);
+
+
+ localbegin=overlap;
+
+ if(neighbors[West]==-1)
+ {
+ localorigin.x()+=overlap.x()*globalGrid.getSpaceSteps().x();
+ localbegin.x()=0;
+ }
+ if(neighbors[Nord]==-1)
+ {
+ localorigin.y()+=overlap.y()*globalGrid.getSpaceSteps().y();
+ localbegin.y()=0;
+ }
+ if(neighbors[Bottom]==-1)
+ {
+ localorigin.z()+=overlap.z()*globalGrid.getSpaceSteps().z();
+ localbegin.z()=0;
+ }
+
+ localgridsize=localsize;
+
+ if(neighbors[West]!=-1)
+ localgridsize.x()+=overlap.x();
+ if(neighbors[East]!=-1)
+ localgridsize.x()+=overlap.x();
+
+ if(neighbors[Nord]!=-1)
+ localgridsize.y()+=overlap.y();
+ if(neighbors[South]!=-1)
+ localgridsize.y()+=overlap.y();
+
+ if(neighbors[Bottom]!=-1)
+ localgridsize.z()+=overlap.z();
+ if(neighbors[Top]!=-1)
+ localgridsize.z()+=overlap.z();
+
+ }
+ }
+
+ void SetupGrid( GridType& grid)
+ {
+ TNL_ASSERT_TRUE(isSet,"DistributedGrid is not set, but used by SetupGrid");
+ grid.setOrigin(localorigin);
+ grid.setDimensions(localgridsize);
+ //compute local proporions by sideefect
+ grid.setSpaceSteps(spaceSteps);
+ grid.SetDistMesh(this);
+ };
+
+ String printProcessCoords()
+ {
+ return convertToString(myproccoord[0])+String("-")+convertToString(myproccoord[1])+String("-")+convertToString(myproccoord[2]);
+ };
+
+ String printProcessDistr()
+ {
+ return convertToString(procsdistr[0])+String("-")+convertToString(procsdistr[1])+String("-")+convertToString(procsdistr[2]);
+ };
+
+ bool IsDistributed(void)
+ {
+ return this->isDistributed;
+ };
+
+ CoordinatesType getOverlap()
+ {
+ return this->overlap;
+ };
+
+ int * getNeighbors()
+ {
+ TNL_ASSERT_TRUE(isSet,"DistributedGrid is not set, but used by getNeighbors");
+ return this->neighbors;
+ }
+
+ CoordinatesType getLocalSize()
+ {
+ return this->localsize;
+ }
+
+ CoordinatesType getLocalGridSize()
+ {
+ return this->localgridsize;
+ }
+
+ CoordinatesType getLocalBegin()
+ {
+ return this->localbegin;
+ }
+
+ //number of elements of global grid
+ CoordinatesType getGlobalSize()
+ {
+ return this->globalsize;
+ }
+
+ //coordinates of begin of local subdomain without overlaps in global grid
+ CoordinatesType getGlobalBegin()
+ {
+ return this->globalbegin;
+ }
+
+ private:
+
+ int getRankOfProcCoord(int x, int y, int z)
+ {
+ return z*procsdistr[0]*procsdistr[1]+y*procsdistr[0]+x;
+ }
+
+
+};
+
+} // namespace DistributedMeshes
+} // namespace Meshes
+} // namespace TNL
diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedMesh.h b/src/TNL/Meshes/DistributedMeshes/DistributedMesh.h
new file mode 100644
index 0000000000000000000000000000000000000000..e03d0975652c41fb4e40a038bc25acf36c67a06c
--- /dev/null
+++ b/src/TNL/Meshes/DistributedMeshes/DistributedMesh.h
@@ -0,0 +1,33 @@
+/***************************************************************************
+ DistributedMesh.h - description
+ -------------------
+ begin : March 17, 2017
+ copyright : (C) 2017 by Tomas Oberhuber
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+/* See Copyright Notice in tnl/Copyright */
+
+#pragma once
+
+namespace TNL {
+namespace Meshes {
+namespace DistributedMeshes {
+
+template< typename MeshType >
+class DistributedMesh
+{
+public:
+ bool IsDistributed(void)
+ {
+ return false;
+ };
+};
+
+} // namespace DistributedMeshes
+} // namespace Meshes
+} // namespace TNL
+
+#include <TNL/Meshes/DistributedMeshes/DistributedGrid_1D.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedGrid_2D.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedGrid_3D.h>
diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedMeshSynchronizer.h b/src/TNL/Meshes/DistributedMeshes/DistributedMeshSynchronizer.h
new file mode 100644
index 0000000000000000000000000000000000000000..ef6929e1eedbdb6415667cb718b442dc0cacded2
--- /dev/null
+++ b/src/TNL/Meshes/DistributedMeshes/DistributedMeshSynchronizer.h
@@ -0,0 +1,34 @@
+/***************************************************************************
+ DistributedMeshSynchronizer.h - description
+ -------------------
+ begin : January 8, 2017
+ copyright : (C) 2017 by Tomas Oberhuber
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+/* See Copyright Notice in tnl/Copyright */
+
+#pragma once
+
+namespace TNL {
+namespace Meshes {
+namespace DistributedMeshes {
+
+template <typename MeshFunctionType>
+class DistributedMeshSynchronizer
+{
+ public:
+ typedef typename MeshFunctionType::DistributedMeshType DistributedMeshType;
+ void SetDistributedGrid(DistributedMeshType *distrgrid)
+ {
+ TNL_ASSERT_TRUE(false,"Distribution of this type of mesh is NOT implemented");
+ };
+
+};
+
+
+} // namespace DistributedMeshes
+} // namespace Meshes
+} // namespace TNL
+
+#include <TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h>
diff --git a/src/TNL/Meshes/Grid.h b/src/TNL/Meshes/Grid.h
index 07053991262385c8fd19e72377693c6748fc84de..c196e8b88708bd228707c3b6d059a35a4039a3f8 100644
--- a/src/TNL/Meshes/Grid.h
+++ b/src/TNL/Meshes/Grid.h
@@ -16,7 +16,7 @@
#include <TNL/Containers/Vector.h>
namespace TNL {
-namespace Meshes {
+namespace Meshes {
template< int Dimension,
typename Real = double,
diff --git a/src/TNL/Meshes/GridDetails/Grid1D.h b/src/TNL/Meshes/GridDetails/Grid1D.h
index c8dbd74897d0030efcacd6b6ed2087f5b328bfe1..e2c456c83bd966beb1aef2f34ba81edb34a25828 100644
--- a/src/TNL/Meshes/GridDetails/Grid1D.h
+++ b/src/TNL/Meshes/GridDetails/Grid1D.h
@@ -17,6 +17,7 @@
#include <TNL/Meshes/GridDetails/NeighborGridEntityGetter.h>
#include <TNL/Meshes/GridEntity.h>
#include <TNL/Meshes/GridEntityConfig.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
namespace TNL {
namespace Meshes {
@@ -37,6 +38,8 @@ class Grid< 1, Real, Device, Index > : public Object
typedef Grid< 1, Real, Devices::Cuda, Index > CudaType;
typedef Grid< 1, Real, Device, Index > ThisType;
+ typedef DistributedMeshes::DistributedMesh <ThisType> DistributedMeshType;
+
// TODO: deprecated and to be removed (GlobalIndexType shall be used instead)
typedef Index IndexType;
@@ -66,10 +69,13 @@ class Grid< 1, Real, Device, Index > : public Object
__cuda_callable__
const CoordinatesType& getDimensions() const;
+
+ void setOrigin( const PointType& origin);
void setDomain( const PointType& origin,
const PointType& proportions );
+
__cuda_callable__
inline const PointType& getOrigin() const;
@@ -94,8 +100,12 @@ class Grid< 1, Real, Device, Index > : public Object
inline Index getEntityIndex( const Entity& entity ) const;
__cuda_callable__
+
+ inline void setSpaceSteps(const PointType& steps);
+
inline const PointType& getSpaceSteps() const;
+
template< int xPow >
__cuda_callable__
const RealType& getSpaceStepsProducts() const;
@@ -115,6 +125,10 @@ class Grid< 1, Real, Device, Index > : public Object
typename GridFunction::RealType getDifferenceLpNorm( const GridFunction& f1,
const GridFunction& f2,
const typename GridFunction::RealType& p ) const;
+
+ void SetDistMesh(DistributedMeshType * distMesh);
+
+ DistributedMeshType * GetDistMesh(void) const;
/****
* Method for saving the object to a file as a binary data
@@ -134,6 +148,10 @@ class Grid< 1, Real, Device, Index > : public Object
protected:
+ void computeProportions();
+
+ void computeSpaceStepPowers();
+
void computeSpaceSteps();
CoordinatesType dimensions;
@@ -145,6 +163,8 @@ class Grid< 1, Real, Device, Index > : public Object
PointType spaceSteps;
RealType spaceStepsProducts[ 5 ];
+
+ DistributedMeshType *distGrid;
};
} // namespace Meshes
diff --git a/src/TNL/Meshes/GridDetails/Grid1D_impl.h b/src/TNL/Meshes/GridDetails/Grid1D_impl.h
index e2e9818048c33e39a747cef78cf11b20e2cefae2..47bc02fff4f64e5db49bfb0eef539808cb982e7f 100644
--- a/src/TNL/Meshes/GridDetails/Grid1D_impl.h
+++ b/src/TNL/Meshes/GridDetails/Grid1D_impl.h
@@ -29,7 +29,8 @@ template< typename Real,
typename Index >
Grid< 1, Real, Device, Index >::Grid()
: numberOfCells( 0 ),
- numberOfVertices( 0 )
+ numberOfVertices( 0 ),
+ distGrid(nullptr)
{
}
@@ -77,13 +78,39 @@ void Grid< 1, Real, Device, Index >::computeSpaceSteps()
if( this->getDimensions().x() != 0 )
{
this->spaceSteps.x() = this->proportions.x() / ( Real ) this->getDimensions().x();
+ this->computeSpaceStepPowers();
+ }
+}
+
+template< typename Real,
+ typename Device,
+ typename Index >
+void Grid< 1, Real, Device, Index > ::computeSpaceStepPowers()
+{
const RealType& hx = this->spaceSteps.x();
this->spaceStepsProducts[ 0 ] = 1.0 / ( hx * hx );
this->spaceStepsProducts[ 1 ] = 1.0 / hx;
this->spaceStepsProducts[ 2 ] = 1.0;
this->spaceStepsProducts[ 3 ] = hx;
this->spaceStepsProducts[ 4 ] = hx * hx;
- }
+
+}
+
+
+template< typename Real,
+ typename Device,
+ typename Index >
+void Grid< 1, Real, Device, Index > ::computeProportions()
+{
+ this->proportions.x()=this->dimensions.x()*this->spaceSteps.x();
+}
+
+template< typename Real,
+ typename Device,
+ typename Index >
+void Grid< 1, Real, Device, Index > :: setOrigin( const PointType& origin)
+{
+ this->origin = origin;
}
template< typename Real,
@@ -223,6 +250,18 @@ getSpaceSteps() const
return this->spaceSteps;
}
+template< typename Real,
+ typename Device,
+ typename Index >
+inline void
+Grid< 1, Real, Device, Index >::
+setSpaceSteps(const typename Grid< 1, Real, Device, Index >::PointType& steps)
+{
+ this->spaceSteps=steps;
+ computeSpaceStepPowers();
+ computeProportions();
+}
+
template< typename Real,
typename Device,
typename Index >
@@ -302,6 +341,23 @@ Grid< 1, Real, Device, Index >::getDifferenceLpNorm( const GridFunction& f1,
return ::pow( lpNorm, 1.0 / p );
}
+template< typename Real,
+ typename Device,
+ typename Index >
+void Grid< 1, Real, Device, Index >:: SetDistMesh(DistributedMeshType * distMesh)
+{
+ this->distGrid=distMesh;
+}
+
+template< typename Real,
+ typename Device,
+ typename Index >
+DistributedMeshes::DistributedMesh <Grid< 1, Real, Device, Index >> *
+Grid< 1, Real, Device, Index >:: GetDistMesh(void) const
+{
+ return this->distGrid;
+}
+
template< typename Real,
typename Device,
typename Index >
diff --git a/src/TNL/Meshes/GridDetails/Grid2D.h b/src/TNL/Meshes/GridDetails/Grid2D.h
index 0f0c8af4dd6c7f57a802c1a25c5d97b85a7e5368..cc860f4f393bbe81001d511013aedf135d194e5f 100644
--- a/src/TNL/Meshes/GridDetails/Grid2D.h
+++ b/src/TNL/Meshes/GridDetails/Grid2D.h
@@ -15,6 +15,7 @@
#include <TNL/Meshes/GridDetails/GridEntityTopology.h>
#include <TNL/Meshes/GridDetails/GridEntityGetter.h>
#include <TNL/Meshes/GridDetails/NeighborGridEntityGetter.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
namespace TNL {
namespace Meshes {
@@ -34,6 +35,8 @@ class Grid< 2, Real, Device, Index > : public Object
typedef Grid< 2, Real, Devices::Host, Index > HostType;
typedef Grid< 2, Real, Devices::Cuda, Index > CudaType;
typedef Grid< 2, Real, Device, Index > ThisType;
+
+ typedef DistributedMeshes::DistributedMesh <ThisType> DistributedMeshType;
// TODO: deprecated and to be removed (GlobalIndexType shall be used instead)
typedef Index IndexType;
@@ -75,6 +78,9 @@ class Grid< 2, Real, Device, Index > : public Object
void setDomain( const PointType& origin,
const PointType& proportions );
+
+ void setOrigin( const PointType& origin);
+
__cuda_callable__
inline const PointType& getOrigin() const;
@@ -99,8 +105,11 @@ class Grid< 2, Real, Device, Index > : public Object
inline Index getEntityIndex( const Entity& entity ) const;
__cuda_callable__
+
inline const PointType& getSpaceSteps() const;
+ inline void setSpaceSteps(const PointType& steps);
+
template< int xPow, int yPow >
__cuda_callable__
const RealType& getSpaceStepsProducts() const;
@@ -127,6 +136,10 @@ class Grid< 2, Real, Device, Index > : public Object
typename GridFunction::RealType getDifferenceLpNorm( const GridFunction& f1,
const GridFunction& f2,
const typename GridFunction::RealType& p ) const;
+
+ void SetDistMesh(DistributedMeshType * distGrid);
+
+ DistributedMeshType * GetDistMesh(void) const;
//! Method for saving the object to a file as a binary data
bool save( File& file ) const;
@@ -141,6 +154,11 @@ class Grid< 2, Real, Device, Index > : public Object
void writeProlog( Logger& logger ) const;
protected:
+
+ void computeProportions();
+
+ __cuda_callable__
+ void computeSpaceStepPowers();
__cuda_callable__
void computeSpaceSteps();
@@ -154,6 +172,8 @@ class Grid< 2, Real, Device, Index > : public Object
PointType spaceSteps;
RealType spaceStepsProducts[ 5 ][ 5 ];
+
+ DistributedMeshType *distGrid;
template< typename, typename, int >
friend class GridEntityGetter;
diff --git a/src/TNL/Meshes/GridDetails/Grid2D_impl.h b/src/TNL/Meshes/GridDetails/Grid2D_impl.h
index 80c18a3691482b6b41055e0b77d40f19cb78f13e..4ef0717f37158c5fa1e6703849b431ea9bc9ad9e 100644
--- a/src/TNL/Meshes/GridDetails/Grid2D_impl.h
+++ b/src/TNL/Meshes/GridDetails/Grid2D_impl.h
@@ -32,7 +32,8 @@ Grid< 2, Real, Device, Index > :: Grid()
numberOfNxFaces( 0 ),
numberOfNyFaces( 0 ),
numberOfFaces( 0 ),
- numberOfVertices( 0 )
+ numberOfVertices( 0 ),
+ distGrid(nullptr)
{
}
@@ -82,6 +83,16 @@ void Grid< 2, Real, Device, Index > :: computeSpaceSteps()
{
this->spaceSteps.x() = this->proportions.x() / ( Real ) this->getDimensions().x();
this->spaceSteps.y() = this->proportions.y() / ( Real ) this->getDimensions().y();
+ this->computeSpaceStepPowers();
+ }
+}
+
+template< typename Real,
+ typename Device,
+ typename Index >
+__cuda_callable__
+void Grid< 2, Real, Device, Index > ::computeSpaceStepPowers()
+{
const RealType& hx = this->spaceSteps.x();
const RealType& hy = this->spaceSteps.y();
@@ -129,7 +140,15 @@ void Grid< 2, Real, Device, Index > :: computeSpaceSteps()
this->spaceStepsProducts[ i ][ j ] = auxX * auxY;
}
}
- }
+}
+
+template< typename Real,
+ typename Device,
+ typename Index >
+void Grid< 2, Real, Device, Index > ::computeProportions()
+{
+ this->proportions.x()=this->dimensions.x()*this->spaceSteps.x();
+ this->proportions.y()=this->dimensions.y()*this->spaceSteps.y();
}
template< typename Real,
@@ -179,6 +198,14 @@ void Grid< 2, Real, Device, Index > :: setDomain( const PointType& origin,
computeSpaceSteps();
}
+template< typename Real,
+ typename Device,
+ typename Index >
+void Grid< 2, Real, Device, Index > :: setOrigin( const PointType& origin)
+{
+ this->origin = origin;
+}
+
template< typename Real,
typename Device,
typename Index >
@@ -277,6 +304,18 @@ getSpaceSteps() const
return this->spaceSteps;
}
+template< typename Real,
+ typename Device,
+ typename Index >
+inline void
+Grid< 2, Real, Device, Index >::
+setSpaceSteps(const typename Grid< 2, Real, Device, Index >::PointType& steps)
+{
+ this->spaceSteps=steps;
+ computeSpaceStepPowers();
+ computeProportions();
+}
+
template< typename Real,
typename Device,
typename Index >
@@ -381,6 +420,23 @@ template< typename Real,
return ::pow( lpNorm, 1.0 / p );
}
+template< typename Real,
+ typename Device,
+ typename Index >
+void Grid< 2, Real, Device, Index >:: SetDistMesh(DistributedMeshType* distMesh)
+{
+ this->distGrid=distMesh;
+}
+
+template< typename Real,
+ typename Device,
+ typename Index >
+DistributedMeshes::DistributedMesh <Grid< 2, Real, Device, Index >> *
+Grid< 2, Real, Device, Index >:: GetDistMesh(void) const
+{
+ return this->distGrid;
+}
+
template< typename Real,
typename Device,
typename Index >
diff --git a/src/TNL/Meshes/GridDetails/Grid3D.h b/src/TNL/Meshes/GridDetails/Grid3D.h
index 0ff0bf045364d6746c5cb834f0f70f3d278349f2..7056e33f655197d14c4c5a61f7f79240335c5876 100644
--- a/src/TNL/Meshes/GridDetails/Grid3D.h
+++ b/src/TNL/Meshes/GridDetails/Grid3D.h
@@ -34,6 +34,8 @@ class Grid< 3, Real, Device, Index > : public Object
typedef Grid< 3, Real, Devices::Host, Index > HostType;
typedef Grid< 3, Real, Devices::Cuda, Index > CudaType;
typedef Grid< 3, Real, Device, Index > ThisType;
+
+ typedef DistributedMeshes::DistributedMesh <ThisType> DistributedMeshType;
// TODO: deprecated and to be removed (GlobalIndexType shall be used instead)
typedef Index IndexType;
@@ -68,6 +70,9 @@ class Grid< 3, Real, Device, Index > : public Object
void setDomain( const PointType& origin,
const PointType& proportions );
+
+ void setOrigin( const PointType& origin);
+
__cuda_callable__
inline const PointType& getOrigin() const;
@@ -93,7 +98,13 @@ class Grid< 3, Real, Device, Index > : public Object
__cuda_callable__
inline const PointType& getSpaceSteps() const;
-
+
+ inline void setSpaceSteps(const PointType& steps);
+
+ void SetDistMesh(DistributedMeshType * distGrid);
+
+ DistributedMeshType * GetDistMesh(void) const;
+
template< int xPow, int yPow, int zPow >
__cuda_callable__
const RealType& getSpaceStepsProducts() const;
@@ -135,6 +146,10 @@ class Grid< 3, Real, Device, Index > : public Object
protected:
+ void computeProportions();
+
+ void computeSpaceStepPowers();
+
void computeSpaceSteps();
CoordinatesType dimensions;
@@ -151,6 +166,8 @@ class Grid< 3, Real, Device, Index > : public Object
PointType spaceSteps;
RealType spaceStepsProducts[ 5 ][ 5 ][ 5 ];
+
+ DistributedMeshType *distGrid;
template< typename, typename, int >
friend class GridEntityGetter;
diff --git a/src/TNL/Meshes/GridDetails/Grid3D_impl.h b/src/TNL/Meshes/GridDetails/Grid3D_impl.h
index 9245a74c661329ae5f30511ff50267372507fd65..e2f4f273088780dfac733d050ed547468a88639d 100644
--- a/src/TNL/Meshes/GridDetails/Grid3D_impl.h
+++ b/src/TNL/Meshes/GridDetails/Grid3D_impl.h
@@ -39,7 +39,8 @@ Grid< 3, Real, Device, Index > :: Grid()
numberOfDzEdges( 0 ),
numberOfDxAndDyEdges( 0 ),
numberOfEdges( 0 ),
- numberOfVertices( 0 )
+ numberOfVertices( 0 ),
+ distGrid(nullptr)
{
}
@@ -91,6 +92,17 @@ void Grid< 3, Real, Device, Index > :: computeSpaceSteps()
this->spaceSteps.x() = this->proportions.x() / ( Real ) this->getDimensions().x();
this->spaceSteps.y() = this->proportions.y() / ( Real ) this->getDimensions().y();
this->spaceSteps.z() = this->proportions.z() / ( Real ) this->getDimensions().z();
+
+ this->computeSpaceStepPowers();
+
+ }
+};
+
+template< typename Real,
+ typename Device,
+ typename Index >
+void Grid< 3, Real, Device, Index > :: computeSpaceStepPowers()
+{
const RealType& hx = this->spaceSteps.x();
const RealType& hy = this->spaceSteps.y();
const RealType& hz = this->spaceSteps.z();
@@ -160,9 +172,38 @@ void Grid< 3, Real, Device, Index > :: computeSpaceSteps()
}
}
}
- }
}
+
+template< typename Real,
+ typename Device,
+ typename Index >
+void Grid< 3, Real, Device, Index > :: computeProportions()
+{
+ this->proportions.x()=this->dimensions.x()*this->spaceSteps.x();
+ this->proportions.y()=this->dimensions.y()*this->spaceSteps.y();
+ this->proportions.z()=this->dimensions.z()*this->spaceSteps.z();
+}
+
+template< typename Real,
+ typename Device,
+ typename Index >
+void Grid< 3, Real, Device, Index > :: setOrigin( const PointType& origin)
+{
+ this->origin=origin;
+}
+
+template< typename Real,
+ typename Device,
+ typename Index >
+void Grid< 3, Real, Device, Index > :: setSpaceSteps(const PointType& steps)
+{
+ this->spaceSteps=steps;
+ computeSpaceStepPowers();
+ computeProportions();
+}
+
+
template< typename Real,
typename Device,
typename Index >
@@ -453,6 +494,23 @@ template< typename Real,
return ::pow( lpNorm, 1.0 / p );
}
+template< typename Real,
+ typename Device,
+ typename Index >
+void Grid< 3, Real, Device, Index >:: SetDistMesh(DistributedMeshType * distMesh)
+{
+ this->distGrid=distMesh;
+}
+
+template< typename Real,
+ typename Device,
+ typename Index >
+DistributedMeshes::DistributedMesh <Grid< 3, Real, Device, Index >> *
+Grid< 3, Real, Device, Index >:: GetDistMesh(void) const
+{
+ return this->distGrid;
+}
+
template< typename Real,
typename Device,
typename Index >
diff --git a/src/TNL/Meshes/GridDetails/NeighborGridEntityGetter1D_impl.h b/src/TNL/Meshes/GridDetails/NeighborGridEntityGetter1D_impl.h
index 14143220eb451521afb6098e67ebc9889559a68a..b248bd163baa1efa14717bd6ffba465e27c2ff3f 100644
--- a/src/TNL/Meshes/GridDetails/NeighborGridEntityGetter1D_impl.h
+++ b/src/TNL/Meshes/GridDetails/NeighborGridEntityGetter1D_impl.h
@@ -475,4 +475,4 @@ class NeighborGridEntityGetter<
};
} // namespace Meshes
-} // namespace TNL
\ No newline at end of file
+} // namespace TNL
diff --git a/src/TNL/Meshes/GridDetails/Traverser_Grid1D_impl.h b/src/TNL/Meshes/GridDetails/Traverser_Grid1D_impl.h
index 6e8c1e3b2a6b03dca0bdfc0d4e58dbfe6d7280df..534fbf8e962b826ed5c63812d3bb17a46ca55412 100644
--- a/src/TNL/Meshes/GridDetails/Traverser_Grid1D_impl.h
+++ b/src/TNL/Meshes/GridDetails/Traverser_Grid1D_impl.h
@@ -11,6 +11,8 @@
#pragma once
#include <TNL/Meshes/GridDetails/GridTraverser.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+
namespace TNL {
namespace Meshes {
@@ -32,13 +34,39 @@ processBoundaryEntities( const GridPointer& gridPointer,
/****
* Boundary cells
*/
- static_assert( GridEntity::getEntityDimension() == 1, "The entity has wrong dimension." );
-
- GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, true >(
- gridPointer,
- CoordinatesType( 0 ),
- gridPointer->getDimensions() - CoordinatesType( 1 ),
- userDataPointer );
+ static_assert( GridEntity::getEntityDimension() == 1, "The entity has wrong dimensions." );
+
+ auto distributedgrid=gridPointer->GetDistMesh();
+ if(distributedgrid==nullptr||!distributedgrid->IsDistributed())
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, true >(
+ gridPointer,
+ CoordinatesType( 0 ),
+ gridPointer->getDimensions() - CoordinatesType( 1 ),
+ userDataPointer );
+ }
+ else
+ {
+ //Distributed
+ if(distributedgrid->getLeft()==-1)
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType( 0 ),
+ CoordinatesType( 0 ),
+ userDataPointer );
+ }
+
+ if(distributedgrid->getRight()==-1)
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ gridPointer->getDimensions() - CoordinatesType( 1 ),
+ gridPointer->getDimensions() - CoordinatesType( 1 ),
+ userDataPointer );
+ }
+ }
+
}
template< typename Real,
@@ -57,11 +85,37 @@ processInteriorEntities( const GridPointer& gridPointer,
*/
static_assert( GridEntity::getEntityDimension() == 1, "The entity has wrong dimension." );
- GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
- gridPointer,
- CoordinatesType( 1 ),
- gridPointer->getDimensions() - CoordinatesType( 2 ),
- userDataPointer );
+ auto distributedgrid=gridPointer->GetDistMesh();
+ if(distributedgrid==nullptr||!distributedgrid->IsDistributed())
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType( 1 ),
+ gridPointer->getDimensions() - CoordinatesType( 2 ),
+ userDataPointer );
+ }
+ else
+ {
+ //Distributed
+ CoordinatesType begin( distributedgrid->getOverlap().x() );
+ CoordinatesType end( gridPointer->getDimensions() - distributedgrid->getOverlap().x()-1 );
+ if(distributedgrid->getLeft()==-1)
+ {
+ begin=CoordinatesType( 1 );
+ }
+
+ if(distributedgrid->getRight()==-1)
+ {
+ end=gridPointer->getDimensions() - CoordinatesType( 2 );
+ }
+
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ begin,
+ end,
+ userDataPointer );
+ }
+
}
template< typename Real,
@@ -79,13 +133,40 @@ processAllEntities(
/****
* All cells
*/
- static_assert( GridEntity::getEntityDimension() == 1, "The entity has wrong dimension." );
- GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
- gridPointer,
- CoordinatesType( 0 ),
- gridPointer->getDimensions() - CoordinatesType( 1 ),
- userDataPointer );
+ static_assert( GridEntity::getEntityDimension() == 1, "The entity has wrong dimensions." );
+
+ auto distributedgrid=gridPointer->GetDistMesh();
+ if(distributedgrid==nullptr||!distributedgrid->IsDistributed())
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType( 0 ),
+ gridPointer->getDimensions() - CoordinatesType( 1 ),
+ userDataPointer );
+ }
+ else
+ {
+ //Distributed
+ CoordinatesType begin( distributedgrid->getOverlap().x() );
+ CoordinatesType end( gridPointer->getDimensions() - distributedgrid->getOverlap().x()-1 );
+ if(distributedgrid->getLeft()==-1)
+ {
+ begin=CoordinatesType( 0 );
+ }
+
+ if(distributedgrid->getRight()==-1)
+ {
+ end=gridPointer->getDimensions() - CoordinatesType( 1 );
+ }
+
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ begin,
+ end,
+ userDataPointer );
+ }
+
}
/****
diff --git a/src/TNL/Meshes/GridDetails/Traverser_Grid2D_impl.h b/src/TNL/Meshes/GridDetails/Traverser_Grid2D_impl.h
index 66d03c949733edbc138a9014b5141e7ba84e78ff..cbacdd8a37cfc24e9f13414a5eee7c654a1db0bd 100644
--- a/src/TNL/Meshes/GridDetails/Traverser_Grid2D_impl.h
+++ b/src/TNL/Meshes/GridDetails/Traverser_Grid2D_impl.h
@@ -34,12 +34,60 @@ processBoundaryEntities( const GridPointer& gridPointer,
*/
static_assert( GridEntity::getEntityDimension() == 2, "The entity has wrong dimension." );
- GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, true, 1, 1 >(
- gridPointer,
- CoordinatesType( 0, 0 ),
- gridPointer->getDimensions() - CoordinatesType( 1, 1 ),
- userDataPointer,
- 0 );
+ auto distributedgrid=gridPointer->GetDistMesh();
+ if(distributedgrid==nullptr||!distributedgrid->IsDistributed())
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, true, 1, 1 >(
+ gridPointer,
+ CoordinatesType( 0, 0 ),
+ gridPointer->getDimensions() - CoordinatesType( 1, 1 ),
+ userDataPointer,
+ 0 );
+ }
+ else
+ {
+ //Distributed
+ int* neighbors=distributedgrid->getNeighbors();
+ if(neighbors[Meshes::DistributedMeshes::Left]==-1)
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType( 0, 0 ),
+ CoordinatesType(0,gridPointer->getDimensions().y()-1),
+ userDataPointer,
+ 0 );
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Right]==-1)
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType( gridPointer->getDimensions().x()-1, 0 ),
+ CoordinatesType(gridPointer->getDimensions().x()-1,gridPointer->getDimensions().y()-1),
+ userDataPointer,
+ 0 );
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Up]==-1)
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType( 0, 0 ),
+ CoordinatesType(gridPointer->getDimensions().x()-1,0),
+ userDataPointer,
+ 0 );
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Down]==-1)
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType( 0, gridPointer->getDimensions().y()-1 ),
+ CoordinatesType(gridPointer->getDimensions().x()-1,gridPointer->getDimensions().y()-1),
+ userDataPointer,
+ 0 );
+ }
+ }
}
template< typename Real,
@@ -56,7 +104,12 @@ processInteriorEntities( const GridPointer& gridPointer,
/****
* Interior cells
*/
- static_assert( GridEntity::getEntityDimension() == 2, "The entity has wrong dimension." );
+
+ static_assert( GridEntity::getEntityDimension() == 2, "The entity has wrong dimensions." );
+
+ auto distributedgrid=gridPointer->GetDistMesh();
+ if(distributedgrid==nullptr||!distributedgrid->IsDistributed())
+ {
GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
gridPointer,
@@ -64,6 +117,40 @@ processInteriorEntities( const GridPointer& gridPointer,
gridPointer->getDimensions() - CoordinatesType( 2, 2 ),
userDataPointer,
0 );
+ }
+ else
+ {
+ //MPI
+ int* neighbors=distributedgrid->getNeighbors();
+ CoordinatesType begin( distributedgrid->getOverlap());
+ CoordinatesType end( gridPointer->getDimensions() - distributedgrid->getOverlap()- CoordinatesType(1,1) );
+ if(neighbors[Meshes::DistributedMeshes::Left]==-1)
+ {
+ begin.x()= 1 ;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Right]==-1)
+ {
+ end.x()=gridPointer->getDimensions().x()-2;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Up]==-1)
+ {
+ begin.y()= 1 ;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Down]==-1)
+ {
+ end.y()=gridPointer->getDimensions().y()-2;
+ }
+
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ begin,
+ end,
+ userDataPointer,
+ 0);
+ }
}
template< typename Real,
@@ -82,12 +169,48 @@ processAllEntities( const GridPointer& gridPointer,
*/
static_assert( GridEntity::getEntityDimension() == 2, "The entity has wrong dimension." );
- GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
- gridPointer,
- CoordinatesType( 0, 0 ),
- gridPointer->getDimensions() - CoordinatesType( 1, 1 ),
- userDataPointer,
- 0 );
+ auto distributedgrid=gridPointer->GetDistMesh();
+ if(distributedgrid==nullptr||!distributedgrid->IsDistributed())
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType( 0, 0 ),
+ gridPointer->getDimensions() - CoordinatesType( 1, 1 ),
+ userDataPointer,
+ 0 );
+ }
+ else
+ {
+ int* neighbors=distributedgrid->getNeighbors();
+ CoordinatesType begin( distributedgrid->getOverlap());
+ CoordinatesType end( gridPointer->getDimensions() - distributedgrid->getOverlap()- CoordinatesType(1,1) );
+ if(neighbors[Meshes::DistributedMeshes::Left]==-1)
+ {
+ begin.x()= 0 ;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Right]==-1)
+ {
+ end.x()=gridPointer->getDimensions().x()-1;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Up]==-1)
+ {
+ begin.y()= 0 ;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Down]==-1)
+ {
+ end.y()=gridPointer->getDimensions().y()-1;
+ }
+
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ begin,
+ end,
+ userDataPointer,
+ 0);
+ }
}
/****
diff --git a/src/TNL/Meshes/GridDetails/Traverser_Grid3D_impl.h b/src/TNL/Meshes/GridDetails/Traverser_Grid3D_impl.h
index 84e44ef17c29aca27da6c940e6ab29841fb9aa86..c2ddff5924e89b0ada944f674196f8413fa023e3 100644
--- a/src/TNL/Meshes/GridDetails/Traverser_Grid3D_impl.h
+++ b/src/TNL/Meshes/GridDetails/Traverser_Grid3D_impl.h
@@ -12,6 +12,8 @@
#include <TNL/Meshes/GridDetails/GridTraverser.h>
+#include "Traverser_Grid3D.h"
+
namespace TNL {
namespace Meshes {
@@ -34,12 +36,79 @@ processBoundaryEntities( const GridPointer& gridPointer,
*/
static_assert( GridEntity::getEntityDimension() == 3, "The entity has wrong dimension." );
- GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, true, 1, 1, 1 >(
- gridPointer,
- CoordinatesType( 0, 0, 0 ),
- gridPointer->getDimensions() - CoordinatesType( 1, 1, 1 ),
- userDataPointer,
- 0 );
+ auto distributedgrid=gridPointer->GetDistMesh();
+ if(distributedgrid==nullptr||!distributedgrid->IsDistributed())
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, true, 1, 1, 1 >(
+ gridPointer,
+ CoordinatesType( 0, 0, 0 ),
+ gridPointer->getDimensions() - CoordinatesType( 1, 1, 1 ),
+ userDataPointer,
+ 0 );
+ }
+ else
+ {
+ int* neighbors=distributedgrid->getNeighbors();
+ if(neighbors[Meshes::DistributedMeshes::West]==-1)
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType(0,0,0),
+ CoordinatesType(0,gridPointer->getDimensions().y()-1,gridPointer->getDimensions().z()-1),
+ userDataPointer,
+ 0 );
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::East]==-1)
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType(gridPointer->getDimensions().x()-1,0,0),
+ CoordinatesType(gridPointer->getDimensions().x()-1,gridPointer->getDimensions().y()-1,gridPointer->getDimensions().z()-1),
+ userDataPointer,
+ 0 );
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Nord]==-1)
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType(0,0,0),
+ CoordinatesType(gridPointer->getDimensions().x()-1,0,gridPointer->getDimensions().z()-1),
+ userDataPointer,
+ 0 );
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::South]==-1)
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType(0,gridPointer->getDimensions().y()-1,0),
+ CoordinatesType(gridPointer->getDimensions().x()-1,gridPointer->getDimensions().y()-1,gridPointer->getDimensions().z()-1),
+ userDataPointer,
+ 0 );
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Bottom]==-1)
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType(0,0,0),
+ CoordinatesType(gridPointer->getDimensions().x()-1,gridPointer->getDimensions().y()-1,0),
+ userDataPointer,
+ 0 );
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Top]==-1)
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType(0,0,gridPointer->getDimensions().z()-1),
+ CoordinatesType(gridPointer->getDimensions().x()-1,gridPointer->getDimensions().y()-1,gridPointer->getDimensions().z()-1),
+ userDataPointer,
+ 0 );
+ }
+ }
}
template< typename Real,
@@ -57,13 +126,60 @@ processInteriorEntities( const GridPointer& gridPointer,
* Interior cells
*/
static_assert( GridEntity::getEntityDimension() == 3, "The entity has wrong dimension." );
-
+
+ auto distributedgrid=gridPointer->GetDistMesh();
+ if(distributedgrid==nullptr||!distributedgrid->IsDistributed())
+ {
GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
gridPointer,
CoordinatesType( 1, 1, 1 ),
gridPointer->getDimensions() - CoordinatesType( 2, 2, 2 ),
userDataPointer,
0 );
+ }
+ else
+ {
+ int* neighbors=distributedgrid->getNeighbors();
+ CoordinatesType begin( distributedgrid->getOverlap());
+ CoordinatesType end( gridPointer->getDimensions() - distributedgrid->getOverlap()- CoordinatesType(1,1,1) );
+ if(neighbors[Meshes::DistributedMeshes::West]==-1)
+ {
+ begin.x()= 1 ;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::East]==-1)
+ {
+ end.x()=gridPointer->getDimensions().x()-2;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Nord]==-1)
+ {
+ begin.y()= 1 ;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::South]==-1)
+ {
+ end.y()=gridPointer->getDimensions().y()-2;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Bottom]==-1)
+ {
+ begin.z()= 1 ;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Top]==-1)
+ {
+ end.z()=gridPointer->getDimensions().z()-2;
+ }
+
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ begin,
+ end,
+ userDataPointer,
+ 0);
+ }
+
}
template< typename Real,
@@ -82,12 +198,58 @@ processAllEntities( const GridPointer& gridPointer,
*/
static_assert( GridEntity::getEntityDimension() == 3, "The entity has wrong dimension." );
- GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
- gridPointer,
- CoordinatesType( 0, 0, 0 ),
- gridPointer->getDimensions() - CoordinatesType( 1, 1, 1 ),
- userDataPointer,
- 0 );
+ auto distributedgrid=gridPointer->GetDistMesh();
+ if(distributedgrid==nullptr||!distributedgrid->IsDistributed())
+ {
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ CoordinatesType( 0, 0, 0 ),
+ gridPointer->getDimensions() - CoordinatesType( 1, 1, 1 ),
+ userDataPointer,
+ 0 );
+ }
+ else
+ {
+ CoordinatesType begin( distributedgrid->getOverlap());
+ CoordinatesType end( gridPointer->getDimensions() - distributedgrid->getOverlap()- CoordinatesType(1,1,1) );
+ int* neighbors=distributedgrid->getNeighbors();
+ if(neighbors[Meshes::DistributedMeshes::West]==-1)
+ {
+ begin.x()= 0 ;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::East]==-1)
+ {
+ end.x()=gridPointer->getDimensions().x()-1;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Nord]==-1)
+ {
+ begin.y()= 0 ;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::South]==-1)
+ {
+ end.y()=gridPointer->getDimensions().y()-1;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Bottom]==-1)
+ {
+ begin.z()= 0 ;
+ }
+
+ if(neighbors[Meshes::DistributedMeshes::Top]==-1)
+ {
+ end.z()=gridPointer->getDimensions().z()-1;
+ }
+
+ GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >(
+ gridPointer,
+ begin,
+ end,
+ userDataPointer,
+ 0 );
+ }
}
/****
diff --git a/src/TNL/Meshes/Mesh.h b/src/TNL/Meshes/Mesh.h
index 109ccc1043a25588e3bdffeec8b1eecbfdf26314..55c5bca319646a16fb927e0e99c8a74a8dd21817 100644
--- a/src/TNL/Meshes/Mesh.h
+++ b/src/TNL/Meshes/Mesh.h
@@ -26,6 +26,8 @@
#include <TNL/Meshes/MeshDetails/MeshLayers/StorageLayer.h>
#include <TNL/Meshes/MeshDetails/MeshLayers/BoundaryTags/LayerFamily.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+
namespace TNL {
namespace Meshes {
@@ -170,6 +172,11 @@ class Mesh
void writeProlog( Logger& logger ) const;
+ DistributedMeshes::DistributedMesh< Mesh<MeshConfig,Device> >* GetDistMesh(void) const
+ {
+ return NULL;
+ };
+
protected:
// Methods for the mesh initializer
using StorageBaseType::setEntitiesCount;
diff --git a/src/TNL/Meshes/TypeResolver/TypeResolver_impl.h b/src/TNL/Meshes/TypeResolver/TypeResolver_impl.h
index 82767a0f02e2862eba1bf0c0849d3367e5806ace..b6800d7e05e6f0c6a09a87b89c00e9f75f4be5e8 100644
--- a/src/TNL/Meshes/TypeResolver/TypeResolver_impl.h
+++ b/src/TNL/Meshes/TypeResolver/TypeResolver_impl.h
@@ -111,10 +111,16 @@ bool resolveMeshType( const String& fileName_,
}
// TODO: reorganize
-template< typename MeshConfig, typename Device >
+template< typename CommunicatorType, typename MeshConfig, typename Device >
bool
-loadMesh( const String& fileName_, Mesh< MeshConfig, Device >& mesh )
+loadMesh( const String& fileName_, Mesh< MeshConfig, Device >& mesh, DistributedMeshes::DistributedMesh<Mesh< MeshConfig, Device >> &distributedMesh )
{
+ if(CommunicatorType::isDistributed())
+ {
+ std::cerr << "Distributed Mesh si not suported yet, only Distributed Grid is supported.";
+ return false;
+ }
+
std::cout << "Loading mesh from file " << fileName_ << " ..." << std::endl;
std::string fileName( fileName_.getString() );
bool status = true;
@@ -145,10 +151,16 @@ loadMesh( const String& fileName_, Mesh< MeshConfig, Device >& mesh )
return true;
}
-template< typename MeshConfig >
+template<typename CommunicatorType, typename MeshConfig >
bool
-loadMesh( const String& fileName, Mesh< MeshConfig, Devices::Cuda >& mesh )
+loadMesh( const String& fileName, Mesh< MeshConfig, Devices::Cuda >& mesh, DistributedMeshes::DistributedMesh<Mesh< MeshConfig, Devices::Cuda >> &distributedMesh )
{
+ if(CommunicatorType::isDistributed())
+ {
+ std::cerr << "Distributed Mesh si not suported yet, only Distributed Grid is supported.";
+ return false;
+ }
+
Mesh< MeshConfig, Devices::Host > hostMesh;
if( ! loadMesh( fileName, hostMesh ) )
return false;
@@ -156,11 +168,46 @@ loadMesh( const String& fileName, Mesh< MeshConfig, Devices::Cuda >& mesh )
return true;
}
-template< int Dimension, typename Real, typename Device, typename Index >
+template<typename CommunicatorType, int Dimension, typename Real, typename Device, typename Index >
bool
-loadMesh( const String& fileName, Grid< Dimension, Real, Device, Index >& mesh )
+loadMesh( const String& fileName, Grid< Dimension, Real, Device, Index >& mesh, DistributedMeshes::DistributedMesh<Grid< Dimension, Real, Device, Index >> &distributedMesh)
{
- std::cout << "Loading mesh from file " << fileName << " ..." << std::endl;
+
+ if(CommunicatorType::isDistributed())
+ {
+ std::cout << "Loading a global mesh from the file " << fileName << "...";
+ Grid< Dimension, Real, Device, Index > globalGrid;
+ if( ! globalGrid.load( fileName ) )
+ {
+ std::cerr << std::endl;
+ std::cerr << "I am not able to load the global mesh from the file " << fileName << "." << std::endl;
+ return false;
+ }
+ std::cout << " [ OK ] " << std::endl;
+
+ typename Meshes::DistributedMeshes::DistributedMesh<Grid< Dimension, Real, Device, Index >>::CoordinatesType overlap;
+ overlap.setValue(1);
+ distributedMesh.template setGlobalGrid<CommunicatorType>(globalGrid,overlap);
+ distributedMesh.SetupGrid(mesh);
+ return true;
+ }
+ else
+ {
+ std::cout << "Loading a mesh from the file " << fileName << "...";
+ if( ! mesh.load( fileName ) )
+ {
+ std::cerr << std::endl;
+ std::cerr << "I am not able to load the mesh from the file " << fileName << "." << std::endl;
+ std::cerr << " You may create it with tools like tnl-grid-setup or tnl-mesh-convert." << std::endl;
+ return false;
+ }
+ std::cout << " [ OK ] " << std::endl;
+ return true;
+ }
+
+
+
+ /* std::cout << "Loading mesh from file " << fileName << " ..." << std::endl;
if( ! mesh.load( fileName ) )
{
std::cerr << "I am not able to load the grid from the file " << fileName << ". "
@@ -168,7 +215,7 @@ loadMesh( const String& fileName, Grid< Dimension, Real, Device, Index >& mesh )
<< std::endl;
return false;
}
- return true;
+ return true;*/
}
} // namespace Meshes
diff --git a/src/TNL/Problems/HeatEquationEocProblem.h b/src/TNL/Problems/HeatEquationEocProblem.h
index fd292dde621b9cdbcea3a88b241a0405a728201c..361e4d48f26d08f99693447df7f3805216628e81 100644
--- a/src/TNL/Problems/HeatEquationEocProblem.h
+++ b/src/TNL/Problems/HeatEquationEocProblem.h
@@ -25,13 +25,14 @@ namespace Problems {
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator = Operators::LinearDiffusion< Mesh,
typename BoundaryCondition::RealType > >
-class HeatEquationEocProblem : public HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >
+class HeatEquationEocProblem : public HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >
{
public:
- typedef HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator > BaseType;
+ typedef HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator > BaseType;
using typename BaseType::MeshPointer;
diff --git a/src/TNL/Problems/HeatEquationEocProblem_impl.h b/src/TNL/Problems/HeatEquationEocProblem_impl.h
index 87eb44d89c2b20a6c6e9f7b36d7fe4f7b7eaaeb3..42b0d61bf6f21bff77caa4bb97d27b3de4067648 100644
--- a/src/TNL/Problems/HeatEquationEocProblem_impl.h
+++ b/src/TNL/Problems/HeatEquationEocProblem_impl.h
@@ -25,9 +25,10 @@ namespace Problems {
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
String
-HeatEquationEocProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationEocProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getTypeStatic()
{
return String( "heatEquationEocSolver< " ) + Mesh :: getTypeStatic() + " >";
@@ -36,9 +37,10 @@ getTypeStatic()
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
bool
-HeatEquationEocProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationEocProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
setup( const MeshPointer& meshPointer,
const Config::ParameterContainer& parameters,
const String& prefix )
diff --git a/src/TNL/Problems/HeatEquationProblem.h b/src/TNL/Problems/HeatEquationProblem.h
index a97c681a232ba33b873f4046a590431432a7685f..28ebf2d65898869f9c18e2b0ba4783998e5e7b57 100644
--- a/src/TNL/Problems/HeatEquationProblem.h
+++ b/src/TNL/Problems/HeatEquationProblem.h
@@ -31,6 +31,7 @@ namespace Problems {
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator = Operators::LinearDiffusion< Mesh,
typename BoundaryCondition::RealType > >
class HeatEquationProblem : public PDEProblem< Mesh,
@@ -58,6 +59,8 @@ class HeatEquationProblem : public PDEProblem< Mesh,
using typename BaseType::MeshDependentDataType;
using typename BaseType::MeshDependentDataPointer;
+ typedef CommType CommunicatorType;
+
static String getTypeStatic();
String getPrologHeader() const;
diff --git a/src/TNL/Problems/HeatEquationProblem_impl.h b/src/TNL/Problems/HeatEquationProblem_impl.h
index 5a27d4da381f7aa25672055ab3933986d62faf80..07040f40001cfe32bd9bf7693dab251859ee236d 100644
--- a/src/TNL/Problems/HeatEquationProblem_impl.h
+++ b/src/TNL/Problems/HeatEquationProblem_impl.h
@@ -24,15 +24,20 @@
#include "HeatEquationProblem.h"
+//#define MPIIO
+#include <TNL/Meshes/DistributedMeshes/DistributedGridIO.h>
+
+
namespace TNL {
namespace Problems {
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
String
-HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getTypeStatic()
{
return String( "HeatEquationProblem< " ) + Mesh :: getTypeStatic() + " >";
@@ -41,9 +46,10 @@ getTypeStatic()
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
String
-HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getPrologHeader() const
{
return String( "Heat equation" );
@@ -52,9 +58,10 @@ getPrologHeader() const
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
void
-HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
writeProlog( Logger& logger, const Config::ParameterContainer& parameters ) const
{
}
@@ -62,9 +69,10 @@ writeProlog( Logger& logger, const Config::ParameterContainer& parameters ) cons
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
bool
-HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
writeEpilog( Logger& logger )
{
return true;
@@ -73,9 +81,10 @@ writeEpilog( Logger& logger )
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
bool
-HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
setup( const MeshPointer& meshPointer,
const Config::ParameterContainer& parameters,
const String& prefix )
@@ -96,9 +105,10 @@ setup( const MeshPointer& meshPointer,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
-typename HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::IndexType
-HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+typename HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::IndexType
+HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getDofs( const MeshPointer& meshPointer ) const
{
/****
@@ -110,9 +120,10 @@ getDofs( const MeshPointer& meshPointer ) const
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
void
-HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
bindDofs( const MeshPointer& meshPointer,
const DofVectorPointer& dofVector )
{
@@ -123,9 +134,10 @@ bindDofs( const MeshPointer& meshPointer,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
bool
-HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
setInitialCondition( const Config::ParameterContainer& parameters,
const MeshPointer& meshPointer,
DofVectorPointer& dofs,
@@ -133,21 +145,32 @@ setInitialCondition( const Config::ParameterContainer& parameters,
{
this->bindDofs( meshPointer, dofs );
const String& initialConditionFile = parameters.getParameter< String >( "initial-condition" );
- if( ! this->uPointer->boundLoad( initialConditionFile ) )
- {
- std::cerr << "I am not able to load the initial condition from the file " << initialConditionFile << "." << std::endl;
- return false;
- }
+ std::cout<<"setInitialCondition" <<std::endl;
+ if(CommunicatorType::isDistributed())
+ {
+ std::cout<<"Nodes Distribution: " << uPointer->getMesh().GetDistMesh()->printProcessDistr() << std::endl;
+ Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType,Meshes::DistributedMeshes::LocalCopy> ::load(initialConditionFile, *uPointer );
+ uPointer->template Synchronize<CommunicatorType>();
+ }
+ else
+ {
+ if( ! this->uPointer->boundLoad( initialConditionFile ) )
+ {
+ std::cerr << "I am not able to load the initial condition from the file " << initialConditionFile << "." << std::endl;
+ return false;
+ }
+ }
return true;
}
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
template< typename MatrixPointer >
bool
-HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
setupLinearSystem( const MeshPointer& meshPointer,
MatrixPointer& matrixPointer )
{
@@ -170,9 +193,10 @@ setupLinearSystem( const MeshPointer& meshPointer,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
bool
-HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
makeSnapshot( const RealType& time,
const IndexType& step,
const MeshPointer& meshPointer,
@@ -187,17 +211,28 @@ makeSnapshot( const RealType& time,
fileName.setFileNameBase( "u-" );
fileName.setExtension( "tnl" );
fileName.setIndex( step );
- if( ! this->uPointer->save( fileName.getFileName() ) )
- return false;
+
+ std::cout<<"Make snapshot" <<std::endl;
+
+ if(CommunicatorType::isDistributed())
+ {
+ Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType,Meshes::DistributedMeshes::LocalCopy> ::save(fileName.getFileName(), *uPointer );
+ }
+ else
+ {
+ if( ! this->uPointer->save( fileName.getFileName() ) )
+ return false;
+ }
return true;
}
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
void
-HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getExplicitUpdate( const RealType& time,
const RealType& tau,
const MeshPointer& meshPointer,
@@ -219,16 +254,18 @@ getExplicitUpdate( const RealType& time,
this->explicitUpdater.setDifferentialOperator( this->differentialOperatorPointer ),
this->explicitUpdater.setBoundaryConditions( this->boundaryConditionPointer ),
this->explicitUpdater.setRightHandSide( this->rightHandSidePointer ),
- this->explicitUpdater.template update< typename Mesh::Cell >( time, tau, meshPointer, this->uPointer, fuPointer );
+ this->explicitUpdater.template update< typename Mesh::Cell, CommType >( time, tau, meshPointer, this->uPointer, fuPointer );
+
}
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
template< typename MatrixPointer >
void
-HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
assemblyLinearSystem( const RealType& time,
const RealType& tau,
const MeshPointer& meshPointer,
diff --git a/src/TNL/SmartPointersRegister.cpp b/src/TNL/SmartPointersRegister.cpp
index 03d3e058f004c201c61d7c307cce7327d5e106c9..f2910ba36ecfde3fdf0f7c57cfcef61fa3c65a0e 100644
--- a/src/TNL/SmartPointersRegister.cpp
+++ b/src/TNL/SmartPointersRegister.cpp
@@ -30,7 +30,7 @@ void SmartPointersRegister::remove( SmartPointer* pointer, int deviceId )
try {
pointersOnDevices.at( deviceId ).erase( pointer );
}
- catch( std::out_of_range ) {
+ catch( const std::out_of_range& ) {
std::cerr << "Given deviceId " << deviceId << " does not have any pointers yet. "
<< "Requested to remove pointer " << pointer << ". "
<< "This is most likely a bug in the smart pointer." << std::endl;
@@ -46,7 +46,7 @@ bool SmartPointersRegister::synchronizeDevice( int deviceId )
( *it ).synchronize();
return TNL_CHECK_CUDA_DEVICE;
}
- catch( std::out_of_range ) {
+ catch( const std::out_of_range& ) {
return false;
}
}
diff --git a/src/TNL/Solvers/ODE/Euler_impl.h b/src/TNL/Solvers/ODE/Euler_impl.h
index 02c21a6c0bd907bb4843abded43e3a96c4cfa8aa..92cb1e5c974b6a4b19c34b3d116f4fcb6d3ec568 100644
--- a/src/TNL/Solvers/ODE/Euler_impl.h
+++ b/src/TNL/Solvers/ODE/Euler_impl.h
@@ -227,8 +227,15 @@ void Euler< Problem > :: computeNewTimeLevel( DofVectorPointer& u,
}
#endif
}
- localResidue /= tau * ( RealType ) size;
- MPIAllreduce( localResidue, currentResidue, 1, MPI_SUM, this->solver_comm );
+
+
+
+ localResidue /= tau * ( RealType ) size;
+#ifdef USE_MPI
+ TNLMPI::Allreduce( localResidue, currentResidue, 1, MPI_SUM);
+#else
+ currentResidue=localResidue;
+#endif
}
diff --git a/src/TNL/Solvers/ODE/ExplicitSolver.h b/src/TNL/Solvers/ODE/ExplicitSolver.h
index 721b472c6e28ef097802dff3c2ea52166d42399f..7ac4f14ecd4bc1eb0ae827a3f67dfd2de4dd1628 100644
--- a/src/TNL/Solvers/ODE/ExplicitSolver.h
+++ b/src/TNL/Solvers/ODE/ExplicitSolver.h
@@ -64,8 +64,6 @@ class ExplicitSolver : public IterativeSolver< typename Problem::RealType,
void setMaxTau( const RealType& maxTau );
const RealType& getMaxTau() const;
-
- void setMPIComm( MPI_Comm comm );
void setVerbose( IndexType v );
@@ -98,8 +96,6 @@ protected:
RealType maxTau;
- MPI_Comm solver_comm;
-
IndexType verbosity;
Timer* timer;
diff --git a/src/TNL/Solvers/ODE/ExplicitSolver_impl.h b/src/TNL/Solvers/ODE/ExplicitSolver_impl.h
index 352c5ca2b2fee5a6a93ce641114553e0feb662fa..78f0b0e69dca0ab2e107a17a055cccc12f32dd60 100644
--- a/src/TNL/Solvers/ODE/ExplicitSolver_impl.h
+++ b/src/TNL/Solvers/ODE/ExplicitSolver_impl.h
@@ -21,7 +21,6 @@ ExplicitSolver()
stopTime( 0.0 ),
tau( 0.0 ),
maxTau( DBL_MAX ),
- solver_comm( MPI_COMM_WORLD ),
verbosity( 0 ),
timer( &defaultTimer ),
testingMode( false ),
@@ -123,14 +122,6 @@ setStopTime( const RealType& stopTime )
this->stopTime = stopTime;
}
-template< class Problem >
-void
-ExplicitSolver< Problem >::
-setMPIComm( MPI_Comm comm )
-{
- this->solver_comm = comm;
-};
-
template< class Problem >
void
ExplicitSolver< Problem >::
diff --git a/src/TNL/Solvers/ODE/Merson_impl.h b/src/TNL/Solvers/ODE/Merson_impl.h
index d206087f1cc3d5ba78a888ef916b06f40e76235a..92f754af50aadfafebe8ed67e2afccf3bcb1c61f 100644
--- a/src/TNL/Solvers/ODE/Merson_impl.h
+++ b/src/TNL/Solvers/ODE/Merson_impl.h
@@ -213,7 +213,9 @@ bool Merson< Problem > :: solve( DofVectorPointer& u )
{
currentTau *= 0.8 * ::pow( adaptivity / eps, 0.2 );
currentTau = min( currentTau, this->getMaxTau() );
- MPIBcast( currentTau, 1, 0, this->solver_comm );
+#ifdef USE_MPI
+ TNLMPI::Bcast( currentTau, 1, 0 );
+#endif
}
if( time + currentTau > this->getStopTime() )
currentTau = this->getStopTime() - time; //we don't want to keep such tau
@@ -419,7 +421,11 @@ typename Problem :: RealType Merson< Problem > :: computeError( const RealType t
}
#endif
}
- MPIAllreduce( eps, maxEps, 1, MPI_MAX, this->solver_comm );
+ #ifdef USE_MPI
+ TNLMPI::Allreduce( eps, maxEps, 1, MPI_MAX);
+ #else
+ maxEps=eps;
+ #endif
return maxEps;
}
@@ -484,8 +490,13 @@ void Merson< Problem >::computeNewTimeLevel( DofVectorPointer& u,
#endif
}
+
localResidue /= tau * ( RealType ) size;
- MPIAllreduce( localResidue, currentResidue, 1, MPI_SUM, this->solver_comm );
+#ifdef USE_MPI
+ TNLMPI::Allreduce( localResidue, currentResidue, 1, MPI_SUM);
+#else
+ currentResidue=localResidue;
+#endif
}
diff --git a/src/TNL/Solvers/PDE/ExplicitUpdater.h b/src/TNL/Solvers/PDE/ExplicitUpdater.h
index 264ecef7c0a83a3f6ead920de11a0225fb61cd36..21608dc84e406cc1e54c8a6d3b0ad5cd97374eb6 100644
--- a/src/TNL/Solvers/PDE/ExplicitUpdater.h
+++ b/src/TNL/Solvers/PDE/ExplicitUpdater.h
@@ -19,6 +19,10 @@
#include <TNL/Meshes/GridDetails/Traverser_Grid3D.h>
#include <TNL/Solvers/PDE/ExplicitUpdater.h>
+#ifdef USE_MPI
+ #include <TNL/Meshes/DistributedGridSynchronizer.h>
+#endif
+
namespace TNL {
namespace Solvers {
@@ -109,7 +113,8 @@ class ExplicitUpdater
this->userDataPointer->rightHandSide = &rightHandSidePointer.template getData< DeviceType >();
}
- template< typename EntityType >
+ template< typename EntityType,
+ typename CommunicatorType >
void update( const RealType& time,
const RealType& tau,
const MeshPointer& meshPointer,
@@ -143,7 +148,10 @@ class ExplicitUpdater
TraverserBoundaryEntitiesProcessor >
( meshPointer,
userDataPointer );
-
+
+ if(CommunicatorType::isDistributed())
+ fuPointer->template Synchronize<CommunicatorType>();
+
}
diff --git a/src/TNL/Solvers/PDE/TimeDependentPDESolver.h b/src/TNL/Solvers/PDE/TimeDependentPDESolver.h
index af4889ac329fa032bc71bf355e0044327f79c5bb..d0a9cb7c5cba57e940e8e1eafd11064c388e59fb 100644
--- a/src/TNL/Solvers/PDE/TimeDependentPDESolver.h
+++ b/src/TNL/Solvers/PDE/TimeDependentPDESolver.h
@@ -17,6 +17,9 @@
#include <TNL/Solvers/PDE/PDESolver.h>
#include <TNL/Solvers/PDE/MeshDependentTimeSteps.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+
+
namespace TNL {
namespace Solvers {
namespace PDE {
@@ -84,6 +87,8 @@ class TimeDependentPDESolver
MeshPointer meshPointer;
+ Meshes::DistributedMeshes::DistributedMesh<MeshType> distrMesh;
+
DofVectorPointer dofsPointer;
MeshDependentDataPointer meshDependentDataPointer;
diff --git a/src/TNL/Solvers/PDE/TimeDependentPDESolver_impl.h b/src/TNL/Solvers/PDE/TimeDependentPDESolver_impl.h
index a8a01aa9c506e288cedb2d0f366cbbd3d7de366a..8e5bb166da2b7b708374512a00e3174701560ec9 100644
--- a/src/TNL/Solvers/PDE/TimeDependentPDESolver_impl.h
+++ b/src/TNL/Solvers/PDE/TimeDependentPDESolver_impl.h
@@ -30,6 +30,7 @@ TimeDependentPDESolver()
{
}
+
template< typename Problem,
typename DiscreteSolver,
typename TimeStepper >
@@ -62,9 +63,8 @@ setup( const Config::ParameterContainer& parameters,
* Load the mesh from the mesh file
*/
const String& meshFile = parameters.getParameter< String >( "mesh" );
- if( ! Meshes::loadMesh( meshFile, *meshPointer ) )
+ if( ! Meshes::loadMesh<typename Problem::CommunicatorType>( meshFile, *meshPointer, distrMesh ) )
return false;
-
/****
* Setup the problem
*/
diff --git a/src/TNL/Solvers/Solver.h b/src/TNL/Solvers/Solver.h
index 0d5925a1eb80b1fc7a81c94d129c70c5de242545..446569993b0add6b279e690a1ae57f8a9c6351ec 100644
--- a/src/TNL/Solvers/Solver.h
+++ b/src/TNL/Solvers/Solver.h
@@ -15,7 +15,7 @@
namespace TNL {
namespace Solvers {
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
template< typename ConfTag > class ProblemConfig,
typename ConfigTag = DefaultBuildConfigTag >
class Solver
diff --git a/src/TNL/Solvers/SolverInitiator.h b/src/TNL/Solvers/SolverInitiator.h
index 40a6d9cddd773c29e58813b996d813373af60b67..31fad1ee81184d585c1476db02665652c17e6685 100644
--- a/src/TNL/Solvers/SolverInitiator.h
+++ b/src/TNL/Solvers/SolverInitiator.h
@@ -17,8 +17,9 @@
namespace TNL {
namespace Solvers {
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter , typename CommunicatorType > class ProblemSetter,
typename ConfigTag >
+
class SolverInitiator : public Object
{
public:
diff --git a/src/TNL/Solvers/SolverInitiator_impl.h b/src/TNL/Solvers/SolverInitiator_impl.h
index 4736e585705f9c785dfe828cb823e8473d66db4b..e940379f5b6d39403c5ea8b2125a3156cd3d6467 100644
--- a/src/TNL/Solvers/SolverInitiator_impl.h
+++ b/src/TNL/Solvers/SolverInitiator_impl.h
@@ -23,23 +23,26 @@
#include <TNL/Solvers/SolverStarter.h>
#include <TNL/Meshes/DummyMesh.h>
+#include <TNL/Communicators/NoDistrCommunicator.h>
+#include <TNL/Communicators/MpiCommunicator.h>
+
namespace TNL {
namespace Solvers {
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
typename Real,
typename ConfigTag,
bool enabled = ConfigTagReal< ConfigTag, Real >::enabled >
class SolverInitiatorRealResolver {};
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
typename Real,
typename Device,
typename ConfigTag,
bool enabled = ConfigTagDevice< ConfigTag, Device >::enabled >
class SolverInitiatorDeviceResolver {};
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
typename Real,
typename Device,
typename Index,
@@ -47,16 +50,25 @@ template< template< typename Real, typename Device, typename Index, typename Mes
bool enabled = ConfigTagIndex< ConfigTag, Index >::enabled >
class SolverInitiatorIndexResolver {};
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
+ typename Real,
+ typename Device,
+ typename Index,
+ typename ConfigTag,
+ bool enabled = true >
+class CommunicatorTypeResolver {};
+
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
typename Real,
typename Device,
typename Index,
typename ConfigTag,
+ typename CommunicatorType,
bool enabled = ConfigTagMeshResolve< ConfigTag >::enabled >
class SolverInitiatorMeshResolver {};
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
typename ConfigTag >
bool SolverInitiator< ProblemSetter, ConfigTag > :: run( const Config::ParameterContainer& parameters )
{
@@ -73,8 +85,7 @@ bool SolverInitiator< ProblemSetter, ConfigTag > :: run( const Config::Parameter
return false;
};
-
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
typename Real,
typename ConfigTag >
class SolverInitiatorRealResolver< ProblemSetter, Real, ConfigTag, true >
@@ -97,7 +108,7 @@ class SolverInitiatorRealResolver< ProblemSetter, Real, ConfigTag, true >
}
};
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
typename Real,
typename ConfigTag >
class SolverInitiatorRealResolver< ProblemSetter, Real, ConfigTag, false >
@@ -110,8 +121,7 @@ class SolverInitiatorRealResolver< ProblemSetter, Real, ConfigTag, false >
}
};
-
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
typename Real,
typename Device,
typename ConfigTag >
@@ -134,7 +144,7 @@ class SolverInitiatorDeviceResolver< ProblemSetter, Real, Device, ConfigTag, tru
}
};
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
typename Real,
typename Device,
typename ConfigTag >
@@ -148,8 +158,7 @@ class SolverInitiatorDeviceResolver< ProblemSetter, Real, Device, ConfigTag, fal
}
};
-
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
typename Real,
typename Device,
typename Index,
@@ -164,7 +173,7 @@ class SolverInitiatorIndexResolver< ProblemSetter, Real, Device, Index, ConfigTa
}
};
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
typename Real,
typename Device,
typename Index,
@@ -174,17 +183,41 @@ class SolverInitiatorIndexResolver< ProblemSetter, Real, Device, Index, ConfigTa
public:
static bool run( const Config::ParameterContainer& parameters )
{
- return SolverInitiatorMeshResolver< ProblemSetter, Real, Device, Index, ConfigTag >::run( parameters );
+ return CommunicatorTypeResolver< ProblemSetter, Real, Device, Index, ConfigTag, true >::run( parameters );
}
};
-
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
typename Real,
typename Device,
typename Index,
typename ConfigTag >
-class SolverInitiatorMeshResolver< ProblemSetter, Real, Device, Index, ConfigTag, false >
+class CommunicatorTypeResolver< ProblemSetter, Real, Device, Index, ConfigTag, true >
+{
+ public:
+ static bool run( const Config::ParameterContainer& parameters )
+ {
+#ifdef HAVE_MPI
+ if(Communicators::MpiCommunicator::isDistributed())
+ {
+ bool ret=SolverInitiatorMeshResolver< ProblemSetter, Real, Device, Index, ConfigTag, Communicators::MpiCommunicator >::run( parameters );
+ Communicators::MpiCommunicator::Finalize();
+ return ret;
+ }
+ Communicators::MpiCommunicator::Finalize();
+#endif
+ return SolverInitiatorMeshResolver< ProblemSetter, Real, Device, Index, ConfigTag, Communicators::NoDistrCommunicator >::run( parameters );
+
+ }
+};
+
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
+ typename Real,
+ typename Device,
+ typename Index,
+ typename ConfigTag,
+ typename CommunicatorType >
+class SolverInitiatorMeshResolver< ProblemSetter, Real, Device, Index, ConfigTag, CommunicatorType, false >
{
public:
static bool run( const Config::ParameterContainer& parameters )
@@ -194,20 +227,21 @@ class SolverInitiatorMeshResolver< ProblemSetter, Real, Device, Index, ConfigTag
Index,
Meshes::DummyMesh< Real, Device, Index >,
ConfigTag,
- SolverStarter< ConfigTag > >::template run< Real, Device, Index, ConfigTag >( parameters );
+ SolverStarter< ConfigTag >, CommunicatorType >::template run< Real, Device, Index, ConfigTag >( parameters );
}
};
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename ConfigTag, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
typename Real,
typename Device,
typename Index,
- typename ConfigTag >
-class SolverInitiatorMeshResolver< ProblemSetter, Real, Device, Index, ConfigTag, true >
+ typename ConfigTag,
+ typename CommunicatorType >
+class SolverInitiatorMeshResolver< ProblemSetter, Real, Device, Index, ConfigTag,CommunicatorType, true >
{
// wrapper for MeshTypeResolver
template< typename MeshType >
- using ProblemSetterWrapper = ProblemSetter< Real, Device, Index, MeshType, ConfigTag, SolverStarter< ConfigTag > >;
+ using ProblemSetterWrapper = ProblemSetter< Real, Device, Index, MeshType, ConfigTag, SolverStarter< ConfigTag >, CommunicatorType >;
public:
static bool run( const Config::ParameterContainer& parameters )
diff --git a/src/TNL/Solvers/Solver_impl.h b/src/TNL/Solvers/Solver_impl.h
index f7787359bf8b3beea531fa4e65a64e4d73a8a69a..c07d743708fcb7c943b7010f7a23c02f3cf66b28 100644
--- a/src/TNL/Solvers/Solver_impl.h
+++ b/src/TNL/Solvers/Solver_impl.h
@@ -14,11 +14,13 @@
#include <TNL/Solvers/SolverStarter.h>
#include <TNL/Solvers/SolverConfig.h>
#include <TNL/Devices/Cuda.h>
+#include <TNL/Communicators/NoDistrCommunicator.h>
+#include <TNL/Communicators/MpiCommunicator.h>
namespace TNL {
namespace Solvers {
-template< template< typename Real, typename Device, typename Index, typename MeshType, typename MeshConfig, typename SolverStarter > class ProblemSetter,
+template< template< typename Real, typename Device, typename Index, typename MeshType, typename MeshConfig, typename SolverStarter, typename CommunicatorType > class ProblemSetter,
template< typename MeshConfig > class ProblemConfig,
typename MeshConfig >
bool
@@ -33,11 +35,19 @@ run( int argc, char* argv[] )
Devices::Host::configSetup( configDescription );
Devices::Cuda::configSetup( configDescription );
+ //iniicialization needs argc and argc-> needs to be close to main
+ Communicators::NoDistrCommunicator::Init(argc,argv, true);
+#ifdef HAVE_MPI
+ Communicators::MpiCommunicator::Init(argc,argv, true);
+#endif
+
if( ! parseCommandLine( argc, argv, configDescription, parameters ) )
return false;
SolverInitiator< ProblemSetter, MeshConfig > solverInitiator;
- return solverInitiator.run( parameters );
+ bool ret= solverInitiator.run( parameters );
+
+ return ret;
};
} // namespace Solvers
diff --git a/src/TNL/String.cpp b/src/TNL/String.cpp
index 34c4e3822b940b520861052af809c6665bba6a13..3f733526d935e12f0d6534de22e9d06cdbe91058 100644
--- a/src/TNL/String.cpp
+++ b/src/TNL/String.cpp
@@ -15,7 +15,7 @@
#include <TNL/Containers/List.h>
#include <TNL/File.h>
#include <TNL/Math.h>
-#ifdef HAVE_MPI
+#ifdef USE_MPI
#include <mpi.h>
#endif
@@ -390,9 +390,9 @@ bool String::load( File& file )
return true;
}
-void String::MPIBcast( int root, MPI_Comm comm )
+/*void String :: MPIBcast( int root, MPI_Comm comm )
{
-#ifdef HAVE_MPI
+#ifdef USE_MPI
dbgFunctionName( "mString", "MPIBcast" );
int iproc;
MPI_Comm_rank( MPI_COMM_WORLD, &iproc );
@@ -416,8 +416,8 @@ void String::MPIBcast( int root, MPI_Comm comm )
dbgExpr( string );
#endif
}
-
-bool String::getLine( std::istream& stream )
+*/
+bool String :: getLine( std::istream& stream )
{
std::string str;
getline( stream, str );
diff --git a/src/TNL/String.h b/src/TNL/String.h
index 5c012fca03f5970beab192622bff910fcdcd59a6..0868ac44a46872681367ed48e48f24fdc0a15f59 100644
--- a/src/TNL/String.h
+++ b/src/TNL/String.h
@@ -12,8 +12,6 @@
#include <iostream>
#include <sstream>
-#include <TNL/mpi-supp.h>
-
namespace TNL {
@@ -22,6 +20,8 @@ namespace Containers {
template< class T > class List;
}
+class String;
+
template< typename T >
String convertToString( const T& value );
@@ -146,7 +146,7 @@ public:
bool load( File& file );
//! Broadcast to other nodes in MPI cluster
- void MPIBcast( int root, MPI_Comm mpi_comm = MPI_COMM_WORLD );
+// void MPIBcast( int root, MPI_Comm mpi_comm = MPI_COMM_WORLD );
//! Read one line from given stream.
bool getLine( std::istream& stream );
diff --git a/src/TNL/mpi-supp.cpp b/src/TNL/mpi-supp.cpp
deleted file mode 100644
index d45bd1da83503f7921c11fea44da71ea6d1aef72..0000000000000000000000000000000000000000
--- a/src/TNL/mpi-supp.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/***************************************************************************
- mpi-supp.cpp - description
- -------------------
- begin : 2007/06/21
- copyright : (C) 2007 by Tomas Oberhuber
- email : tomas.oberhuber@fjfi.cvut.cz
- ***************************************************************************/
-
-/* See Copyright Notice in tnl/Copyright */
-
-#include <TNL/mpi-supp.h>
-
-namespace TNL {
-
-void MPIInit( int* argc, char** argv[] )
-{
-#ifdef HAVE_MPI
- MPI_Init( argc, argv );
-#endif
-}
-
-void MPIFinalize()
-{
-#ifdef HAVE_MPI
- MPI_Finalize();
-#endif
-}
-
-bool HaveMPI()
-{
-#ifdef HAVE_MPI
- return true;
-#else
- return false;
-#endif
-}
-
-int MPIGetRank( MPI_Comm comm )
-{
-#ifdef HAVE_MPI
- int rank;
- MPI_Comm_rank( MPI_COMM_WORLD, &rank );
- return rank;
-#else
- return 0;
-#endif
-}
-
-int MPIGetSize( MPI_Comm comm )
-{
-#ifdef HAVE_MPI
- int size;
- MPI_Comm_size( comm, &size );
- return size;
-#else
- return 0;
-#endif
-}
-
-
-void MPIBarrier( MPI_Comm comm )
-{
-#ifdef HAVE_MPI
- MPI_Barrier( comm );
-#endif
-}
-
-} // namespace TNL
-
diff --git a/src/TNL/mpi-supp.h b/src/TNL/mpi-supp.h
deleted file mode 100644
index 398fe5f793bedee2cce7520091d730131950a61a..0000000000000000000000000000000000000000
--- a/src/TNL/mpi-supp.h
+++ /dev/null
@@ -1,168 +0,0 @@
-/***************************************************************************
- mpi-supp.h - description
- -------------------
- begin : 2005/04/23
- copyright : (C) 2005 by Tomas Oberhuber
- email : tomas.oberhuber@fjfi.cvut.cz
- ***************************************************************************/
-
-/* See Copyright Notice in tnl/Copyright */
-
-#pragma once
-
-#include <iostream>
-#include <cstdlib>
-
-#ifdef HAVE_MPI
- #include <mpi.h>
-#else
- typedef int MPI_Comm;
- typedef int MPI_Op;
- #define MPI_COMM_WORLD 0
- #define MPI_MAX 0
- #define MPI_MIN 0
- #define MPI_SUM 0
-#endif
-
-namespace TNL {
-
-class String;
-
-#ifdef HAVE_MPI
-inline MPI_Datatype MPIDataType( const signed char ) { return MPI_CHAR; };
-inline MPI_Datatype MPIDataType( const signed short int ) { return MPI_SHORT; };
-inline MPI_Datatype MPIDataType( const signed int ) { return MPI_INT; };
-inline MPI_Datatype MPIDataType( const signed long int ) { return MPI_LONG; };
-inline MPI_Datatype MPIDataType( const unsigned char ) { return MPI_UNSIGNED_CHAR; };
-inline MPI_Datatype MPIDataType( const unsigned short int ) { return MPI_UNSIGNED_SHORT; };
-inline MPI_Datatype MPIDataType( const unsigned int ) { return MPI_UNSIGNED; };
-inline MPI_Datatype MPIDataType( const unsigned long int ) { return MPI_UNSIGNED_LONG; };
-inline MPI_Datatype MPIDataType( const float& ) { return MPI_FLOAT; };
-inline MPI_Datatype MPIDataType( const double& ) { return MPI_DOUBLE; };
-inline MPI_Datatype MPIDataType( const long double& ) { return MPI_LONG_DOUBLE; };
-#endif
-
-
-void MPIInit( int* argc, char** argv[] );
-
-void MPIFinalize();
-
-bool HaveMPI();
-
-int MPIGetRank( MPI_Comm comm = MPI_COMM_WORLD );
-
-int MPIGetSize( MPI_Comm comm = MPI_COMM_WORLD );
-
-void MPIBarrier( MPI_Comm comm = MPI_COMM_WORLD );
-
-#ifdef HAVE_MPI
-template< class T > void MPISend( const T& data,
- int count,
- int dest,
- MPI_Comm comm = MPI_COMM_WORLD )
-{
- MPI_Send( &data, count, MPIDataType( data ), dest, 0, comm );
-};
-#else
-template< class T > void MPISend( const T&,
- int,
- int,
- MPI_Comm )
-{
-};
-#endif
-
-#ifdef HAVE_MPI
-template< class T > void MPIRecv( T& data,
- int count,
- int src,
- MPI_Comm comm = MPI_COMM_WORLD )
-{
- MPI_Status stat;
- MPI_Recv( data, count, MPIDataType( data ), src, 0, comm, &stat );
-};
-#else
-template< class T > void MPIRecv( T&,
- int,
- int,
- MPI_Comm = MPI_COMM_WORLD )
-{};
-#endif
-
-#ifdef HAVE_MPI
-template< class T > void MPIBcast( T& data,
- int count,
- int root,
- MPI_Comm comm = MPI_COMM_WORLD )
-{
- MPI_Bcast( &data, count, MPIDataType( data ), root, comm );
-};
-
-inline void MPIBcast( String& data, intstd::cout, int root, MPI_Comm comm = MPI_COMM_WORLD )
-{
- std::cerr << "Call method MPIBcast of mString instead of function MPIBcast( mString&, ... ) " << std::endl;
- abort();
-}
-#else
-template< class T > void MPIBcast( T&,
- int,
- int,
- MPI_Comm = MPI_COMM_WORLD )
-{
-}
-#endif
-
-#ifdef HAVE_MPI
-template< typename T > void MPIReduce( T& data,
- T& reduced_data,
- int count,
- MPI_Op op,
- int root,
- MPI_Comm comm )
-{
- MPI_Reduce( &data,
- &reduced_data,
- count,
- MPIDataType( data ),
- op,
- root,
- comm );
-};
-#else
-template< typename T > void MPIReduce( T& data,
- T& reduced_data,
- int,
- MPI_Op,
- int,
- MPI_Comm )
-{
- reduced_data = data;
-};
-#endif
-
-#ifdef HAVE_MPI
-template< typename T > void MPIAllreduce( T& data,
- T& reduced_data,
- int count,
- MPI_Op op,
- MPI_Comm comm )
-{
- MPI_Allreduce( &data,
- &reduced_data,
- count,
- MPIDataType( data ),
- op,
- comm );
-};
-#else
-template< typename T > void MPIAllreduce( T& data,
- T& reduced_data,
- int,
- MPI_Op,
- MPI_Comm )
-{
- reduced_data = data;
-};
-#endif
-
-} // namespace TNL
diff --git a/src/Tools/tnl-init.cpp b/src/Tools/tnl-init.cpp
index f42b9ad3c9aeb02690ef4f7def74acf6c2ce3087..49ee117eae5e262fac28686a445060d32ab1db6d 100644
--- a/src/Tools/tnl-init.cpp
+++ b/src/Tools/tnl-init.cpp
@@ -17,6 +17,10 @@
#include <TNL/Meshes/DummyMesh.h>
#include <TNL/Meshes/Grid.h>
+#include <TNL/Communicators/NoDistrCommunicator.h>
+#include <TNL/Communicators/MpiCommunicator.h>
+
+
using namespace TNL;
void setupConfig( Config::ConfigDescription& config )
@@ -43,12 +47,21 @@ void setupConfig( Config::ConfigDescription& config )
Functions::TestFunction< 1 >::configSetup( config );
}
+
+
int main( int argc, char* argv[] )
{
+
Config::ParameterContainer parameters;
Config::ConfigDescription conf_desc;
setupConfig( conf_desc );
+
+ //iniicialization needs argc and argc-> needs to be close to main
+ Communicators::NoDistrCommunicator::Init(argc,argv, true);
+#ifdef HAVE_MPI
+ Communicators::MpiCommunicator::Init(argc,argv,true);
+#endif
if( ! parseCommandLine( argc, argv, conf_desc, parameters ) )
return EXIT_FAILURE;
@@ -69,5 +82,6 @@ int main( int argc, char* argv[] )
}
if( ! resolveMeshType( parsedMeshType, parameters ) )
return EXIT_FAILURE;
+
return EXIT_SUCCESS;
}
diff --git a/src/Tools/tnl-init.h b/src/Tools/tnl-init.h
index 355c553f02695d28f45c3994ef80891ead05b4cd..67fbba0a5ec58093dcad37d2805e946cbd95446c 100644
--- a/src/Tools/tnl-init.h
+++ b/src/Tools/tnl-init.h
@@ -19,20 +19,47 @@
#include <TNL/FileName.h>
#include <TNL/Functions/MeshFunction.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedGridIO.h>
+
+#include <TNL/Communicators/NoDistrCommunicator.h>
+#include <TNL/Communicators/MpiCommunicator.h>
+
using namespace TNL;
template< typename MeshType,
typename RealType,
+ typename CommunicatorType,
int xDiff,
int yDiff,
int zDiff >
bool renderFunction( const Config::ParameterContainer& parameters )
{
+
+ Meshes::DistributedMeshes::DistributedMesh<MeshType> distributedMesh;
SharedPointer< MeshType > meshPointer;
- String meshFile = parameters.getParameter< String >( "mesh" );
- std::cout << "+ -> Loading mesh from " << meshFile << " ... " << std::endl;
- if( ! meshPointer->load( meshFile ) )
- return false;
+ MeshType globalMesh;
+
+ if(CommunicatorType::isDistributed())
+ {
+ //suppose global mesh loaded from single file
+ String meshFile = parameters.getParameter< String >( "mesh" );
+ std::cout << "+ -> Loading mesh from " << meshFile << " ... " << std::endl;
+ if( ! globalMesh.load( meshFile ) )
+ return false;
+
+ typename Meshes::DistributedMeshes::DistributedMesh<MeshType>::CoordinatesType overlap;
+ overlap.setValue(1);
+ distributedMesh.template setGlobalGrid<CommunicatorType>(globalMesh,overlap);
+ distributedMesh.SetupGrid(*meshPointer);
+ }
+ else
+ {
+ String meshFile = parameters.getParameter< String >( "mesh" );
+ std::cout << "+ -> Loading mesh from " << meshFile << " ... " << std::endl;
+ if( ! meshPointer->load( meshFile ) )
+ return false;
+ }
typedef Functions::TestFunction< MeshType::getMeshDimension(), RealType > FunctionType;
typedef SharedPointer< FunctionType, typename MeshType::DeviceType > FunctionPointer;
@@ -87,14 +114,46 @@ bool renderFunction( const Config::ParameterContainer& parameters )
}
else
std::cout << "+ -> Writing the function to " << outputFile << " ... " << std::endl;
- if( ! meshFunction->save( outputFile) )
+
+ if(CommunicatorType::isDistributed())
+ {
+ Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType> ::save(outputFile, *meshFunction );
+ }
+ else
+ {
+ if( ! meshFunction->save( outputFile) )
return false;
+ }
+
time += tau;
step ++;
}
return true;
}
+template< typename MeshType,
+ typename RealType,
+ int xDiff,
+ int yDiff,
+ int zDiff >
+bool resolvCommunicator( const Config::ParameterContainer& parameters )
+{
+#ifdef HAVE_MPI
+ if(Communicators::MpiCommunicator::isDistributed())
+ {
+ Communicators::NoDistrCommunicator::Finalize();
+ bool ret=renderFunction<MeshType,RealType, Communicators::MpiCommunicator,xDiff,yDiff,zDiff>(parameters);
+ Communicators::MpiCommunicator::Finalize();
+ return ret;
+ }
+#endif
+
+ bool ret=renderFunction<MeshType,RealType, Communicators::NoDistrCommunicator,xDiff,yDiff,zDiff>(parameters);
+ Communicators::NoDistrCommunicator::Finalize();
+ return ret;
+
+}
+
template< typename MeshType,
typename RealType >
bool resolveDerivatives( const Config::ParameterContainer& parameters )
@@ -112,75 +171,75 @@ bool resolveDerivatives( const Config::ParameterContainer& parameters )
return false;
}
if( xDiff == 0 && yDiff == 0 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 0, 0, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 0, 0 >( parameters );
if( xDiff == 0 && yDiff == 0 && zDiff == 1 )
- return renderFunction< MeshType, RealType, 0, 0, 1 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 0, 1 >( parameters );
if( xDiff == 0 && yDiff == 0 && zDiff == 2 )
- return renderFunction< MeshType, RealType, 0, 0, 2 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 0, 2 >( parameters );
if( xDiff == 0 && yDiff == 0 && zDiff == 3 )
- return renderFunction< MeshType, RealType, 0, 0, 3 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 0, 3 >( parameters );
if( xDiff == 0 && yDiff == 0 && zDiff == 4 )
- return renderFunction< MeshType, RealType, 0, 0, 4 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 0, 4 >( parameters );
if( xDiff == 0 && yDiff == 1 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 0, 1, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 1, 0 >( parameters );
if( xDiff == 0 && yDiff == 1 && zDiff == 1 )
- return renderFunction< MeshType, RealType, 0, 1, 1 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 1, 1 >( parameters );
if( xDiff == 0 && yDiff == 1 && zDiff == 2 )
- return renderFunction< MeshType, RealType, 0, 1, 2 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 1, 2 >( parameters );
if( xDiff == 0 && yDiff == 1 && zDiff == 3 )
- return renderFunction< MeshType, RealType, 0, 1, 3 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 1, 3 >( parameters );
if( xDiff == 0 && yDiff == 2 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 0, 2, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 2, 0 >( parameters );
if( xDiff == 0 && yDiff == 2 && zDiff == 1 )
- return renderFunction< MeshType, RealType, 0, 2, 1 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 2, 1 >( parameters );
if( xDiff == 0 && yDiff == 2 && zDiff == 2 )
- return renderFunction< MeshType, RealType, 0, 2, 2 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 2, 2 >( parameters );
if( xDiff == 0 && yDiff == 3 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 0, 3, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 3, 0 >( parameters );
if( xDiff == 0 && yDiff == 3 && zDiff == 1 )
- return renderFunction< MeshType, RealType, 0, 3, 1 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 3, 1 >( parameters );
if( xDiff == 0 && yDiff == 4 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 0, 4, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 0, 4, 0 >( parameters );
if( xDiff == 1 && yDiff == 0 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 1, 0, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 1, 0, 0 >( parameters );
if( xDiff == 1 && yDiff == 0 && zDiff == 1 )
- return renderFunction< MeshType, RealType, 1, 0, 1 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 1, 0, 1 >( parameters );
if( xDiff == 1 && yDiff == 0 && zDiff == 2 )
- return renderFunction< MeshType, RealType, 1, 0, 2 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 1, 0, 2 >( parameters );
if( xDiff == 1 && yDiff == 0 && zDiff == 3 )
- return renderFunction< MeshType, RealType, 1, 0, 3 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 1, 0, 3 >( parameters );
if( xDiff == 1 && yDiff == 1 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 1, 1, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 1, 1, 0 >( parameters );
if( xDiff == 1 && yDiff == 1 && zDiff == 1 )
- return renderFunction< MeshType, RealType, 1, 1, 1 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 1, 1, 1 >( parameters );
if( xDiff == 1 && yDiff == 1 && zDiff == 2 )
- return renderFunction< MeshType, RealType, 1, 1, 2 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 1, 1, 2 >( parameters );
if( xDiff == 1 && yDiff == 2 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 1, 2, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 1, 2, 0 >( parameters );
if( xDiff == 1 && yDiff == 2 && zDiff == 1 )
- return renderFunction< MeshType, RealType, 1, 2, 1 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 1, 2, 1 >( parameters );
if( xDiff == 1 && yDiff == 3 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 1, 3, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 1, 3, 0 >( parameters );
if( xDiff == 2 && yDiff == 0 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 2, 0, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 2, 0, 0 >( parameters );
if( xDiff == 2 && yDiff == 0 && zDiff == 1 )
- return renderFunction< MeshType, RealType, 2, 0, 1 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 2, 0, 1 >( parameters );
if( xDiff == 2 && yDiff == 0 && zDiff == 2 )
- return renderFunction< MeshType, RealType, 2, 0, 2 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 2, 0, 2 >( parameters );
if( xDiff == 2 && yDiff == 1 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 2, 1, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 2, 1, 0 >( parameters );
if( xDiff == 2 && yDiff == 1 && zDiff == 1 )
- return renderFunction< MeshType, RealType, 2, 1, 1 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 2, 1, 1 >( parameters );
if( xDiff == 2 && yDiff == 2 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 2, 2, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 2, 2, 0 >( parameters );
if( xDiff == 3 && yDiff == 0 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 3, 0, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 3, 0, 0 >( parameters );
if( xDiff == 3 && yDiff == 0 && zDiff == 1 )
- return renderFunction< MeshType, RealType, 3, 0, 1 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 3, 0, 1 >( parameters );
if( xDiff == 3 && yDiff == 1 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 3, 1, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 3, 1, 0 >( parameters );
if( xDiff == 4 && yDiff == 0 && zDiff == 0 )
- return renderFunction< MeshType, RealType, 4, 0, 0 >( parameters );
+ return resolvCommunicator< MeshType, RealType, 4, 0, 0 >( parameters );
return false;
}
@@ -264,4 +323,6 @@ bool resolveMeshType( const Containers::List< String >& parsedMeshType,
return false;
}
+
+
#endif /* TNL_INIT_H_ */
diff --git a/src/Tools/tnl-mesh-converter.cpp b/src/Tools/tnl-mesh-converter.cpp
index 0bc4e050f9e1e2ec66f88695f08673e3937bbde8..dd7ec0b2049ed7421ff6ec72ad4aeece22ff0e34 100644
--- a/src/Tools/tnl-mesh-converter.cpp
+++ b/src/Tools/tnl-mesh-converter.cpp
@@ -13,6 +13,9 @@
#include <TNL/Meshes/Writers/VTKWriter.h>
#include <TNL/Meshes/Writers/NetgenWriter.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+#include <TNL/Communicators/NoDistrCommunicator.h>
+
using namespace TNL;
struct MeshConverterConfigTag {};
@@ -69,7 +72,8 @@ struct MeshConverter
static bool run( const String& inputFileName, const String& outputFileName, const String& outputFormat )
{
Mesh mesh;
- if( ! loadMesh( inputFileName, mesh ) ) {
+ Meshes::DistributedMeshes::DistributedMesh<Mesh> distributedMesh;
+ if( ! Meshes::loadMesh<Communicators::NoDistrCommunicator>( inputFileName, mesh, distributedMesh ) ) {
std::cerr << "Failed to load mesh from file '" << inputFileName << "'." << std::endl;
return false;
}
diff --git a/src/Tools/tnl-view.h b/src/Tools/tnl-view.h
index 42f4fd1b300cf7c738efdac37713a77534ce3611..ed81bdf0878f709ee47878b7b4cbfa0add139f7f 100644
--- a/src/Tools/tnl-view.h
+++ b/src/Tools/tnl-view.h
@@ -21,6 +21,9 @@
#include <TNL/Functions/MeshFunction.h>
#include <TNL/Functions/VectorField.h>
+#include <TNL/Communicators/NoDistrCommunicator.h>
+#include <TNL/Meshes/TypeResolver/TypeResolver.h>
+
using namespace TNL;
bool getOutputFileName( const String& inputFileName,
@@ -424,8 +427,11 @@ struct FilesProcessor
MeshPointer meshPointer;
if( meshFile != "" )
- if( ! loadMesh( meshFile, *meshPointer ) )
+ {
+ Meshes::DistributedMeshes::DistributedMesh<Mesh> distributedMesh;
+ if( ! Meshes::loadMesh<Communicators::NoDistrCommunicator>( meshFile, *meshPointer, distributedMesh ) )
return false;
+ }
bool checkOutputFile = parameters. getParameter< bool >( "check-output-file" );
Containers::List< String > inputFiles = parameters. getParameter< Containers::List< String > >( "input-files" );
diff --git a/src/UnitTests/CMakeLists.txt b/src/UnitTests/CMakeLists.txt
index fb90e6ee3d41fe06dd4e0724b2356a7f046bddad..3e17aad5d338f543c176d399e1df62b08f2fd1e8 100644
--- a/src/UnitTests/CMakeLists.txt
+++ b/src/UnitTests/CMakeLists.txt
@@ -2,6 +2,7 @@ if( WITH_TESTS STREQUAL "yes" )
ADD_SUBDIRECTORY( Containers )
ADD_SUBDIRECTORY( Matrices )
+ADD_SUBDIRECTORY( Mpi )
ADD_SUBDIRECTORY( Meshes )
ADD_EXECUTABLE( UniquePointerTest UniquePointerTest.cpp )
@@ -35,10 +36,16 @@ TARGET_LINK_LIBRARIES( ObjectTest
${GTEST_BOTH_LIBRARIES}
tnl )
+ADD_EXECUTABLE( SaveAndLoadMeshfunctionTest SaveAndLoadMeshfunctionTest.cpp )
+TARGET_COMPILE_OPTIONS( SaveAndLoadMeshfunctionTest PRIVATE ${CXX_TESTS_FLAGS} )
+TARGET_LINK_LIBRARIES( SaveAndLoadMeshfunctionTest
+ ${GTEST_BOTH_LIBRARIES}
+ tnl )
ADD_TEST( FileTest ${EXECUTABLE_OUTPUT_PATH}/FileTest${CMAKE_EXECUTABLE_SUFFIX} )
ADD_TEST( StringTest ${EXECUTABLE_OUTPUT_PATH}/StringTest${CMAKE_EXECUTABLE_SUFFIX} )
ADD_TEST( ObjectTest ${EXECUTABLE_OUTPUT_PATH}/ObjectTest${CMAKE_EXECUTABLE_SUFFIX} )
ADD_TEST( UniquePointerTest ${EXECUTABLE_OUTPUT_PATH}/UniquePointerTest${CMAKE_EXECUTABLE_SUFFIX} )
+ADD_TEST( SaveAndLoadMeshfunctionTest ${EXECUTABLE_OUTPUT_PATH}/SaveAndLoadMeshfunctionTest${CMAKE_EXECUTABLE_SUFFIX} )
endif( WITH_TESTS STREQUAL "yes" )
diff --git a/src/UnitTests/Meshes/MeshReaderTest.h b/src/UnitTests/Meshes/MeshReaderTest.h
index 64fe92cc190656001ed066fa46d4a2e917bd4239..17bc6a3178fbd5395a7964150bc67ebf89373516 100644
--- a/src/UnitTests/Meshes/MeshReaderTest.h
+++ b/src/UnitTests/Meshes/MeshReaderTest.h
@@ -2,6 +2,7 @@
#include <TNL/Meshes/DefaultConfig.h>
#include <TNL/Meshes/BuildConfigTags.h>
#include <TNL/Meshes/TypeResolver/TypeResolver.h>
+#include <TNL/Communicators/NoDistrCommunicator.h>
#ifdef HAVE_VTK
#include <TNL/Meshes/Readers/VTKReader_libvtk.h>
@@ -138,7 +139,8 @@ public:
timer.start();
MeshType mesh;
- if( ! loadMesh( fileName, mesh ) )
+ Meshes::DistributedMeshes::DistributedMesh<MeshType> distributedMesh;
+ if( ! loadMesh<Communicators::NoDistrCommunicator>( fileName, mesh, distributedMesh ) )
return false;
timer.stop();
diff --git a/src/UnitTests/Mpi/CMakeLists.txt b/src/UnitTests/Mpi/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5dca444aa40876bd5df0286b3393b7934168149b
--- /dev/null
+++ b/src/UnitTests/Mpi/CMakeLists.txt
@@ -0,0 +1,54 @@
+if( ${CXX_COMPILER_NAME} STREQUAL "mpic++" )
+ADD_EXECUTABLE( DistributedGridTest_1D DistributedGridTest_1D.cpp )
+ TARGET_COMPILE_OPTIONS( DistributedGridTest_1D PRIVATE ${CXX_TESTS_FLAGS} )
+ TARGET_LINK_LIBRARIES( DistributedGridTest_1D
+ ${GTEST_BOTH_LIBRARIES}
+ tnl )
+
+ADD_EXECUTABLE( DistributedGridTest_2D DistributedGridTest_2D.cpp )
+ TARGET_COMPILE_OPTIONS( DistributedGridTest_2D PRIVATE ${CXX_TESTS_FLAGS} )
+ TARGET_LINK_LIBRARIES( DistributedGridTest_2D
+ ${GTEST_BOTH_LIBRARIES}
+ tnl)
+
+ADD_EXECUTABLE( DistributedGridTest_3D DistributedGridTest_3D.cpp )
+ TARGET_COMPILE_OPTIONS( DistributedGridTest_3D PRIVATE ${CXX_TESTS_FLAGS} )
+ TARGET_LINK_LIBRARIES( DistributedGridTest_3D
+ ${GTEST_BOTH_LIBRARIES}
+ tnl )
+
+ADD_EXECUTABLE( CopyEntitesTest CopyEntitiesTest.cpp )
+ TARGET_COMPILE_OPTIONS( CopyEntitesTest PRIVATE ${CXX_TESTS_FLAGS} )
+ TARGET_LINK_LIBRARIES( CopyEntitesTest
+ ${GTEST_BOTH_LIBRARIES}
+ tnl )
+
+ADD_EXECUTABLE( DistributedGridIOTest DistributedGridIOTest.cpp )
+ TARGET_COMPILE_OPTIONS( DistributedGridIOTest PRIVATE ${CXX_TESTS_FLAGS} )
+ TARGET_LINK_LIBRARIES( DistributedGridIOTest
+ ${GTEST_BOTH_LIBRARIES}
+ tnl )
+
+ADD_TEST( NAME CopyEntitesTest COMMAND ${EXECUTABLE_OUTPUT_PATH}/CopyEntitesTest${CMAKE_EXECUTABLE_SUFFIX} )
+
+SET (mpi_test_parameters_1d -np 4 -H localhost:4 "${EXECUTABLE_OUTPUT_PATH}/DistributedGridTest_1D${CMAKE_EXECUTABLE_SUFFIX}")
+ADD_TEST( NAME DistributedGridTest_1D COMMAND "mpirun" ${mpi_test_parameters_1d})
+
+SET (mpi_test_parameters_2d -np 9 -H localhost:9 "${EXECUTABLE_OUTPUT_PATH}/DistributedGridTest_2D${CMAKE_EXECUTABLE_SUFFIX}")
+ADD_TEST( NAME DistributedGridTest_2D COMMAND "mpirun" ${mpi_test_parameters_2d})
+
+SET (mpi_test_parameters_3d -np 27 -H localhost:27 "${EXECUTABLE_OUTPUT_PATH}/DistributedGridTest_3D${CMAKE_EXECUTABLE_SUFFIX}")
+ADD_TEST( NAME DistributedGridTest_3D COMMAND "mpirun" ${mpi_test_parameters_3d})
+
+SET (mpi_test_parameters_IO -np 4 "${EXECUTABLE_OUTPUT_PATH}/DistributedGridIOTest${CMAKE_EXECUTABLE_SUFFIX}")
+ADD_TEST( NAME DistributedGridIOTest COMMAND "mpirun" ${mpi_test_parameters_IO})
+
+IF( BUILD_CUDA )
+ CUDA_ADD_EXECUTABLE( GPUDistributedGridIOTest GPUDistributedGridIOTest.cu OPTIONS ${CXX_TESTS_FLAGS})
+ TARGET_LINK_LIBRARIES( GPUDistributedGridIOTest
+ ${GTEST_BOTH_LIBRARIES}
+ tnl )
+ENDIF( BUILD_CUDA )
+
+endif()
+
diff --git a/src/UnitTests/Mpi/CopyEntitiesTest.cpp b/src/UnitTests/Mpi/CopyEntitiesTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ae0be78b99e8c46bb832dbad640c8bcc935a7a80
--- /dev/null
+++ b/src/UnitTests/Mpi/CopyEntitiesTest.cpp
@@ -0,0 +1,237 @@
+/***************************************************************************
+ CopyEntitiesTest.cpp - description
+ -------------------
+ begin : Aug 30, 2017
+ copyright : (C) 2017 by Tomas Oberhuber et al.
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+#include <TNL/Meshes/DistributedMeshes/CopyEntitiesHelper.h>
+#include <TNL/Functions/MeshFunction.h>
+
+
+#ifdef HAVE_GTEST
+#include <gtest/gtest.h>
+
+#include "Functions.h"
+
+using namespace TNL::Containers;
+using namespace TNL::Meshes;
+using namespace TNL::Functions;
+using namespace TNL::Devices;
+using namespace TNL::Meshes::DistributedMeshes;
+
+//================================TEST RESULT============================================
+template <typename MeshFunctionType,
+ int dim=MeshFunctionType::getMeshDimension()>
+class TestMovedMeshfunction
+{
+ public:
+ static void Test(MeshFunctionType &meshFunction)
+ {};
+};
+
+template <typename MeshFunctionType>
+class TestMovedMeshfunction<MeshFunctionType,1>
+{
+ public:
+ static void Test(MeshFunctionType &meshFunction)
+ {
+
+ for(int i=0;i<meshFunction.getData().getSize();i++)
+ EXPECT_EQ( meshFunction.getData()[i], i+1) << "Copy entities Failed for: "<< i;
+ }
+};
+
+template <typename MeshFunctionType>
+class TestMovedMeshfunction<MeshFunctionType,2>
+{
+ public:
+ static void Test(MeshFunctionType &meshFunction)
+ {
+ typename MeshFunctionType::MeshType::Cell entity(meshFunction.getMesh());
+ auto size=meshFunction.getMesh().getDimensions();
+ for(int j=0;j<size.y();j++)
+ for(int i=0;i<size.x();i++)
+ {
+ entity.getCoordinates().x()=i;
+ entity.getCoordinates().y()=j;
+ entity.refresh();
+ EXPECT_EQ( meshFunction.getData()[entity.getIndex()], 10*(j+1)+(i+1)) << "Copy entities Failed for: "<< i;
+ }
+ };
+};
+
+template <typename MeshFunctionType>
+class TestMovedMeshfunction<MeshFunctionType,3>
+{
+ public:
+ static void Test(MeshFunctionType &meshFunction)
+ {
+ typename MeshFunctionType::MeshType::Cell entity(meshFunction.getMesh());
+ auto size=meshFunction.getMesh().getDimensions();
+ for(int k=0;k<size.z();k++)
+ for(int j=0;j<size.y();j++)
+ for(int i=0;i<size.x();i++)
+ {
+ entity.getCoordinates().x()=i;
+ entity.getCoordinates().y()=j;
+ entity.getCoordinates().z()=k;
+ entity.refresh();
+ EXPECT_EQ( meshFunction.getData()[entity.getIndex()], 100*(k+1)+10*(j+1)+(i+1)) << "Copy entities Failed for: "<< i;
+ }
+ };
+};
+
+//================================SET INPUT============================================
+
+template <typename MeshFunctionType,
+ int dim=MeshFunctionType::getMeshDimension()>
+class EvalMeshFunction
+{
+ public:
+ static void Eval(MeshFunctionType &meshFunction)
+ {};
+};
+
+template <typename MeshFunctionType>
+class EvalMeshFunction<MeshFunctionType,1>
+{
+ public:
+ static void Eval(MeshFunctionType &meshFunction)
+ {
+ for(int i=0;i<meshFunction.getData().getSize();i++)
+ meshFunction.getData()[i]=i;
+ }
+};
+
+template <typename MeshFunctionType>
+class EvalMeshFunction<MeshFunctionType,2>
+{
+ public:
+ static void Eval(MeshFunctionType &meshFunction)
+ {
+ typename MeshFunctionType::MeshType::Cell entity(meshFunction.getMesh());
+ auto size=meshFunction.getMesh().getDimensions();
+ for(int j=0;j<size.y();j++)
+ for(int i=0;i<size.x();i++)
+ {
+ entity.getCoordinates().x()=i;
+ entity.getCoordinates().y()=j;
+ entity.refresh();
+ meshFunction.getData()[entity.getIndex()]= 10*j+i;
+ }
+ };
+};
+
+template <typename MeshFunctionType>
+class EvalMeshFunction<MeshFunctionType,3>
+{
+ public:
+ static void Eval(MeshFunctionType &meshFunction)
+ {
+ typename MeshFunctionType::MeshType::Cell entity(meshFunction.getMesh());
+ auto size=meshFunction.getMesh().getDimensions();
+ for(int k=0;k<size.z();k++)
+ for(int j=0;j<size.y();j++)
+ for(int i=0;i<size.x();i++)
+ {
+ entity.getCoordinates().x()=i;
+ entity.getCoordinates().y()=j;
+ entity.getCoordinates().z()=k;
+ entity.refresh();
+ meshFunction.getData()[entity.getIndex()]=100*k+10*j+i;
+ }
+ };
+};
+
+
+//=====================================TEST CLASS==============================================
+
+template <int dim>
+class TestCopyEntities
+{
+ public:
+ static void Test()
+ {
+ typedef Grid<dim,double,Host,int> MeshType;
+ typedef MeshFunction<MeshType> MeshFunctionType;
+ typedef Vector<double,Host,int> DofType;
+
+ typedef typename MeshType::PointType PointType;
+ typedef typename MeshType::CoordinatesType CoordinatesType;
+ typedef typename MeshType::Cell Cell;
+
+ PointType origin;
+ PointType proportions;
+ SharedPointer<MeshType> gridptr;
+
+ origin.setValue(-0.5);
+ proportions.setValue(10);
+
+ gridptr->setDimensions(proportions);
+ gridptr->setDomain(origin,proportions);
+
+ DofType inputDof(gridptr-> template getEntitiesCount< Cell >());
+
+ MeshFunctionType inputMeshFunction;
+ inputMeshFunction.bind(gridptr,inputDof);
+
+ EvalMeshFunction<MeshFunctionType> :: Eval(inputMeshFunction);
+
+
+ PointType originOut;
+ PointType proportionsOut;
+ SharedPointer<MeshType> gridOutPtr;
+
+ originOut.setValue(0.5);
+ proportionsOut.setValue(8);
+ gridOutPtr->setDimensions(proportionsOut);
+ gridOutPtr->setDomain(originOut,proportionsOut);
+ DofType outDof(gridOutPtr-> template getEntitiesCount< Cell >());
+
+ MeshFunctionType outputMeshFunction;
+ outputMeshFunction.bind(gridOutPtr,outDof);
+
+ CoordinatesType zero;
+ zero.setValue(0);
+ CoordinatesType begin;
+ begin.setValue(1);
+ CoordinatesType size;
+ size.setValue(8);
+
+ CopyEntitiesHelper< MeshFunctionType >::Copy(inputMeshFunction,outputMeshFunction, begin,zero, size);
+
+ TestMovedMeshfunction<MeshFunctionType>::Test(outputMeshFunction);
+ };
+};
+
+TEST( CopyEntitiesTest, 1D )
+{
+ TestCopyEntities<1>::Test();
+}
+
+/*TEST( CopyEntitiesTest, 2D )
+{
+ TestCopyEntities<2>::Test();
+}
+
+TEST( CopyEntitiesTest, 3D )
+{
+ TestCopyEntities<3>::Test();
+}*/
+
+
+#endif
+
+#include "../GtestMissingError.h"
+int main( int argc, char* argv[] )
+{
+#ifdef HAVE_GTEST
+ ::testing::InitGoogleTest( &argc, argv );
+ return RUN_ALL_TESTS();
+#else
+ throw GtestMissingError();
+#endif
+}
+
diff --git a/src/UnitTests/Mpi/DistributedGridIOTest.cpp b/src/UnitTests/Mpi/DistributedGridIOTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b0f51781517e000ee4983b6cefebb4e4e1479f00
--- /dev/null
+++ b/src/UnitTests/Mpi/DistributedGridIOTest.cpp
@@ -0,0 +1,110 @@
+
+#ifdef HAVE_GTEST
+
+#include <gtest/gtest.h>
+
+#ifdef HAVE_MPI
+
+#include "DistributedGridIOTest.h"
+
+TEST( DistributedGridIO, Save_1D )
+{
+ TestDistributedGridIO<1,Host>::TestSave();
+}
+
+TEST( DistributedGridIO, Save_2D )
+{
+ TestDistributedGridIO<2,Host>::TestSave();
+}
+
+TEST( DistributedGridIO, Save_3D )
+{
+ TestDistributedGridIO<3,Host>::TestSave();
+}
+
+TEST( DistributedGridIO, Load_1D )
+{
+ TestDistributedGridIO<1,Host>::TestLoad();
+}
+
+TEST( DistributedGridIO, Load_2D )
+{
+ TestDistributedGridIO<2,Host>::TestLoad();
+}
+
+TEST( DistributedGridIO, Load_3D )
+{
+ TestDistributedGridIO<3,Host>::TestLoad();
+}
+
+#else
+TEST(NoMPI, NoTest)
+{
+ ASSERT_TRUE(true) << ":-(";
+}
+#endif
+
+#endif
+
+#if (defined(HAVE_GTEST) && defined(HAVE_MPI))
+#include <sstream>
+
+ class MinimalistBuffredPrinter : public ::testing::EmptyTestEventListener {
+
+ private:
+ std::stringstream sout;
+
+ public:
+
+ // Called before a test starts.
+ virtual void OnTestStart(const ::testing::TestInfo& test_info) {
+ sout<< test_info.test_case_name() <<"." << test_info.name() << " Start." <<std::endl;
+ }
+
+ // Called after a failed assertion or a SUCCEED() invocation.
+ virtual void OnTestPartResult(
+ const ::testing::TestPartResult& test_part_result) {
+ sout << (test_part_result.failed() ? "====Failure=== " : "===Success=== ")
+ << test_part_result.file_name() << " "
+ << test_part_result.line_number() <<std::endl
+ << test_part_result.summary() <<std::endl;
+ }
+
+ // Called after a test ends.
+ virtual void OnTestEnd(const ::testing::TestInfo& test_info)
+ {
+ int rank=CommunicatorType::GetRank();
+ sout<< test_info.test_case_name() <<"." << test_info.name() << " End." <<std::endl;
+ std::cout << rank << ":" << std::endl << sout.str()<< std::endl;
+ sout.str( std::string() );
+ sout.clear();
+ }
+ };
+#endif
+
+#include "../GtestMissingError.h"
+int main( int argc, char* argv[] )
+{
+#ifdef HAVE_GTEST
+ ::testing::InitGoogleTest( &argc, argv );
+
+ #ifdef HAVE_MPI
+ ::testing::TestEventListeners& listeners =
+ ::testing::UnitTest::GetInstance()->listeners();
+
+ delete listeners.Release(listeners.default_result_printer());
+ listeners.Append(new MinimalistBuffredPrinter);
+
+ CommunicatorType::Init(argc,argv, false);
+ #endif
+ int result= RUN_ALL_TESTS();
+
+ #ifdef HAVE_MPI
+ CommunicatorType::Finalize();
+ #endif
+ return result;
+#else
+
+ throw GtestMissingError();
+#endif
+}
diff --git a/src/UnitTests/Mpi/DistributedGridIOTest.h b/src/UnitTests/Mpi/DistributedGridIOTest.h
new file mode 100644
index 0000000000000000000000000000000000000000..1a2874665eb55bcf29fb67aa863b2b461356b5bc
--- /dev/null
+++ b/src/UnitTests/Mpi/DistributedGridIOTest.h
@@ -0,0 +1,364 @@
+/***************************************************************************
+ CopyEntitiesTest.cpp - description
+ -------------------
+ begin : Nov 1, 2017
+ copyright : (C) 2017 by Tomas Oberhuber et al.
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+#include <TNL/Communicators/MpiCommunicator.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+#include <TNL/Functions/MeshFunction.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedGridIO.h>
+
+
+#include "Functions.h"
+
+using namespace TNL::Containers;
+using namespace TNL::Meshes;
+using namespace TNL::Functions;
+using namespace TNL::Devices;
+using namespace TNL::Communicators;
+using namespace TNL::Meshes::DistributedMeshes;
+
+
+//================Parameters===================================
+template <int dim, typename Device>
+class ParameterProvider
+{
+ public:
+
+ typedef Grid<dim,double,Device,int> MeshType;
+ typedef typename MeshType::CoordinatesType CoordinatesType;
+ typedef typename MeshType::PointType PointType;
+
+ PointType getOrigin(int rank)
+ {
+ };
+
+ PointType getProportions(int rank)
+ {
+ };
+
+ int* getDistr(void)
+ {
+ return NULL;
+ };
+};
+
+template<typename Device>
+class ParameterProvider<1,Device>
+{
+ public:
+
+ int distr[1];
+
+ typedef Grid<1,double,Device,int> MeshType;
+ typedef typename MeshType::CoordinatesType CoordinatesType;
+ typedef typename MeshType::PointType PointType;
+
+ PointType getOrigin(int rank)
+ {
+ if(rank==0)
+ return PointType(-0.5);
+ if(rank==1)
+ return PointType(4.5);
+ if(rank==2)
+ return PointType(9.5);
+ if(rank==3)
+ return PointType(14.5);
+
+ return PointType(0);
+ };
+
+ PointType getProportions(int rank)
+ {
+ if(rank==0)
+ return PointType(5);
+ if(rank==1)
+ return PointType(5);
+ if(rank==2)
+ return PointType(5);
+ if(rank==3)
+ return PointType(5);
+ return PointType(0);
+ };
+
+ int* getDistr()
+ {
+ distr[0]=4;
+ return distr;
+ };
+};
+
+template<typename Device>
+class ParameterProvider<2,Device>
+{
+ public:
+
+ int distr[2];
+
+ typedef Grid<2,double,Device,int> MeshType;
+ typedef typename MeshType::CoordinatesType CoordinatesType;
+ typedef typename MeshType::PointType PointType;
+
+ PointType getOrigin(int rank)
+ {
+ if(rank==0)
+ return PointType(-0.5,-0.5);
+ if(rank==1)
+ return PointType(9.5,-0.5);
+ if(rank==2)
+ return PointType(-0.5,9.5);
+ if(rank==3)
+ return PointType(9.5,9.5);
+
+ return PointType(0,0);
+ };
+
+ PointType getProportions(int rank)
+ {
+ if(rank==0)
+ return PointType(10,10);
+ if(rank==1)
+ return PointType(10,10);
+ if(rank==2)
+ return PointType(10,10);
+ if(rank==3)
+ return PointType(10,10);
+ return PointType(0,0);
+ };
+
+ int* getDistr()
+ {
+ distr[0]=2;
+ distr[1]=2;
+ return distr;
+ };
+};
+
+template<typename Device>
+class ParameterProvider<3,Device>
+{
+ public:
+
+ int distr[3];
+
+ typedef Grid<3,double,Device,int> MeshType;
+ typedef typename MeshType::CoordinatesType CoordinatesType;
+ typedef typename MeshType::PointType PointType;
+
+ PointType getOrigin(int rank)
+ {
+ if(rank==0)
+ return PointType(-0.5,-0.5,-0.5);
+ if(rank==1)
+ return PointType(9.5,-0.5,-0.5);
+ if(rank==2)
+ return PointType(-0.5,9.5,-0.5);
+ if(rank==3)
+ return PointType(9.5,9.5,-0.5);
+
+ return PointType(0,0,0);
+ };
+
+ PointType getProportions(int rank)
+ {
+ if(rank==0)
+ return PointType(10,10,20);
+ if(rank==1)
+ return PointType(10,10,20);
+ if(rank==2)
+ return PointType(10,10,20);
+ if(rank==3)
+ return PointType(10,10,20);
+ return PointType(0,0,0);
+ };
+
+ int* getDistr()
+ {
+ distr[0]=2;
+ distr[1]=2;
+ distr[2]=1;
+ return distr;
+ };
+};
+
+//------------------------------------------------------------------------------
+
+typedef MpiCommunicator CommunicatorType;
+
+template <int dim, typename Device>
+class TestDistributedGridIO{
+ public:
+
+ typedef Grid<dim,double,Device,int> MeshType;
+ typedef MeshFunction<MeshType> MeshFunctionType;
+ typedef Vector<double,Device,int> DofType;
+ typedef typename MeshType::Cell Cell;
+ typedef typename MeshType::IndexType IndexType;
+ typedef typename MeshType::PointType PointType;
+ typedef DistributedMesh<MeshType> DistributedGridType;
+
+ typedef typename DistributedGridType::CoordinatesType CoordinatesType;
+ typedef LinearFunction<double,dim> LinearFunctionType;
+
+ static void TestSave()
+ {
+ SharedPointer< LinearFunctionType, Device > linearFunctionPtr;
+ MeshFunctionEvaluator< MeshFunctionType, LinearFunctionType > linearFunctionEvaluator;
+
+ ParameterProvider<dim,Device> parametry;
+
+ //save distributed meshfunction into files
+ PointType globalOrigin;
+ globalOrigin.setValue(-0.5);
+
+ PointType globalProportions;
+ globalProportions.setValue(20);
+
+
+ MeshType globalGrid;
+ globalGrid.setDimensions(globalProportions);
+ globalGrid.setDomain(globalOrigin,globalProportions);
+
+ int *distr=parametry.getDistr();
+
+ CoordinatesType overlap;
+ overlap.setValue(1);
+ DistributedGridType distrgrid;
+ distrgrid.template setGlobalGrid<CommunicatorType>(globalGrid,overlap,distr);
+
+ std::cout << distrgrid.printProcessDistr() <<std::endl;
+
+ SharedPointer<MeshType> gridptr;
+ SharedPointer<MeshFunctionType> meshFunctionptr;
+ distrgrid.SetupGrid(*gridptr);
+
+ DofType dof(gridptr->template getEntitiesCount< Cell >());
+ dof.setValue(0);
+ meshFunctionptr->bind(gridptr,dof);
+
+ linearFunctionEvaluator.evaluateAllEntities(meshFunctionptr , linearFunctionPtr);
+
+ String FileName=String("/tmp/test-file.tnl");
+ DistributedGridIO<MeshFunctionType> ::save(FileName, *meshFunctionptr );
+
+
+ //create similar local mesh function and evaluate linear function on it
+ PointType localOrigin=parametry.getOrigin(CommunicatorType::GetRank());
+ PointType localProportions=parametry.getProportions(CommunicatorType::GetRank());;
+
+ SharedPointer<MeshType> localGridptr;
+ localGridptr->setDimensions(localProportions);
+ localGridptr->setDomain(localOrigin,localProportions);
+
+ DofType localDof(localGridptr->template getEntitiesCount< Cell >());
+
+ SharedPointer<MeshFunctionType> localMeshFunctionptr;
+ localMeshFunctionptr->bind(localGridptr,localDof);
+ linearFunctionEvaluator.evaluateAllEntities(localMeshFunctionptr , linearFunctionPtr);
+
+ //load other meshfunction on same localgrid from created file
+ SharedPointer<MeshType> loadGridptr;
+ loadGridptr->setDimensions(localProportions);
+ loadGridptr->setDomain(localOrigin,localProportions);
+
+ DofType loadDof(localGridptr->template getEntitiesCount< Cell >());
+ SharedPointer<MeshFunctionType> loadMeshFunctionptr;
+ loadMeshFunctionptr->bind(loadGridptr,loadDof);
+
+ loadDof.setValue(-1);
+
+ File file;
+ file.open( FileName+String("-")+distrgrid.printProcessCoords(), IOMode::read );
+ loadMeshFunctionptr->boundLoad(file);
+ file.close();
+
+ for(int i=0;i<localDof.getSize();i++)
+ {
+ EXPECT_EQ( localDof.getElement(i), loadDof.getElement(i)) << "Compare Loaded and evaluated Dof Failed for: "<< i;
+ }
+ }
+
+ static void TestLoad()
+ {
+ SharedPointer< LinearFunctionType, Device > linearFunctionPtr;
+ MeshFunctionEvaluator< MeshFunctionType, LinearFunctionType > linearFunctionEvaluator;
+
+ ParameterProvider<dim,Device> parametry;
+
+ //Crete distributed grid
+ PointType globalOrigin;
+ globalOrigin.setValue(-0.5);
+
+ PointType globalProportions;
+ globalProportions.setValue(20);
+
+ MeshType globalGrid;
+ globalGrid.setDimensions(globalProportions);
+ globalGrid.setDomain(globalOrigin,globalProportions);
+
+ int *distr=parametry.getDistr();
+
+ CoordinatesType overlap;
+ overlap.setValue(1);
+ DistributedGridType distrgrid;
+ distrgrid.template setGlobalGrid<CommunicatorType>(globalGrid,overlap, distr);
+
+ //save files from local mesh
+ PointType localOrigin=parametry.getOrigin(CommunicatorType::GetRank());
+ PointType localProportions=parametry.getProportions(CommunicatorType::GetRank());;
+
+ SharedPointer<MeshType> localGridptr;
+ localGridptr->setDimensions(localProportions);
+ localGridptr->setDomain(localOrigin,localProportions);
+
+ DofType localDof(localGridptr->template getEntitiesCount< Cell >());
+
+ SharedPointer<MeshFunctionType> localMeshFunctionptr;
+ localMeshFunctionptr->bind(localGridptr,localDof);
+ linearFunctionEvaluator.evaluateAllEntities(localMeshFunctionptr , linearFunctionPtr);
+
+
+ String FileName=String("/tmp/test-file.tnl");
+ File file;
+ file.open( FileName+String("-")+distrgrid.printProcessCoords(), IOMode::write );
+ localMeshFunctionptr->save(file);
+ file.close();
+
+
+
+ //Crete "distributedgrid driven" grid filed by load
+ SharedPointer<MeshType> loadGridptr;
+ SharedPointer<MeshFunctionType> loadMeshFunctionptr;
+ distrgrid.SetupGrid(*loadGridptr);
+
+ DofType loadDof(loadGridptr->template getEntitiesCount< Cell >());
+ loadDof.setValue(0);
+ loadMeshFunctionptr->bind(loadGridptr,loadDof);
+
+ DistributedGridIO<MeshFunctionType> ::load(FileName, *loadMeshFunctionptr );
+
+ loadMeshFunctionptr->template Synchronize<CommunicatorType>(); //need synchronization for overlaps to be filled corectly in loadDof
+
+
+ //Crete "distributedgrid driven" grid filed by evaluated linear function
+ SharedPointer<MeshType> gridptr;
+ SharedPointer<MeshFunctionType> meshFunctionptr;
+ distrgrid.SetupGrid(*gridptr);
+
+ DofType dof(gridptr->template getEntitiesCount< Cell >());
+ dof.setValue(-1);
+ meshFunctionptr->bind(gridptr,dof);
+
+ linearFunctionEvaluator.evaluateAllEntities(meshFunctionptr , linearFunctionPtr);
+ meshFunctionptr->template Synchronize<CommunicatorType>();
+
+ for(int i=0;i<dof.getSize();i++)
+ {
+ EXPECT_EQ( dof.getElement(i), loadDof.getElement(i)) << "Compare Loaded and evaluated Dof Failed for: "<< i;
+ }
+ }
+};
+
diff --git a/src/UnitTests/Mpi/DistributedGridTest_1D.cpp b/src/UnitTests/Mpi/DistributedGridTest_1D.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ef560fc0fbe063e940ce976773a949ec0251f1a2
--- /dev/null
+++ b/src/UnitTests/Mpi/DistributedGridTest_1D.cpp
@@ -0,0 +1,301 @@
+/***************************************************************************
+ DistributedGridTest.cpp - description
+ -------------------
+ begin : Sep 6, 2017
+ copyright : (C) 2017 by Tomas Oberhuber et al.
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+
+#ifdef HAVE_GTEST
+#include <gtest/gtest.h>
+
+#ifdef HAVE_MPI
+
+#include <TNL/Communicators/MpiCommunicator.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+#include <TNL/Functions/MeshFunction.h>
+
+#include "Functions.h"
+
+using namespace TNL;
+using namespace TNL::Containers;
+using namespace TNL::Meshes;
+using namespace TNL::Meshes::DistributedMeshes;
+using namespace TNL::Functions;
+using namespace TNL::Devices;
+using namespace TNL::Communicators;
+
+
+template<typename DofType>
+void setDof_1D(DofType &dof, typename DofType::RealType value)
+{
+ for(int i=0;i<dof.getSize();i++)
+ dof[i]=value;
+}
+
+template<typename DofType>
+void check_Boundary_1D(int rank, int nproc, DofType dof, typename DofType::RealType expectedValue)
+{
+ if(rank==0)//Left
+ {
+ EXPECT_EQ( dof[0], expectedValue) << "Left boundary test failed";
+ return;
+ }
+
+ if(rank==(nproc-1))//Right
+ {
+ EXPECT_EQ( dof[dof.getSize()-1], expectedValue) << "Right boundary test failed";
+ return;
+ }
+
+};
+
+template<typename DofType>
+void check_Overlap_1D(int rank, int nproc, DofType dof, typename DofType::RealType expectedValue)
+{
+ if(rank==0)//Left
+ {
+ EXPECT_EQ( dof[dof.getSize()-1], expectedValue) << "Left boundary node overlap test failed";
+ return;
+ }
+
+ if(rank==(nproc-1))
+ {
+ EXPECT_EQ( dof[0], expectedValue) << "Right boundary node overlap test failed";
+ return;
+ }
+
+ EXPECT_EQ( dof[0], expectedValue) << "left overlap test failed";
+ EXPECT_EQ( dof[dof.getSize()-1], expectedValue)<< "right overlap test failed";
+};
+
+template<typename DofType>
+void check_Inner_1D(int rank, int nproc, DofType dof, typename DofType::RealType expectedValue)
+{
+ for(int i=1;i<(dof.getSize()-2);i++) //buÄ je vlevo hranice, nebo overlap
+ EXPECT_EQ( dof[i], expectedValue) << " "<< i;
+};
+
+/*
+ * Light check of 1D distriover grid and its synchronization.
+ * Number of process is not limitated.
+ * Overlap is limitated to 1
+ * Only double is tested as dof Real type -- it may be changed, extend test
+ * Global size is hardcoded as 10 -- it can be changed, extend test
+ */
+
+typedef MpiCommunicator CommunicatorType;
+typedef Grid<1,double,Host,int> MeshType;
+typedef MeshFunction<MeshType> MeshFunctionType;
+typedef Vector<double,Host,int> DofType;
+typedef typename MeshType::Cell Cell;
+typedef typename MeshType::IndexType IndexType;
+typedef typename MeshType::PointType PointType;
+typedef DistributedMesh<MeshType> DistributedMeshType;
+
+class DistributedGirdTest_1D : public ::testing::Test {
+ protected:
+
+ static DistributedMesh<MeshType> *distrgrid;
+ static DofType *dof;
+
+ static SharedPointer<MeshType> gridptr;
+ static SharedPointer<MeshFunctionType> meshFunctionptr;
+
+ static MeshFunctionEvaluator< MeshFunctionType, ConstFunction<double,1> > constFunctionEvaluator;
+ static SharedPointer< ConstFunction<double,1>, Host > constFunctionPtr;
+
+ static MeshFunctionEvaluator< MeshFunctionType, LinearFunction<double,1> > linearFunctionEvaluator;
+ static SharedPointer< LinearFunction<double,1>, Host > linearFunctionPtr;
+
+ static int rank;
+ static int nproc;
+
+ // Per-test-case set-up.
+ // Called before the first test in this test case.
+ // Can be omitted if not needed.
+ static void SetUpTestCase() {
+
+ int size=10;
+ rank=CommunicatorType::GetRank();
+ nproc=CommunicatorType::GetSize();
+
+ PointType globalOrigin;
+ PointType globalProportions;
+ MeshType globalGrid;
+
+ globalOrigin.x()=-0.5;
+ globalProportions.x()=size;
+
+
+ globalGrid.setDimensions(size);
+ globalGrid.setDomain(globalOrigin,globalProportions);
+
+ typename DistributedMeshType::CoordinatesType overlap;
+ overlap.setValue(1);
+ distrgrid=new DistributedMeshType();
+ distrgrid->template setGlobalGrid<CommunicatorType>(globalGrid,overlap);
+
+ distrgrid->SetupGrid(*gridptr);
+ dof=new DofType(gridptr->template getEntitiesCount< Cell >());
+
+ meshFunctionptr->bind(gridptr,*dof);
+
+ constFunctionPtr->Number=rank;
+ }
+
+ // Per-test-case tear-down.
+ // Called after the last test in this test case.
+ // Can be omitted if not needed.
+ static void TearDownTestCase() {
+ delete dof;
+ delete distrgrid;
+ }
+};
+
+DistributedMesh<MeshType> *DistributedGirdTest_1D::distrgrid=NULL;
+DofType *DistributedGirdTest_1D::dof=NULL;
+SharedPointer<MeshType> DistributedGirdTest_1D::gridptr;
+SharedPointer<MeshFunctionType> DistributedGirdTest_1D::meshFunctionptr;
+MeshFunctionEvaluator< MeshFunctionType, ConstFunction<double,1> > DistributedGirdTest_1D::constFunctionEvaluator;
+SharedPointer< ConstFunction<double,1>, Host > DistributedGirdTest_1D::constFunctionPtr;
+MeshFunctionEvaluator< MeshFunctionType, LinearFunction<double,1> > DistributedGirdTest_1D::linearFunctionEvaluator;
+SharedPointer< LinearFunction<double,1>, Host > DistributedGirdTest_1D::linearFunctionPtr;
+int DistributedGirdTest_1D::rank;
+int DistributedGirdTest_1D::nproc;
+
+TEST_F(DistributedGirdTest_1D, evaluateAllEntities)
+{
+ //Check Traversars
+ //All entities, witout overlap
+ setDof_1D(*dof,-1);
+ constFunctionEvaluator.evaluateAllEntities( meshFunctionptr , constFunctionPtr );
+ //Printer<MeshType,DofType>::print_dof(rank,*gridptr,*dof);
+ check_Boundary_1D(rank, nproc, *dof, rank);
+ check_Overlap_1D(rank, nproc, *dof, -1);
+ check_Inner_1D(rank, nproc, *dof, rank);
+}
+
+TEST_F(DistributedGirdTest_1D, evaluateBoundaryEntities)
+{
+ //Boundary entities, witout overlap
+ setDof_1D(*dof,-1);
+ constFunctionEvaluator.evaluateBoundaryEntities( meshFunctionptr , constFunctionPtr );
+ check_Boundary_1D(rank, nproc, *dof, rank);
+ check_Overlap_1D(rank, nproc, *dof, -1);
+ check_Inner_1D(rank, nproc, *dof, -1);
+}
+
+TEST_F(DistributedGirdTest_1D, evaluateInteriorEntities)
+{
+ //Inner entities, witout overlap
+ setDof_1D(*dof,-1);
+ constFunctionEvaluator.evaluateInteriorEntities( meshFunctionptr , constFunctionPtr );
+ check_Boundary_1D(rank, nproc, *dof, -1);
+ check_Overlap_1D(rank, nproc, *dof, -1);
+ check_Inner_1D(rank, nproc, *dof, rank);
+}
+
+TEST_F(DistributedGirdTest_1D, LinearFunctionTest)
+{
+ //fill meshfunction with linear function (physical center of cell corresponds with its coordinates in grid)
+ setDof_1D(*dof,-1);
+ linearFunctionEvaluator.evaluateAllEntities(meshFunctionptr, linearFunctionPtr);
+ meshFunctionptr->template Synchronize<CommunicatorType>();
+
+ auto entite= gridptr->template getEntity< Cell >(0);
+ entite.refresh();
+ EXPECT_EQ(meshFunctionptr->getValue(entite), (*linearFunctionPtr)(entite)) << "Linear function Overlap error on left Edge.";
+
+ auto entite2= gridptr->template getEntity< Cell >((*dof).getSize()-1);
+ entite2.refresh();
+ EXPECT_EQ(meshFunctionptr->getValue(entite), (*linearFunctionPtr)(entite)) << "Linear function Overlap error on right Edge.";
+}
+
+TEST_F(DistributedGirdTest_1D, SynchronizerNeighborTest)
+{
+ setDof_1D(*dof,-1);
+ constFunctionEvaluator.evaluateAllEntities( meshFunctionptr , constFunctionPtr );
+ meshFunctionptr->template Synchronize<CommunicatorType>();
+
+ if(rank!=0)
+ EXPECT_EQ((*dof)[0],rank-1)<< "Left Overlap was filled by wrong process.";
+ if(rank!=nproc-1)
+ EXPECT_EQ((*dof)[dof->getSize()-1],rank+1)<< "Right Overlap was filled by wrong process.";
+}
+
+#else
+TEST(NoMPI, NoTest)
+{
+ ASSERT_TRUE(true) << ":-(";
+}
+#endif
+
+#endif
+
+
+#if (defined(HAVE_GTEST) && defined(HAVE_MPI))
+#include <sstream>
+
+ class MinimalistBuffredPrinter : public ::testing::EmptyTestEventListener {
+
+ private:
+ std::stringstream sout;
+
+ public:
+
+ // Called before a test starts.
+ virtual void OnTestStart(const ::testing::TestInfo& test_info) {
+ sout<< test_info.test_case_name() <<"." << test_info.name() << " Start." <<std::endl;
+ }
+
+ // Called after a failed assertion or a SUCCEED() invocation.
+ virtual void OnTestPartResult(
+ const ::testing::TestPartResult& test_part_result) {
+ sout << (test_part_result.failed() ? "====Failure=== " : "===Success=== ")
+ << test_part_result.file_name() << " "
+ << test_part_result.line_number() <<std::endl
+ << test_part_result.summary() <<std::endl;
+ }
+
+ // Called after a test ends.
+ virtual void OnTestEnd(const ::testing::TestInfo& test_info)
+ {
+ int rank=MPI::COMM_WORLD.Get_rank();
+ sout<< test_info.test_case_name() <<"." << test_info.name() << " End." <<std::endl;
+ std::cout << rank << ":" << std::endl << sout.str()<< std::endl;
+ sout.str( std::string() );
+ sout.clear();
+ }
+ };
+#endif
+
+#include "../../src/UnitTests/GtestMissingError.h"
+int main( int argc, char* argv[] )
+{
+#ifdef HAVE_GTEST
+ ::testing::InitGoogleTest( &argc, argv );
+
+ #ifdef HAVE_MPI
+ ::testing::TestEventListeners& listeners =
+ ::testing::UnitTest::GetInstance()->listeners();
+
+ delete listeners.Release(listeners.default_result_printer());
+ listeners.Append(new MinimalistBuffredPrinter);
+
+ CommunicatorType::Init(argc,argv);
+ #endif
+ int result= RUN_ALL_TESTS();
+
+ #ifdef HAVE_MPI
+ CommunicatorType::Finalize();
+ #endif
+ return result;
+#else
+
+ throw GtestMissingError();
+#endif
+}
+
+
diff --git a/src/UnitTests/Mpi/DistributedGridTest_2D.cpp b/src/UnitTests/Mpi/DistributedGridTest_2D.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c3657cede1f0e175d91b9a96d45731dd1bfe2be1
--- /dev/null
+++ b/src/UnitTests/Mpi/DistributedGridTest_2D.cpp
@@ -0,0 +1,572 @@
+/***************************************************************************
+ DistributedGridTest.cpp - description
+ -------------------
+ begin : Sep 6, 2017
+ copyright : (C) 2017 by Tomas Oberhuber et al.
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+
+#ifdef HAVE_GTEST
+#include <gtest/gtest.h>
+
+#ifdef HAVE_MPI
+
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+#include <TNL/Functions/MeshFunction.h>
+#include <TNL/Communicators/MpiCommunicator.h>
+
+#include "Functions.h"
+
+using namespace TNL;
+using namespace TNL::Containers;
+using namespace TNL::Meshes;
+using namespace TNL::Functions;
+using namespace TNL::Devices;
+using namespace TNL::Communicators;
+using namespace TNL::Meshes::DistributedMeshes;
+
+
+
+template<typename DofType>
+void setDof_2D(DofType &dof, typename DofType::RealType value)
+{
+ for(int i=0;i<dof.getSize();i++)
+ dof[i]=value;
+}
+
+template<typename DofType,typename GridType>
+void checkLeftEdge(GridType &grid, DofType &dof, bool with_first, bool with_last, typename DofType::RealType expectedValue)
+{
+ int maxx=grid.getDimensions().x();
+ int maxy=grid.getDimensions().y();
+ int begin=0;
+ int end=maxy;
+ if(!with_first)
+ begin++;
+ if(!with_last)
+ end--;
+
+ for(int i=begin;i<end;i++) //posledni je overlap
+ EXPECT_EQ( dof[maxx*i], expectedValue) << "Left Edge test failed " << i<<" " << maxx << " "<< maxy;
+}
+
+template<typename DofType,typename GridType>
+void checkRightEdge(GridType &grid, DofType &dof, bool with_first, bool with_last, typename DofType::RealType expectedValue)
+{
+ int maxx=grid.getDimensions().x();
+ int maxy=grid.getDimensions().y();
+ int begin=0;
+ int end=maxy;
+ if(!with_first)
+ begin++;
+ if(!with_last)
+ end--;
+
+ for(int i=begin;i<end;i++)
+ EXPECT_EQ( dof[maxx*i+(maxx-1)], expectedValue) << "Right Edge test failed " << i <<" " << maxx << " "<< maxy;
+}
+
+template<typename DofType,typename GridType>
+void checkUpEdge(GridType &grid, DofType &dof, bool with_first, bool with_last, typename DofType::RealType expectedValue)
+{
+ int maxx=grid.getDimensions().x();
+ int maxy=grid.getDimensions().y();
+ int begin=0;
+ int end=maxx;
+ if(!with_first)
+ begin++;
+ if(!with_last)
+ end--;
+
+ for(int i=begin;i<end;i++) //posledni je overlap
+ EXPECT_EQ( dof[i], expectedValue) << "Up Edge test failed " << i<<" " << maxx << " "<< maxy;
+}
+
+template<typename DofType,typename GridType>
+void checkDownEdge(GridType &grid, DofType &dof, bool with_first, bool with_last, typename DofType::RealType expectedValue)
+{
+ int maxx=grid.getDimensions().x();
+ int maxy=grid.getDimensions().y();
+ int begin=0;
+ int end=maxx;
+ if(!with_first)
+ begin++;
+ if(!with_last)
+ end--;
+
+ for(int i=begin;i<end;i++) //posledni je overlap
+ EXPECT_EQ( dof[maxx*(maxy-1)+i], expectedValue) << "Down Edge test failed " << i<<" " << maxx << " "<< maxy;
+}
+
+template<typename DofType,typename GridType>
+void checkConner(GridType &grid, DofType &dof, bool up, bool left, typename DofType::RealType expectedValue )
+{
+ int maxx=grid.getDimensions().x();
+ int maxy=grid.getDimensions().y();
+ if(up&&left)
+ {
+ EXPECT_EQ( dof[0], expectedValue) << "Up Left Conner test failed ";
+ }
+ if(up && !left)
+ {
+ EXPECT_EQ( dof[maxx-1], expectedValue) << "Up Right Conner test failed ";
+ }
+ if(!up && left)
+ {
+ EXPECT_EQ( dof[(maxy-1)*maxx], expectedValue) << "Down Left Conner test failed ";
+ }
+ if(!up && !left)
+ {
+ EXPECT_EQ( dof[(maxy-1)*maxx+maxx-1], expectedValue) << "Down right Conner test failed ";
+ }
+}
+
+
+/*expect 9 process*/
+template<typename DofType,typename GridType>
+void check_Boundary_2D(int rank, GridType &grid, DofType &dof, typename DofType::RealType expectedValue)
+{
+
+ if(rank==0)//Up Left
+ {
+ checkUpEdge(grid,dof,true,false,expectedValue);//posledni je overlap
+ checkLeftEdge(grid,dof,true,false, expectedValue);//posledni je overlap
+ }
+
+ if(rank==1)//Up Center
+ {
+ checkUpEdge(grid,dof,false,false, expectedValue);//prvni a posledni je overlap
+ }
+
+ if(rank==2)//Up Right
+ {
+ checkUpEdge(grid,dof,false,true,expectedValue);//prvni je overlap
+ checkRightEdge(grid,dof,true,false,expectedValue);//posledni je overlap
+ }
+
+ if(rank==3)//Center Left
+ {
+ checkLeftEdge(grid,dof,false,false,expectedValue);//prvni a posledni je overlap
+ }
+
+ if(rank==4)//Center Center
+ {
+ //No boundary
+ }
+
+ if(rank==5)//Center Right
+ {
+ checkRightEdge(grid,dof,false,false,expectedValue);
+ }
+
+ if(rank==6)//Down Left
+ {
+ checkDownEdge(grid,dof,true,false,expectedValue);
+ checkLeftEdge(grid,dof,false,true,expectedValue);
+ }
+
+ if(rank==7) //Down Center
+ {
+ checkDownEdge(grid,dof,false,false,expectedValue);
+ }
+
+ if(rank==8) //Down Right
+ {
+ checkDownEdge(grid,dof,false,true,expectedValue);
+ checkRightEdge(grid,dof,false,true,expectedValue);
+ }
+};
+
+/*expect 9 process
+ * Known BUG of Traversars: Process boundary is writing over overlap.
+ * it should be true, true, every where, but we dont chcek boundary overalp on boundary
+ * so boundary overlap is not checked (it is filled incorectly by boundary condition).
+ */
+template<typename DofType,typename GridType>
+void check_Overlap_2D(int rank, GridType &grid, DofType &dof, typename DofType::RealType expectedValue)
+{
+ if(rank==0)//Up Left
+ {
+ checkRightEdge(grid,dof,false,true,expectedValue);
+ checkDownEdge(grid,dof,false,true,expectedValue);
+ }
+
+ if(rank==1)//Up Center
+ {
+ checkDownEdge(grid,dof,true,true,expectedValue);
+ checkLeftEdge(grid,dof,false,true,expectedValue);
+ checkRightEdge(grid,dof,false,true,expectedValue);
+ }
+
+ if(rank==2)//Up Right
+ {
+ checkDownEdge(grid,dof,true,false,expectedValue);//prvni je overlap
+ checkLeftEdge(grid,dof,false,true,expectedValue);
+ }
+
+ if(rank==3)//Center Left
+ {
+ checkUpEdge(grid,dof,false,true,expectedValue);
+ checkDownEdge(grid,dof,false,true,expectedValue);
+ checkRightEdge(grid,dof,true,true,expectedValue);
+ }
+
+ if(rank==4)//Center Center
+ {
+ checkUpEdge(grid,dof,true,true,expectedValue);
+ checkDownEdge(grid,dof,true,true,expectedValue);
+ checkRightEdge(grid,dof,true,true,expectedValue);
+ checkLeftEdge(grid,dof,true,true,expectedValue);
+ }
+
+ if(rank==5)//Center Right
+ {
+ checkUpEdge(grid,dof,true,false,expectedValue);
+ checkDownEdge(grid,dof,true,false,expectedValue);
+ checkLeftEdge(grid,dof,true,true,expectedValue);
+ }
+
+ if(rank==6)//Down Left
+ {
+ checkUpEdge(grid,dof,false,true,expectedValue);
+ checkRightEdge(grid,dof,true,false,expectedValue);
+ }
+
+ if(rank==7) //Down Center
+ {
+ checkUpEdge(grid,dof,true,true,expectedValue);
+ checkLeftEdge(grid,dof,true,false,expectedValue);
+ checkRightEdge(grid,dof,true,false,expectedValue);
+ }
+
+ if(rank==8) //Down Right
+ {
+ checkUpEdge(grid,dof,true,false,expectedValue);
+ checkLeftEdge(grid,dof,true,false,expectedValue);
+ }
+}
+
+/*Expect 9 process
+ */
+template<typename DofType,typename GridType>
+void checkNeighbor_2D(int rank, GridType &grid, DofType &dof)
+{
+ if(rank==0)//Up Left
+ {
+ checkRightEdge(grid,dof,true,false,1);
+ checkDownEdge(grid,dof,true,false,3);
+ checkConner(grid,dof,false,false,4);
+
+ }
+
+ if(rank==1)//Up Center
+ {
+ checkLeftEdge(grid,dof,true,false,0);
+ checkRightEdge(grid,dof,true,false,2);
+ checkConner(grid,dof,false,true,3);
+ checkDownEdge(grid,dof,false,false,4);
+ checkConner(grid,dof,false,false,5);
+ }
+
+ if(rank==2)//Up Right
+ {
+ checkLeftEdge(grid,dof,true,false,1);
+ checkConner(grid,dof,false,true,4);
+ checkDownEdge(grid,dof,false,true,5);
+ }
+
+ if(rank==3)//Center Left
+ {
+ checkUpEdge(grid,dof,true,false,0);
+ checkConner(grid,dof,true,false,1);
+ checkRightEdge(grid,dof,false,false,4);
+ checkDownEdge(grid,dof,true,false,6);
+ checkConner(grid,dof,false,false,7);
+ }
+
+ if(rank==4)//Center Center
+ {
+ checkConner(grid,dof,true,true,0);
+ checkUpEdge(grid,dof,false,false,1);
+ checkConner(grid,dof,true,false,2);
+ checkLeftEdge(grid,dof,false,false,3);
+ checkRightEdge(grid,dof,false,false,5);
+ checkConner(grid,dof,false,true,6);
+ checkDownEdge(grid,dof,false,false,7);
+ checkConner(grid,dof,false,false,8);
+ }
+
+ if(rank==5)//Center Right
+ {
+ checkConner(grid,dof,true,true,1);
+ checkUpEdge(grid,dof,false,true,2);
+ checkLeftEdge(grid,dof,false,false,4);
+ checkConner(grid,dof,false,true,7);
+ checkDownEdge(grid,dof,false,true,8);
+ }
+
+ if(rank==6)//Down Left
+ {
+ checkUpEdge(grid,dof,true,false,3);
+ checkConner(grid,dof,true,false,4);
+ checkRightEdge(grid,dof,false,true,7);
+ }
+
+ if(rank==7) //Down Center
+ {
+ checkConner(grid,dof,true,true,3);
+ checkUpEdge(grid,dof,false,false,4);
+ checkConner(grid,dof,true,false,5);
+ checkLeftEdge(grid,dof,false,true,6);
+ checkRightEdge(grid,dof,false,true,8);
+ }
+
+ if(rank==8) //Down Right
+ {
+ checkConner(grid,dof,true,true,4);
+ checkUpEdge(grid,dof,false,true,5);
+ checkLeftEdge(grid,dof,false,true,7);
+ }
+}
+
+
+template<typename DofType,typename GridType>
+void check_Inner_2D(int rank, GridType grid, DofType dof, typename DofType::RealType expectedValue)
+{
+ int maxx=grid.getDimensions().x();
+ int maxy=grid.getDimensions().y();
+ for(int j=1;j<maxy-1;j++)//prvni a posledni jsou buÄ hranice, nebo overlap
+ for(int i=1;i<maxx-1;i++) //buÄ je vlevo hranice, nebo overlap
+ EXPECT_EQ( dof[j*maxx+i], expectedValue) << " "<< j<<" "<<i << " " << maxx << " " << maxy;
+}
+
+
+
+
+
+/*
+ * Light check of 2D distributed grid and its synchronization.
+ * expected 9 processors
+ */
+
+typedef MpiCommunicator CommunicatorType;
+typedef Grid<2,double,Host,int> MeshType;
+typedef MeshFunction<MeshType> MeshFunctionType;
+typedef Vector<double,Host,int> DofType;
+typedef typename MeshType::Cell Cell;
+typedef typename MeshType::IndexType IndexType;
+typedef typename MeshType::PointType PointType;
+typedef DistributedMesh<MeshType> DistributedGridType;
+
+class DistributedGirdTest_2D : public ::testing::Test {
+ protected:
+
+ static DistributedGridType *distrgrid;
+ static DofType *dof;
+
+ static SharedPointer<MeshType> gridptr;
+ static SharedPointer<MeshFunctionType> meshFunctionptr;
+
+ static MeshFunctionEvaluator< MeshFunctionType, ConstFunction<double,2> > constFunctionEvaluator;
+ static SharedPointer< ConstFunction<double,2>, Host > constFunctionPtr;
+
+ static MeshFunctionEvaluator< MeshFunctionType, LinearFunction<double,2> > linearFunctionEvaluator;
+ static SharedPointer< LinearFunction<double,2>, Host > linearFunctionPtr;
+
+ static int rank;
+ static int nproc;
+
+ // Per-test-case set-up.
+ // Called before the first test in this test case.
+ // Can be omitted if not needed.
+ static void SetUpTestCase() {
+
+ int size=10;
+ rank=CommunicatorType::GetRank();
+ nproc=CommunicatorType::GetSize();
+
+ PointType globalOrigin;
+ PointType globalProportions;
+ MeshType globalGrid;
+
+ globalOrigin.x()=-0.5;
+ globalOrigin.y()=-0.5;
+ globalProportions.x()=size;
+ globalProportions.y()=size;
+
+ globalGrid.setDimensions(size,size);
+ globalGrid.setDomain(globalOrigin,globalProportions);
+
+ typename DistributedGridType::CoordinatesType overlap;
+ overlap.setValue(1);
+ distrgrid=new DistributedGridType();
+ int distr[2];
+ distr[0]=3;
+ distr[1]=3;
+ distrgrid->template setGlobalGrid<CommunicatorType>(globalGrid,overlap, distr);
+
+ distrgrid->SetupGrid(*gridptr);
+ dof=new DofType(gridptr->template getEntitiesCount< Cell >());
+
+ meshFunctionptr->bind(gridptr,*dof);
+
+ constFunctionPtr->Number=rank;
+
+ }
+
+ // Per-test-case tear-down.
+ // Called after the last test in this test case.
+ // Can be omitted if not needed.
+ static void TearDownTestCase() {
+ delete dof;
+ delete distrgrid;
+
+ }
+
+};
+
+DistributedMesh<MeshType> *DistributedGirdTest_2D::distrgrid=NULL;
+DofType *DistributedGirdTest_2D::dof=NULL;
+SharedPointer<MeshType> DistributedGirdTest_2D::gridptr;
+SharedPointer<MeshFunctionType> DistributedGirdTest_2D::meshFunctionptr;
+MeshFunctionEvaluator< MeshFunctionType, ConstFunction<double,2> > DistributedGirdTest_2D::constFunctionEvaluator;
+SharedPointer< ConstFunction<double,2>, Host > DistributedGirdTest_2D::constFunctionPtr;
+MeshFunctionEvaluator< MeshFunctionType, LinearFunction<double,2> > DistributedGirdTest_2D::linearFunctionEvaluator;
+SharedPointer< LinearFunction<double,2>, Host > DistributedGirdTest_2D::linearFunctionPtr;
+int DistributedGirdTest_2D::rank;
+int DistributedGirdTest_2D::nproc;
+
+TEST_F(DistributedGirdTest_2D, evaluateAllEntities)
+{
+
+ //Check Traversars
+ //All entities, witout overlap
+ setDof_2D(*dof,-1);
+ constFunctionEvaluator.evaluateAllEntities( meshFunctionptr , constFunctionPtr );
+ //Printer<MeshType,DofType>::print_dof(rank,*gridptr,*dof);
+ check_Boundary_2D(rank, *gridptr, *dof, rank);
+ check_Overlap_2D(rank, *gridptr, *dof, -1);
+ check_Inner_2D(rank, *gridptr, *dof, rank);
+}
+
+TEST_F(DistributedGirdTest_2D, evaluateBoundaryEntities)
+{
+ //Boundary entities, witout overlap
+ setDof_2D(*dof,-1);
+ constFunctionEvaluator.evaluateBoundaryEntities( meshFunctionptr , constFunctionPtr );
+ //print_dof_2D(rank,*gridptr,dof);
+ check_Boundary_2D(rank, *gridptr, *dof, rank);
+ check_Overlap_2D(rank, *gridptr, *dof, -1);
+ check_Inner_2D(rank, *gridptr, *dof, -1);
+}
+
+TEST_F(DistributedGirdTest_2D, evaluateInteriorEntities)
+{
+ //Inner entities, witout overlap
+ setDof_2D(*dof,-1);
+ constFunctionEvaluator.evaluateInteriorEntities( meshFunctionptr , constFunctionPtr );
+ check_Boundary_2D(rank, *gridptr, *dof, -1);
+ check_Overlap_2D(rank, *gridptr, *dof, -1);
+ check_Inner_2D(rank, *gridptr, *dof, rank);
+}
+
+TEST_F(DistributedGirdTest_2D, LinearFunctionTest)
+{
+ //fill meshfunction with linear function (physical center of cell corresponds with its coordinates in grid)
+ setDof_2D(*dof,-1);
+ linearFunctionEvaluator.evaluateAllEntities(meshFunctionptr, linearFunctionPtr);
+ meshFunctionptr->template Synchronize<CommunicatorType>();
+
+ int count =gridptr->template getEntitiesCount< Cell >();
+ for(int i=0;i<count;i++)
+ {
+ auto entity= gridptr->template getEntity< Cell >(i);
+ entity.refresh();
+ EXPECT_EQ(meshFunctionptr->getValue(entity), (*linearFunctionPtr)(entity)) << "Linear function doesnt fit recievd data. " << entity.getCoordinates().x() << " "<<entity.getCoordinates().y() << " "<< gridptr->getDimensions().x() <<" "<<gridptr->getDimensions().y();
+ }
+}
+
+TEST_F(DistributedGirdTest_2D, SynchronizerNeighborTest)
+{
+ setDof_2D(*dof,-1);
+ constFunctionEvaluator.evaluateAllEntities( meshFunctionptr , constFunctionPtr );
+ meshFunctionptr->template Synchronize<CommunicatorType>();
+ checkNeighbor_2D(rank, *gridptr, *dof);
+}
+
+
+#else
+TEST(NoMPI, NoTest)
+{
+ ASSERT_TRUE(true) << ":-(";
+}
+#endif
+
+#endif
+
+
+#if (defined(HAVE_GTEST) && defined(HAVE_MPI))
+#include <sstream>
+
+ class MinimalistBuffredPrinter : public ::testing::EmptyTestEventListener {
+
+ private:
+ std::stringstream sout;
+
+ public:
+
+ // Called before a test starts.
+ virtual void OnTestStart(const ::testing::TestInfo& test_info) {
+ sout<< test_info.test_case_name() <<"." << test_info.name() << " Start." <<std::endl;
+ }
+
+ // Called after a failed assertion or a SUCCEED() invocation.
+ virtual void OnTestPartResult(
+ const ::testing::TestPartResult& test_part_result) {
+ sout << (test_part_result.failed() ? "====Failure=== " : "===Success=== ")
+ << test_part_result.file_name() << " "
+ << test_part_result.line_number() <<std::endl
+ << test_part_result.summary() <<std::endl;
+ }
+
+ // Called after a test ends.
+ virtual void OnTestEnd(const ::testing::TestInfo& test_info)
+ {
+ int rank=CommunicatorType::GetRank();
+ sout<< test_info.test_case_name() <<"." << test_info.name() << " End." <<std::endl;
+ std::cout << rank << ":" << std::endl << sout.str()<< std::endl;
+ sout.str( std::string() );
+ sout.clear();
+ }
+ };
+#endif
+
+#include "../../src/UnitTests/GtestMissingError.h"
+int main( int argc, char* argv[] )
+{
+#ifdef HAVE_GTEST
+ ::testing::InitGoogleTest( &argc, argv );
+
+ #ifdef HAVE_MPI
+ ::testing::TestEventListeners& listeners =
+ ::testing::UnitTest::GetInstance()->listeners();
+
+ delete listeners.Release(listeners.default_result_printer());
+ listeners.Append(new MinimalistBuffredPrinter);
+
+ CommunicatorType::Init(argc,argv);
+ #endif
+ int result= RUN_ALL_TESTS();
+
+ #ifdef HAVE_MPI
+ CommunicatorType::Finalize();
+ #endif
+ return result;
+#else
+
+ throw GtestMissingError();
+#endif
+}
+
+
diff --git a/src/UnitTests/Mpi/DistributedGridTest_3D.cpp b/src/UnitTests/Mpi/DistributedGridTest_3D.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..758c7dcaca984c18963260f73b31a3ce3e8a2330
--- /dev/null
+++ b/src/UnitTests/Mpi/DistributedGridTest_3D.cpp
@@ -0,0 +1,814 @@
+#ifdef HAVE_GTEST
+#include <gtest/gtest.h>
+
+#ifdef HAVE_MPI
+
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+#include <TNL/Functions/MeshFunction.h>
+#include <TNL/Communicators/MpiCommunicator.h>
+
+#include "Functions.h"
+
+using namespace TNL;
+using namespace TNL::Containers;
+using namespace TNL::Meshes;
+using namespace TNL::Functions;
+using namespace TNL::Devices;
+using namespace TNL::Communicators;
+using namespace TNL::Meshes::DistributedMeshes;
+
+template<typename DofType>
+void setDof_3D(DofType &dof, typename DofType::RealType value)
+{
+ for(int i=0;i<dof.getSize();i++)
+ dof[i]=value;
+}
+
+template<typename GridType>
+int getAdd(GridType &grid,bool bottom, bool nord, bool west )
+{
+ int maxx=grid.getDimensions().x();
+ int maxy=grid.getDimensions().y();
+ int maxz=grid.getDimensions().z();
+
+ int add=0;
+ if(!west)
+ add+=maxx-1;
+ if(!nord)
+ add+=(maxy-1)*maxx;
+ if(!bottom)
+ add+=(maxz-1)*maxx*maxy;
+
+ return add;
+}
+
+template<typename DofType,typename GridType>
+void checkConner(GridType &grid, DofType &dof,bool bottom, bool nord, bool west, typename DofType::RealType expectedValue )
+{
+ int i=getAdd(grid,bottom,nord,west);
+ EXPECT_EQ( dof[i], expectedValue) << "Conner test failed";
+
+}
+
+template<typename DofType,typename GridType>
+void checkXDirectionEdge(GridType &grid, DofType &dof, bool bottom, bool nord, typename DofType::RealType expectedValue)
+{
+ int add=getAdd(grid,bottom,nord,true);
+ for(int i=1;i<grid.getDimensions().x()-1;i++)
+ EXPECT_EQ( dof[i+add], expectedValue) << "X direction Edge test failed " << i;
+}
+
+
+template<typename DofType,typename GridType>
+void checkYDirectionEdge(GridType &grid, DofType &dof, bool bottom, bool west, typename DofType::RealType expectedValue)
+{
+ int add=getAdd(grid,bottom,true,west);
+ for(int i=1;i<grid.getDimensions().y()-1;i++)
+ EXPECT_EQ( dof[grid.getDimensions().x()*i+add], expectedValue) << "Y direction Edge test failed " << i;
+}
+
+template<typename DofType,typename GridType>
+void checkZDirectionEdge(GridType &grid, DofType &dof, bool nord, bool west, typename DofType::RealType expectedValue)
+{
+ int add=getAdd(grid,true,nord,west);
+ for(int i=1;i<grid.getDimensions().z()-1;i++)
+ EXPECT_EQ( dof[grid.getDimensions().y()*grid.getDimensions().x()*i+add], expectedValue) << "Z direction Edge test failed " << i;
+}
+
+template<typename DofType,typename GridType>
+void checkZFace(GridType &grid, DofType &dof, bool bottom, typename DofType::RealType expectedValue)
+{
+ int add=getAdd(grid,bottom,true,true);
+ for(int i=1;i<grid.getDimensions().y()-1;i++)
+ for(int j=1; j<grid.getDimensions().x()-1;j++)
+ {
+ EXPECT_EQ( dof[grid.getDimensions().x()*i+j+add], expectedValue) << "Z Face test failed "<<i<< " " << j;
+ }
+}
+
+template<typename DofType,typename GridType>
+void checkYFace(GridType &grid, DofType &dof, bool nord, typename DofType::RealType expectedValue)
+{
+ int add=getAdd(grid,true,nord,true);
+ for(int i=1;i<grid.getDimensions().z()-1;i++)
+ for(int j=1; j<grid.getDimensions().x()-1;j++)
+ {
+ EXPECT_EQ( dof[grid.getDimensions().y()*grid.getDimensions().x()*i+j+add], expectedValue) << "Y Face test failed "<<i<< " " << j;
+ }
+}
+
+template<typename DofType,typename GridType>
+void checkXFace(GridType &grid, DofType &dof, bool west, typename DofType::RealType expectedValue)
+{
+ int add=getAdd(grid,true,true,west);
+ for(int i=1;i<grid.getDimensions().z()-1;i++)
+ for(int j=1; j<grid.getDimensions().y()-1;j++)
+ {
+ EXPECT_EQ( dof[grid.getDimensions().y()*grid.getDimensions().x()*i+grid.getDimensions().x()*j+add], expectedValue) << "X Face test failed "<<i<< " " << j;
+ }
+}
+
+/*
+Expected 27 process
+*/
+template<typename DofType,typename GridType>
+void check_Boundary_3D(int rank, GridType &grid, DofType &dof, typename DofType::RealType expectedValue)
+{
+ if(rank==0)//Bottom Nord West
+ {
+ checkConner(grid,dof,true, true, true, expectedValue );
+ checkXDirectionEdge(grid,dof,true,true,expectedValue);
+ checkYDirectionEdge(grid,dof,true,true,expectedValue);
+ checkZDirectionEdge(grid,dof,true,true,expectedValue);
+ checkXFace(grid, dof, true, expectedValue);
+ checkYFace(grid, dof, true, expectedValue);
+ checkZFace(grid, dof, true, expectedValue);
+ }
+
+ if(rank==1)//Bottom Nord Center
+ {
+ checkXDirectionEdge(grid,dof,true,true,expectedValue);
+ checkYFace(grid, dof, true, expectedValue);
+ checkZFace(grid, dof, true, expectedValue);
+ }
+
+ if(rank==2)//Bottom Nord East
+ {
+ checkConner(grid,dof,true, true, false, expectedValue );
+ checkXDirectionEdge(grid,dof,true,true,expectedValue);
+ checkYDirectionEdge(grid,dof,true,false,expectedValue);
+ checkZDirectionEdge(grid,dof,true,false,expectedValue);
+ checkXFace(grid, dof, false, expectedValue);
+ checkYFace(grid, dof, true, expectedValue);
+ checkZFace(grid, dof, true, expectedValue);
+ }
+
+ if(rank==3)//Bottom Center West
+ {
+ checkYDirectionEdge(grid,dof,true,true,expectedValue);
+ checkXFace(grid, dof, true, expectedValue);
+ checkZFace(grid, dof, true, expectedValue);
+ }
+
+ if(rank==4)//Bottom Center Center
+ {
+ checkZFace(grid, dof, true, expectedValue);
+ }
+
+
+ if(rank==5)//Bottom Center East
+ {
+ checkYDirectionEdge(grid,dof,true,false,expectedValue);
+ checkXFace(grid, dof, false, expectedValue);
+ checkZFace(grid, dof, true, expectedValue);
+ }
+
+ if(rank==6)//Bottom South West
+ {
+ checkConner(grid,dof,true, false, true, expectedValue );
+ checkXDirectionEdge(grid,dof,true,false,expectedValue);
+ checkYDirectionEdge(grid,dof,true,true,expectedValue);
+ checkZDirectionEdge(grid,dof,false,true,expectedValue);
+ checkXFace(grid, dof, true, expectedValue);
+ checkYFace(grid, dof, false, expectedValue);
+ checkZFace(grid, dof, true, expectedValue);
+ }
+
+ if(rank==7)//Bottom South Center
+ {
+ checkXDirectionEdge(grid,dof,true,false,expectedValue);
+ checkYFace(grid, dof, false, expectedValue);
+ checkZFace(grid, dof, true, expectedValue);
+ }
+
+ if(rank==8)//Bottom South East
+ {
+ checkConner(grid,dof,true, false, false, expectedValue );
+ checkXDirectionEdge(grid,dof,true,false,expectedValue);
+ checkYDirectionEdge(grid,dof,true,false,expectedValue);
+ checkZDirectionEdge(grid,dof,false,false,expectedValue);
+ checkXFace(grid, dof, false, expectedValue);
+ checkYFace(grid, dof, false, expectedValue);
+ checkZFace(grid, dof, true, expectedValue);
+ }
+
+ if(rank==9)//Center Nord West
+ {
+ checkZDirectionEdge(grid,dof,true,true,expectedValue);
+ checkXFace(grid, dof, true, expectedValue);
+ checkYFace(grid, dof, true, expectedValue);
+ }
+
+ if(rank==10)//Center Nord Center
+ {
+ checkYFace(grid, dof, true, expectedValue);
+ }
+
+ if(rank==11)//Center Nord East
+ {
+ checkZDirectionEdge(grid,dof,true,false,expectedValue);
+ checkXFace(grid, dof, false, expectedValue);
+ checkYFace(grid, dof, true, expectedValue);
+ }
+
+ if(rank==12)//Center Center West
+ {
+ checkXFace(grid, dof, true, expectedValue);
+ }
+
+ if(rank==13)//Center Center Center
+ {
+ //no Boundary
+ }
+
+
+ if(rank==14)//Center Center East
+ {
+ checkXFace(grid, dof, false, expectedValue);
+ }
+
+ if(rank==15)//Center South West
+ {
+ checkZDirectionEdge(grid,dof,false,true,expectedValue);
+ checkXFace(grid, dof, true, expectedValue);
+ checkYFace(grid, dof, false, expectedValue);
+ }
+
+ if(rank==16)//Center South Center
+ {
+ checkYFace(grid, dof, false, expectedValue);
+ }
+
+ if(rank==17)//Center South East
+ {
+ checkZDirectionEdge(grid,dof,false,false,expectedValue);
+ checkXFace(grid, dof, false, expectedValue);
+ checkYFace(grid, dof, false, expectedValue);
+ }
+
+ if(rank==18)//Top Nord West
+ {
+ checkConner(grid,dof,false, true, true, expectedValue );
+ checkXDirectionEdge(grid,dof,false,true,expectedValue);
+ checkYDirectionEdge(grid,dof,false,true,expectedValue);
+ checkZDirectionEdge(grid,dof,true,true,expectedValue);
+ checkXFace(grid, dof, true, expectedValue);
+ checkYFace(grid, dof, true, expectedValue);
+ checkZFace(grid, dof, false, expectedValue);
+ }
+
+ if(rank==19)//Top Nord Center
+ {
+ checkXDirectionEdge(grid,dof,false,true,expectedValue);
+ checkYFace(grid, dof, true, expectedValue);
+ checkZFace(grid, dof, false, expectedValue);
+ }
+
+ if(rank==20)//Top Nord East
+ {
+ checkConner(grid,dof,false, true, false, expectedValue );
+ checkXDirectionEdge(grid,dof,false,true,expectedValue);
+ checkYDirectionEdge(grid,dof,false,false,expectedValue);
+ checkZDirectionEdge(grid,dof,true,false,expectedValue);
+ checkXFace(grid, dof, false, expectedValue);
+ checkYFace(grid, dof, true, expectedValue);
+ checkZFace(grid, dof, false, expectedValue);
+ }
+
+ if(rank==21)//Top Center West
+ {
+ checkYDirectionEdge(grid,dof,false,true,expectedValue);
+ checkXFace(grid, dof, true, expectedValue);
+ checkZFace(grid, dof, false, expectedValue);
+ }
+
+ if(rank==22)//Top Center Center
+ {
+ checkZFace(grid, dof, false, expectedValue);
+ }
+
+
+ if(rank==23)//Top Center East
+ {
+ checkYDirectionEdge(grid,dof,false,false,expectedValue);
+ checkXFace(grid, dof, false, expectedValue);
+ checkZFace(grid, dof, false, expectedValue);
+ }
+
+ if(rank==24)//Top South West
+ {
+ checkConner(grid,dof,false, false, true, expectedValue );
+ checkXDirectionEdge(grid,dof,false,false,expectedValue);
+ checkYDirectionEdge(grid,dof,false,true,expectedValue);
+ checkZDirectionEdge(grid,dof,false,true,expectedValue);
+ checkXFace(grid, dof, true, expectedValue);
+ checkYFace(grid, dof, false, expectedValue);
+ checkZFace(grid, dof, false, expectedValue);
+ }
+
+ if(rank==25)//Top South Center
+ {
+ checkXDirectionEdge(grid,dof,false,false,expectedValue);
+ checkYFace(grid, dof, false, expectedValue);
+ checkZFace(grid, dof, false, expectedValue);
+ }
+
+ if(rank==26)//Top South East
+ {
+ checkConner(grid,dof,false, false, false, expectedValue );
+ checkXDirectionEdge(grid,dof,false,false,expectedValue);
+ checkYDirectionEdge(grid,dof,false,false,expectedValue);
+ checkZDirectionEdge(grid,dof,false,false,expectedValue);
+ checkXFace(grid, dof, false, expectedValue);
+ checkYFace(grid, dof, false, expectedValue);
+ checkZFace(grid, dof, false, expectedValue);
+ }
+
+}
+
+template<typename DofType,typename GridType>
+void CheckConnerNode_Overlap(GridType &grid, DofType &dof,bool bottom, bool nord, bool west, typename DofType::RealType expectedValue)
+{
+ checkConner(grid,dof,!bottom, !nord, !west, expectedValue );
+ checkXDirectionEdge(grid,dof,!bottom,!nord,expectedValue);
+ checkYDirectionEdge(grid,dof,!bottom,!west,expectedValue);
+ checkZDirectionEdge(grid,dof,!nord,!west,expectedValue);
+ checkXFace(grid, dof, !west, expectedValue);
+ checkYFace(grid, dof, !nord, expectedValue);
+ checkZFace(grid, dof, !bottom, expectedValue);
+}
+
+template<typename DofType,typename GridType>
+void CheckXEdgeNode_Overlap(GridType &grid, DofType &dof,bool bottom, bool nord, typename DofType::RealType expectedValue)
+{
+ checkConner(grid,dof,!bottom, !nord, false, expectedValue );
+ checkConner(grid,dof,!bottom, !nord, true, expectedValue );
+ checkXDirectionEdge(grid,dof,!bottom, !nord,expectedValue);
+ checkYDirectionEdge(grid,dof,!bottom,false,expectedValue);
+ checkYDirectionEdge(grid,dof,!bottom,true,expectedValue);
+ checkZDirectionEdge(grid,dof,!nord,false,expectedValue);
+ checkZDirectionEdge(grid,dof,!nord,true,expectedValue);
+ checkXFace(grid, dof, false, expectedValue);
+ checkXFace(grid, dof, true, expectedValue);
+ checkYFace(grid, dof, !nord, expectedValue);
+ checkZFace(grid, dof, !bottom, expectedValue);
+}
+
+template<typename DofType,typename GridType>
+void CheckYEdgeNode_Overlap(GridType &grid, DofType &dof,bool bottom, bool west, typename DofType::RealType expectedValue)
+{
+ checkConner(grid,dof,!bottom, false, !west, expectedValue );
+ checkConner(grid,dof,!bottom, true, !west, expectedValue );
+ checkXDirectionEdge(grid,dof,!bottom,false,expectedValue);
+ checkXDirectionEdge(grid,dof,!bottom,true,expectedValue);
+ checkYDirectionEdge(grid,dof,!bottom,!west,expectedValue);
+ checkZDirectionEdge(grid,dof,false,!west,expectedValue);
+ checkZDirectionEdge(grid,dof,true,!west,expectedValue);
+ checkXFace(grid, dof, !west, expectedValue);
+ checkYFace(grid, dof, false, expectedValue);
+ checkYFace(grid, dof, true, expectedValue);
+ checkZFace(grid, dof, !bottom, expectedValue);
+}
+
+template<typename DofType,typename GridType>
+void CheckZEdgeNode_Overlap(GridType &grid, DofType &dof,bool nord, bool west, typename DofType::RealType expectedValue)
+{
+ checkConner(grid,dof,false, !nord, !west, expectedValue );
+ checkConner(grid,dof,true, !nord, !west, expectedValue );
+ checkXDirectionEdge(grid,dof,false,!nord,expectedValue);
+ checkXDirectionEdge(grid,dof,true,!nord,expectedValue);
+ checkYDirectionEdge(grid,dof,false,!west,expectedValue);
+ checkYDirectionEdge(grid,dof,true,!west,expectedValue);
+ checkZDirectionEdge(grid,dof,!nord,!west,expectedValue);
+ checkXFace(grid, dof, !west, expectedValue);
+ checkYFace(grid, dof, !nord, expectedValue);
+ checkZFace(grid, dof, false, expectedValue);
+ checkZFace(grid, dof, true, expectedValue);
+}
+
+template<typename DofType,typename GridType>
+void CheckXFaceNode_Overlap(GridType &grid, DofType &dof,bool west, typename DofType::RealType expectedValue)
+{
+ checkConner(grid,dof,false, false, !west, expectedValue );
+ checkConner(grid,dof,false, true, !west, expectedValue );
+ checkConner(grid,dof,true, false, !west, expectedValue );
+ checkConner(grid,dof,true, true, !west, expectedValue );
+ checkXDirectionEdge(grid,dof,false,false,expectedValue);
+ checkXDirectionEdge(grid,dof,false,true,expectedValue);
+ checkXDirectionEdge(grid,dof,true,false,expectedValue);
+ checkXDirectionEdge(grid,dof,true,true,expectedValue);
+ checkYDirectionEdge(grid,dof,false,!west,expectedValue);
+ checkYDirectionEdge(grid,dof,true,!west,expectedValue);
+ checkZDirectionEdge(grid,dof,false,!west,expectedValue);
+ checkZDirectionEdge(grid,dof,true,!west,expectedValue);
+ checkXFace(grid, dof, !west, expectedValue);
+ checkYFace(grid, dof, false, expectedValue);
+ checkYFace(grid, dof, true, expectedValue);
+ checkZFace(grid, dof, false, expectedValue);
+ checkZFace(grid, dof, true, expectedValue);
+}
+
+template<typename DofType,typename GridType>
+void CheckYFaceNode_Overlap(GridType &grid, DofType &dof,bool nord, typename DofType::RealType expectedValue)
+{
+ checkConner(grid,dof, false,!nord, false, expectedValue );
+ checkConner(grid,dof, false,!nord, true, expectedValue );
+ checkConner(grid,dof, true, !nord, false, expectedValue );
+ checkConner(grid,dof, true, !nord, true, expectedValue );
+ checkXDirectionEdge(grid,dof,false,!nord,expectedValue);
+ checkXDirectionEdge(grid,dof,true,!nord,expectedValue);
+ checkYDirectionEdge(grid,dof,false,false,expectedValue);
+ checkYDirectionEdge(grid,dof,false,true,expectedValue);
+ checkYDirectionEdge(grid,dof,true,false,expectedValue);
+ checkYDirectionEdge(grid,dof,true,true,expectedValue);
+ checkZDirectionEdge(grid,dof,!nord,false,expectedValue);
+ checkZDirectionEdge(grid,dof,!nord,true,expectedValue);
+ checkXFace(grid, dof, false, expectedValue);
+ checkXFace(grid, dof, true, expectedValue);
+ checkYFace(grid, dof, !nord, expectedValue);
+ checkZFace(grid, dof, false, expectedValue);
+ checkZFace(grid, dof, true, expectedValue);
+}
+
+template<typename DofType,typename GridType>
+void CheckZFaceNode_Overlap(GridType &grid, DofType &dof,bool bottom, typename DofType::RealType expectedValue)
+{
+ checkConner(grid,dof,!bottom, false, false, expectedValue );
+ checkConner(grid,dof,!bottom, false, true, expectedValue );
+ checkConner(grid,dof,!bottom, true, false, expectedValue );
+ checkConner(grid,dof,!bottom, true, true, expectedValue );
+ checkXDirectionEdge(grid,dof,!bottom,false,expectedValue);
+ checkXDirectionEdge(grid,dof,!bottom,true,expectedValue);
+ checkYDirectionEdge(grid,dof,!bottom,false,expectedValue);
+ checkYDirectionEdge(grid,dof,!bottom,true,expectedValue);
+ checkZDirectionEdge(grid,dof,false,false,expectedValue);
+ checkZDirectionEdge(grid,dof,false,true,expectedValue);
+ checkZDirectionEdge(grid,dof,true,false,expectedValue);
+ checkZDirectionEdge(grid,dof,true,true,expectedValue);
+ checkXFace(grid, dof, false, expectedValue);
+ checkXFace(grid, dof, true, expectedValue);
+ checkYFace(grid, dof, false, expectedValue);
+ checkYFace(grid, dof, true, expectedValue);
+ checkZFace(grid, dof, !bottom, expectedValue);
+}
+
+template<typename DofType,typename GridType>
+void CheckCentralNode_Overlap(GridType &grid, DofType &dof,typename DofType::RealType expectedValue)
+{
+ checkConner(grid,dof,false, false, false, expectedValue );
+ checkConner(grid,dof,false, false, true, expectedValue );
+ checkConner(grid,dof,false, true, false, expectedValue );
+ checkConner(grid,dof,false, true, true, expectedValue );
+ checkConner(grid,dof,true, false, false, expectedValue );
+ checkConner(grid,dof,true, false, true, expectedValue );
+ checkConner(grid,dof,true, true, false, expectedValue );
+ checkConner(grid,dof,true, true, true, expectedValue );
+
+ checkXDirectionEdge(grid,dof,false,false,expectedValue);
+ checkXDirectionEdge(grid,dof,false,true,expectedValue);
+ checkXDirectionEdge(grid,dof,true,false,expectedValue);
+ checkXDirectionEdge(grid,dof,true,true,expectedValue);
+ checkYDirectionEdge(grid,dof,false,false,expectedValue);
+ checkYDirectionEdge(grid,dof,false,true,expectedValue);
+ checkYDirectionEdge(grid,dof,true,false,expectedValue);
+ checkYDirectionEdge(grid,dof,true,true,expectedValue);
+ checkZDirectionEdge(grid,dof,false,false,expectedValue);
+ checkZDirectionEdge(grid,dof,false,true,expectedValue);
+ checkZDirectionEdge(grid,dof,true,false,expectedValue);
+ checkZDirectionEdge(grid,dof,true,true,expectedValue);
+
+ checkXFace(grid, dof, false, expectedValue);
+ checkXFace(grid, dof, true, expectedValue);
+ checkYFace(grid, dof, false, expectedValue);
+ checkYFace(grid, dof, true, expectedValue);
+ checkZFace(grid, dof, false, expectedValue);
+ checkZFace(grid, dof, true, expectedValue);
+}
+
+/*
+* Expected 27 procs.
+*/
+template<typename DofType,typename GridType>
+void check_Overlap_3D(int rank, GridType &grid, DofType &dof, typename DofType::RealType expectedValue)
+{
+ if(rank==0)
+ CheckConnerNode_Overlap(grid,dof,true,true,true,expectedValue);
+
+ if(rank==1)
+ CheckXEdgeNode_Overlap(grid,dof,true,true,expectedValue);
+
+ if(rank==2)
+ CheckConnerNode_Overlap(grid,dof,true,true,false,expectedValue);
+
+ if(rank==3)
+ CheckYEdgeNode_Overlap(grid,dof,true,true,expectedValue);
+
+ if(rank==4)
+ CheckZFaceNode_Overlap(grid,dof,true,expectedValue);
+
+ if(rank==5)
+ CheckYEdgeNode_Overlap(grid,dof,true,false,expectedValue);
+
+ if(rank==6)
+ CheckConnerNode_Overlap(grid,dof,true,false,true,expectedValue);
+
+ if(rank==7)
+ CheckXEdgeNode_Overlap(grid,dof,true,false,expectedValue);
+
+ if(rank==8)
+ CheckConnerNode_Overlap(grid,dof,true,false,false,expectedValue);
+
+ if(rank==9)
+ CheckZEdgeNode_Overlap(grid,dof,true,true,expectedValue);
+
+ if(rank==10)
+ CheckYFaceNode_Overlap(grid,dof,true,expectedValue);
+
+ if(rank==11)
+ CheckZEdgeNode_Overlap(grid,dof,true,false,expectedValue);
+
+ if(rank==12)
+ CheckXFaceNode_Overlap(grid,dof,true,expectedValue);
+
+ if(rank==13)
+ CheckCentralNode_Overlap(grid,dof,expectedValue);
+
+ if(rank==14)
+ CheckXFaceNode_Overlap(grid,dof,false,expectedValue);
+
+ if(rank==15)
+ CheckZEdgeNode_Overlap(grid,dof,false,true,expectedValue);
+
+ if(rank==16)
+ CheckYFaceNode_Overlap(grid,dof,false,expectedValue);
+
+ if(rank==17)
+ CheckZEdgeNode_Overlap(grid,dof,false,false,expectedValue);
+
+ if(rank==18)
+ CheckConnerNode_Overlap(grid,dof,false,true,true,expectedValue);
+
+ if(rank==19)
+ CheckXEdgeNode_Overlap(grid,dof,false,true,expectedValue);
+
+ if(rank==20)
+ CheckConnerNode_Overlap(grid,dof,false,true,false,expectedValue);
+
+ if(rank==21)
+ CheckYEdgeNode_Overlap(grid,dof,false,true,expectedValue);
+
+ if(rank==22)
+ CheckZFaceNode_Overlap(grid,dof,false,expectedValue);
+
+ if(rank==23)
+ CheckYEdgeNode_Overlap(grid,dof,false,false,expectedValue);
+
+ if(rank==24)
+ CheckConnerNode_Overlap(grid,dof,false,false,true,expectedValue);
+
+ if(rank==25)
+ CheckXEdgeNode_Overlap(grid,dof,false,false,expectedValue);
+
+ if(rank==26)
+ CheckConnerNode_Overlap(grid,dof,false,false,false,expectedValue);
+
+}
+
+template<typename DofType,typename GridType>
+void check_Inner_3D(int rank, GridType grid, DofType dof, typename DofType::RealType expectedValue)
+{
+ int maxx=grid.getDimensions().x();
+ int maxy=grid.getDimensions().y();
+ int maxz=grid.getDimensions().z();
+ for(int k=1;k<maxz-1;k++)
+ for(int j=1;j<maxy-1;j++)//prvni a posledni jsou buÄ hranice, nebo overlap
+ for(int i=1;i<maxx-1;i++) //buÄ je vlevo hranice, nebo overlap
+ EXPECT_EQ( dof[k*maxx*maxy+j*maxx+i], expectedValue) <<" "<<k <<" "<< j<<" "<<i << " " << maxx << " " << maxy<< " " << maxz;
+}
+
+
+/*
+ * Light check of 3D distributed grid and its synchronization.
+ * expected 27 processors
+ */
+typedef MpiCommunicator CommunicatorType;
+typedef Grid<3,double,Host,int> MeshType;
+typedef MeshFunction<MeshType> MeshFunctionType;
+typedef Vector<double,Host,int> DofType;
+typedef typename MeshType::Cell Cell;
+typedef typename MeshType::IndexType IndexType;
+typedef typename MeshType::PointType PointType;
+typedef DistributedMesh<MeshType> DistributedGridType;
+
+class DistributedGirdTest_3D : public ::testing::Test {
+ protected:
+
+ static DistributedGridType *distrgrid;
+ static DofType *dof;
+
+ static SharedPointer<MeshType> gridptr;
+ static SharedPointer<MeshFunctionType> meshFunctionptr;
+
+ static MeshFunctionEvaluator< MeshFunctionType, ConstFunction<double,3> > constFunctionEvaluator;
+ static SharedPointer< ConstFunction<double,3>, Host > constFunctionPtr;
+
+ static MeshFunctionEvaluator< MeshFunctionType, LinearFunction<double,3> > linearFunctionEvaluator;
+ static SharedPointer< LinearFunction<double,3>, Host > linearFunctionPtr;
+
+ static int rank;
+ static int nproc;
+
+ // Per-test-case set-up.
+ // Called before the first test in this test case.
+ // Can be omitted if not needed.
+ static void SetUpTestCase() {
+
+ int size=10;
+ rank=MPI::COMM_WORLD.Get_rank();
+ nproc=MPI::COMM_WORLD.Get_size();
+
+ PointType globalOrigin;
+ PointType globalProportions;
+ MeshType globalGrid;
+
+ globalOrigin.x()=-0.5;
+ globalOrigin.y()=-0.5;
+ globalOrigin.z()=-0.5;
+ globalProportions.x()=size;
+ globalProportions.y()=size;
+ globalProportions.z()=size;
+
+ globalGrid.setDimensions(size,size,size);
+ globalGrid.setDomain(globalOrigin,globalProportions);
+
+ typename DistributedGridType::CoordinatesType overlap;
+ overlap.setValue(1);
+ distrgrid=new DistributedGridType();
+ int distr[2];
+ distr[0]=3;
+ distr[1]=3;
+ distr[2]=3;
+ distrgrid->template setGlobalGrid<CommunicatorType>(globalGrid,overlap,distr);
+
+ distrgrid->SetupGrid(*gridptr);
+ dof=new DofType(gridptr->template getEntitiesCount< Cell >());
+
+ meshFunctionptr->bind(gridptr,*dof);
+ constFunctionPtr->Number=rank;
+
+ }
+
+ // Per-test-case tear-down.
+ // Called after the last test in this test case.
+ // Can be omitted if not needed.
+ static void TearDownTestCase() {
+ delete dof;
+ delete distrgrid;
+
+ }
+
+};
+
+DistributedGridType *DistributedGirdTest_3D::distrgrid=NULL;
+DofType *DistributedGirdTest_3D::dof=NULL;
+SharedPointer<MeshType> DistributedGirdTest_3D::gridptr;
+SharedPointer<MeshFunctionType> DistributedGirdTest_3D::meshFunctionptr;
+MeshFunctionEvaluator< MeshFunctionType, ConstFunction<double,3> > DistributedGirdTest_3D::constFunctionEvaluator;
+SharedPointer< ConstFunction<double,3>, Host > DistributedGirdTest_3D::constFunctionPtr;
+MeshFunctionEvaluator< MeshFunctionType, LinearFunction<double,3> > DistributedGirdTest_3D::linearFunctionEvaluator;
+SharedPointer< LinearFunction<double,3>, Host > DistributedGirdTest_3D::linearFunctionPtr;
+int DistributedGirdTest_3D::rank;
+int DistributedGirdTest_3D::nproc;
+
+TEST_F(DistributedGirdTest_3D, evaluateAllEntities)
+{
+
+ //Check Traversars
+ //All entities, witout overlap
+ setDof_3D(*dof,-1);
+ constFunctionEvaluator.evaluateAllEntities( meshFunctionptr , constFunctionPtr );
+ //Printer<MeshType,DofType>::print_dof(rank,*gridptr,*dof);
+ check_Boundary_3D(rank, *gridptr, *dof, rank);
+ check_Overlap_3D(rank, *gridptr, *dof, -1);
+ check_Inner_3D(rank, *gridptr, *dof, rank);
+}
+
+TEST_F(DistributedGirdTest_3D, evaluateBoundaryEntities)
+{
+ //Boundary entities, witout overlap
+ setDof_3D(*dof,-1);
+ constFunctionEvaluator.evaluateBoundaryEntities( meshFunctionptr , constFunctionPtr );
+ check_Boundary_3D(rank, *gridptr, *dof, rank);
+ check_Overlap_3D(rank, *gridptr, *dof, -1);
+ check_Inner_3D(rank, *gridptr, *dof, -1);
+}
+
+TEST_F(DistributedGirdTest_3D, evaluateInteriorEntities)
+{
+ //Inner entities, witout overlap
+ setDof_3D(*dof,-1);
+ constFunctionEvaluator.evaluateInteriorEntities( meshFunctionptr , constFunctionPtr );
+ check_Boundary_3D(rank, *gridptr, *dof, -1);
+ check_Overlap_3D(rank, *gridptr, *dof, -1);
+ check_Inner_3D(rank, *gridptr, *dof, rank);
+}
+
+TEST_F(DistributedGirdTest_3D, LinearFunctionTest)
+{
+ //fill meshfunction with linear function (physical center of cell corresponds with its coordinates in grid)
+ setDof_3D(*dof,-1);
+ linearFunctionEvaluator.evaluateAllEntities(meshFunctionptr, linearFunctionPtr);
+ meshFunctionptr->template Synchronize<CommunicatorType>();
+
+ int count =gridptr->template getEntitiesCount< Cell >();
+ for(int i=0;i<count;i++)
+ {
+ auto entity= gridptr->template getEntity< Cell >(i);
+ entity.refresh();
+ EXPECT_EQ(meshFunctionptr->getValue(entity), (*linearFunctionPtr)(entity)) << "Linear function doesnt fit recievd data. " << entity.getCoordinates().x() << " "<<entity.getCoordinates().y() << " "<< gridptr->getDimensions().x() <<" "<<gridptr->getDimensions().y();
+ }
+}
+
+/* not implemented
+TEST_F(DistributedGirdTest_3D, SynchronizerNeighborTest)
+{
+
+}
+*/
+
+#else
+TEST(NoMPI, NoTest)
+{
+ ASSERT_TRUE(true) << ":-(";
+}
+#endif
+
+#endif
+
+
+#if (defined(HAVE_GTEST) && defined(HAVE_MPI))
+#include <sstream>
+
+ class MinimalistBuffredPrinter : public ::testing::EmptyTestEventListener {
+
+ private:
+ std::stringstream sout;
+
+ public:
+
+ // Called before a test starts.
+ virtual void OnTestStart(const ::testing::TestInfo& test_info) {
+ sout<< test_info.test_case_name() <<"." << test_info.name() << " Start." <<std::endl;
+ }
+
+ // Called after a failed assertion or a SUCCEED() invocation.
+ virtual void OnTestPartResult(
+ const ::testing::TestPartResult& test_part_result) {
+ sout << (test_part_result.failed() ? "====Failure=== " : "===Success=== ")
+ << test_part_result.file_name() << " "
+ << test_part_result.line_number() <<std::endl
+ << test_part_result.summary() <<std::endl;
+ }
+
+ // Called after a test ends.
+ virtual void OnTestEnd(const ::testing::TestInfo& test_info)
+ {
+ int rank=CommunicatorType::GetRank();
+ sout<< test_info.test_case_name() <<"." << test_info.name() << " End." <<std::endl;
+ std::cout << rank << ":" << std::endl << sout.str()<< std::endl;
+ sout.str( std::string() );
+ sout.clear();
+ }
+ };
+#endif
+
+#include "../../src/UnitTests/GtestMissingError.h"
+int main( int argc, char* argv[] )
+{
+#ifdef HAVE_GTEST
+ ::testing::InitGoogleTest( &argc, argv );
+
+ #ifdef HAVE_MPI
+ ::testing::TestEventListeners& listeners =
+ ::testing::UnitTest::GetInstance()->listeners();
+
+ delete listeners.Release(listeners.default_result_printer());
+ listeners.Append(new MinimalistBuffredPrinter);
+
+ CommunicatorType::Init(argc,argv);
+ #endif
+ int result= RUN_ALL_TESTS();
+
+ #ifdef HAVE_MPI
+ CommunicatorType::Finalize();
+ #endif
+ return result;
+#else
+
+ throw GtestMissingError();
+#endif
+}
+
+
+
+
diff --git a/src/UnitTests/Mpi/Functions.h b/src/UnitTests/Mpi/Functions.h
new file mode 100644
index 0000000000000000000000000000000000000000..59a0335c15af53b880f716318e571688f593d91c
--- /dev/null
+++ b/src/UnitTests/Mpi/Functions.h
@@ -0,0 +1,212 @@
+/***************************************************************************
+ DistributedGridTest.cpp - description
+ -------------------
+ begin : Sep 6, 2017
+ copyright : (C) 2017 by Tomas Oberhuber et al.
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+#pragma once
+
+
+#include <iostream>
+#include <sstream>
+#include <TNL/Functions/MeshFunction.h>
+
+
+using namespace TNL;
+using namespace TNL::Functions;
+
+template <typename Real,
+ int Dim>
+class LinearFunction{};
+
+
+template <typename Real,
+ int Dim>
+class ConstFunction{};
+
+template < typename GridType,
+ typename DofType,
+ int Dim=GridType::getMeshDimension()>
+class Printer
+{};
+
+//=================================1D=====================================================
+template <typename Real>
+class LinearFunction<Real,1> : public Functions::Domain< 1, Functions::MeshDomain >
+{
+ public:
+ typedef Real RealType;
+ LinearFunction( )
+ {};
+
+ template< typename EntityType >
+ __cuda_callable__ RealType operator()( const EntityType& meshEntity,
+ const RealType& time = 0.0 ) const
+ {
+ return meshEntity.getCenter().x();
+
+ }
+};
+
+template <typename Real>
+class ConstFunction<Real,1> : public Functions::Domain< 1, Functions::MeshDomain >
+{
+ public:
+ typedef Real RealType;
+
+ Real Number;
+
+ ConstFunction( )
+ {};
+
+ template< typename EntityType >
+ __cuda_callable__ RealType operator()( const EntityType& meshEntity,
+ const RealType& time = 0.0 ) const
+ {
+ return Number;
+
+ }
+};
+
+template<typename GridType, typename DofType>
+class Printer< GridType,DofType,1>
+{
+ public:
+ void static print_dof(int rank, GridType grid, DofType dof)
+ {
+ std::stringstream sout;
+ for(int i=0;i<dof.getSize();i++)
+ sout<< dof[i] << " ";
+
+ std::cout << rank << ": " << sout.str() << std::endl;
+ };
+};
+
+//=================================2D======================================================
+
+template <typename Real>
+class LinearFunction<Real,2> : public Functions::Domain< 2, Functions::MeshDomain >
+{
+ public:
+ typedef Real RealType;
+ LinearFunction( )
+ {};
+
+ template< typename EntityType >
+ __cuda_callable__ RealType operator()( const EntityType& meshEntity,
+ const RealType& time = 0.0 ) const
+ {
+ //return meshEntity.getCoordinates().y()*10+meshEntity.getCoordinates().x();
+ return meshEntity.getCenter().y()*100+meshEntity.getCenter().x();
+ }
+};
+
+template <typename Real>
+class ConstFunction<Real,2> : public Functions::Domain< 2, Functions::MeshDomain >
+{
+ public:
+ typedef Real RealType;
+
+ Real Number;
+ ConstFunction( )
+ {};
+
+ template< typename EntityType >
+ __cuda_callable__ RealType operator()( const EntityType& meshEntity,
+ const RealType& time = 0.0 ) const
+ {
+ //return meshEntity.getCoordinates().y()*10+meshEntity.getCoordinates().x();
+ return this->Number;
+
+ }
+};
+
+template<typename GridType, typename DofType>
+class Printer< GridType,DofType,2>
+{
+ public:
+ void static print_dof(int rank,GridType grid, DofType dof)
+ {
+ int maxx=grid.getDimensions().x();
+ int maxy=grid.getDimensions().y();
+ std::stringstream sout;
+ sout<< rank<<":" <<std::endl;
+ for(int j=0;j<maxy;j++)
+ {
+ for(int i=0;i<maxx;i++)
+ sout<< dof[j*maxx+i] << " ";
+ sout<<std::endl;
+ }
+ std::cout << sout.str() << std::endl<< std::endl;
+ };
+};
+
+//============================3D============================================================
+template <typename Real>
+class LinearFunction<Real,3> : public Functions::Domain< 3, Functions::MeshDomain >
+{
+ public:
+ typedef Real RealType;
+ LinearFunction( )
+ {};
+
+ template< typename EntityType >
+ __cuda_callable__ RealType operator()( const EntityType& meshEntity,
+ const RealType& time = 0.0 ) const
+ {
+ //return meshEntity.getCoordinates().y()*10+meshEntity.getCoordinates().x();
+ return meshEntity.getCenter().z()*10000+meshEntity.getCenter().y()*100+meshEntity.getCenter().x();
+ }
+};
+
+template <typename Real>
+class ConstFunction<Real,3> : public Functions::Domain< 3, Functions::MeshDomain >
+{
+ public:
+ typedef Real RealType;
+
+ Real Number;
+ ConstFunction( )
+ {};
+
+ template< typename EntityType >
+ __cuda_callable__ RealType operator()( const EntityType& meshEntity,
+ const RealType& time = 0.0 ) const
+ {
+ //return meshEntity.getCoordinates().y()*10+meshEntity.getCoordinates().x();
+ return this->Number;
+
+ }
+};
+
+template<typename GridType, typename DofType>
+class Printer< GridType,DofType,3>
+{
+ public:
+ static void print_dof(int rank,GridType grid, DofType dof)
+ {
+ //print local dof
+ int maxx=grid.getDimensions().x();
+ int maxy=grid.getDimensions().y();
+ int maxz=grid.getDimensions().z();
+
+ std::stringstream sout;
+ sout<< rank<<":" <<std::endl;
+ for(int k=0;k<maxz;k++)
+ {
+ for(int j=0;j<maxy;j++)
+ {
+ for(int ii=0;ii<k;ii++)
+ sout<<" ";
+ for(int i=0;i<maxx;i++)
+ {
+ sout <<dof[k*maxx*maxy+maxx*j+i]<<" ";
+ }
+ sout << std::endl;
+ }
+ }
+ std::cout << sout.str()<< std::endl<<std::endl;
+ };
+};
diff --git a/src/UnitTests/Mpi/GPUDistributedGridIOTest.cu b/src/UnitTests/Mpi/GPUDistributedGridIOTest.cu
new file mode 100644
index 0000000000000000000000000000000000000000..dfc57f86cf715d6b9f4d5eb02336388bf58fe897
--- /dev/null
+++ b/src/UnitTests/Mpi/GPUDistributedGridIOTest.cu
@@ -0,0 +1,110 @@
+
+#ifdef HAVE_GTEST
+
+#include <gtest/gtest.h>
+
+#ifdef HAVE_MPI
+
+#include "DistributedGridIOTest.h"
+
+TEST( DistributedGridIO, Save_1D )
+{
+ TestDistributedGridIO<1,Cuda>::TestSave();
+}
+
+TEST( DistributedGridIO, Save_2D )
+{
+ TestDistributedGridIO<2,Cuda>::TestSave();
+}
+
+TEST( DistributedGridIO, Save_3D )
+{
+ TestDistributedGridIO<3,Cuda>::TestSave();
+}
+
+TEST( DistributedGridIO, Load_1D )
+{
+ TestDistributedGridIO<1,Cuda>::TestLoad();
+}
+
+TEST( DistributedGridIO, Load_2D )
+{
+ TestDistributedGridIO<2,Cuda>::TestLoad();
+}
+
+TEST( DistributedGridIO, Load_3D )
+{
+ TestDistributedGridIO<3,Cuda>::TestLoad();
+}
+
+#else
+TEST(NoMPI, NoTest)
+{
+ ASSERT_TRUE(true) << ":-(";
+}
+#endif
+
+#endif
+
+#if (defined(HAVE_GTEST) && defined(HAVE_MPI))
+#include <sstream>
+
+ class MinimalistBuffredPrinter : public ::testing::EmptyTestEventListener {
+
+ private:
+ std::stringstream sout;
+
+ public:
+
+ // Called before a test starts.
+ virtual void OnTestStart(const ::testing::TestInfo& test_info) {
+ sout<< test_info.test_case_name() <<"." << test_info.name() << " Start." <<std::endl;
+ }
+
+ // Called after a failed assertion or a SUCCEED() invocation.
+ virtual void OnTestPartResult(
+ const ::testing::TestPartResult& test_part_result) {
+ sout << (test_part_result.failed() ? "====Failure=== " : "===Success=== ")
+ << test_part_result.file_name() << " "
+ << test_part_result.line_number() <<std::endl
+ << test_part_result.summary() <<std::endl;
+ }
+
+ // Called after a test ends.
+ virtual void OnTestEnd(const ::testing::TestInfo& test_info)
+ {
+ int rank=CommunicatorType::GetRank();
+ sout<< test_info.test_case_name() <<"." << test_info.name() << " End." <<std::endl;
+ std::cout << rank << ":" << std::endl << sout.str()<< std::endl;
+ sout.str( std::string() );
+ sout.clear();
+ }
+ };
+#endif
+
+#include "../GtestMissingError.h"
+int main( int argc, char* argv[] )
+{
+#ifdef HAVE_GTEST
+ ::testing::InitGoogleTest( &argc, argv );
+
+ #ifdef HAVE_MPI
+ ::testing::TestEventListeners& listeners =
+ ::testing::UnitTest::GetInstance()->listeners();
+
+ delete listeners.Release(listeners.default_result_printer());
+ listeners.Append(new MinimalistBuffredPrinter);
+
+ CommunicatorType::Init(argc,argv);
+ #endif
+ int result= RUN_ALL_TESTS();
+
+ #ifdef HAVE_MPI
+ CommunicatorType::Finalize();
+ #endif
+ return result;
+#else
+
+ throw GtestMissingError();
+#endif
+}
diff --git a/src/UnitTests/SaveAndLoadMeshfunctionTest.cpp b/src/UnitTests/SaveAndLoadMeshfunctionTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..77dbc74663330dd2d8d4d7fa32576d99c933c485
--- /dev/null
+++ b/src/UnitTests/SaveAndLoadMeshfunctionTest.cpp
@@ -0,0 +1,125 @@
+/***************************************************************************
+ SaveAndLoadMeshfunctionTest.cpp - description
+ -------------------
+ begin : Dec 2, 2017
+ copyright : (C) 2017 by Tomas Oberhuber et al.
+ email : tomas.oberhuber@fjfi.cvut.cz
+ ***************************************************************************/
+
+#include <TNL/Functions/MeshFunction.h>
+
+#ifdef HAVE_GTEST
+#include <gtest/gtest.h>
+
+#include "Mpi/Functions.h"
+
+#include <iostream>
+
+using namespace TNL::Containers;
+using namespace TNL::Meshes;
+using namespace TNL::Functions;
+using namespace TNL::Devices;
+
+
+#define FILENAME "./test-file.tnl"
+
+//=====================================TEST CLASS==============================================
+
+template <int dim>
+class TestSaveAndLoadMeshfunction
+{
+ public:
+ static void Test()
+ {
+
+ typedef Grid<dim,double,Host,int> MeshType;
+ typedef MeshFunction<MeshType> MeshFunctionType;
+ typedef Vector<double,Host,int> DofType;
+ typedef typename MeshType::Cell Cell;
+ typedef typename MeshType::IndexType IndexType;
+ typedef typename MeshType::PointType PointType;
+
+ typedef typename MeshType::CoordinatesType CoordinatesType;
+ typedef LinearFunction<double,dim> LinearFunctionType;
+
+ SharedPointer< LinearFunctionType, Host > linearFunctionPtr;
+ MeshFunctionEvaluator< MeshFunctionType, LinearFunctionType > linearFunctionEvaluator;
+
+
+ PointType localOrigin;
+ localOrigin.setValue(-0.5);
+ PointType localProportions;
+ localProportions.setValue(10);
+
+ SharedPointer<MeshType> localGridptr;
+ localGridptr->setDimensions(localProportions);
+ localGridptr->setDomain(localOrigin,localProportions);
+
+ DofType localDof(localGridptr->template getEntitiesCount< Cell >());
+
+ SharedPointer<MeshFunctionType> localMeshFunctionptr;
+ localMeshFunctionptr->bind(localGridptr,localDof);
+ linearFunctionEvaluator.evaluateAllEntities(localMeshFunctionptr , linearFunctionPtr);
+
+ File file;
+ ASSERT_TRUE( file.open( String( FILENAME), IOMode::write ));
+ ASSERT_TRUE( localMeshFunctionptr->save(file));
+ ASSERT_TRUE( file.close() );
+
+ //load other meshfunction on same localgrid from created file
+ SharedPointer<MeshType> loadGridptr;
+ loadGridptr->setDimensions(localProportions);
+ loadGridptr->setDomain(localOrigin,localProportions);
+
+ DofType loadDof(loadGridptr->template getEntitiesCount< Cell >());
+ SharedPointer<MeshFunctionType> loadMeshFunctionptr;
+ loadMeshFunctionptr->bind(loadGridptr,loadDof);
+
+ for(int i=0;i<loadDof.getSize();i++)
+ {
+ loadDof[i]=-1;
+ }
+
+ ASSERT_TRUE( file.open( String( FILENAME ), IOMode::read ));
+ ASSERT_TRUE( loadMeshFunctionptr->boundLoad(file));
+ ASSERT_TRUE( file.close());
+
+ for(int i=0;i<localDof.getSize();i++)
+ {
+ EXPECT_EQ( localDof[i], loadDof[i]) << "Compare Loaded and evaluated Dof Failed for: "<< i;
+ }
+
+ EXPECT_EQ( std::remove( FILENAME ), 0 );
+
+ };
+};
+
+TEST( CopyEntitiesTest, 1D )
+{
+ TestSaveAndLoadMeshfunction<1>::Test();
+}
+
+TEST( CopyEntitiesTest, 2D )
+{
+ TestSaveAndLoadMeshfunction<2>::Test();
+}
+
+TEST( CopyEntitiesTest, 3D )
+{
+ TestSaveAndLoadMeshfunction<3>::Test();
+}
+
+
+#endif
+
+#include "GtestMissingError.h"
+int main( int argc, char* argv[] )
+{
+#ifdef HAVE_GTEST
+ ::testing::InitGoogleTest( &argc, argv );
+ return RUN_ALL_TESTS();
+#else
+ throw GtestMissingError();
+#endif
+}
+
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 22266594a9b622be6d0345b3d8ec92a33c979878..21090988a1322a52fdb10006ef413e95db0504a7 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -5,5 +5,6 @@ ADD_SUBDIRECTORY( benchmarks )
#ADD_SUBDIRECTORY( unit-tests )
ADD_SUBDIRECTORY( long-time-unit-tests )
ADD_SUBDIRECTORY( mic )
+ADD_SUBDIRECTORY( mpi )
-unset( ENABLE_CODECOVERAGE )
\ No newline at end of file
+unset( ENABLE_CODECOVERAGE )
diff --git a/tests/benchmarks/heat-equation-benchmark/HeatEquationBenchmarkProblem.h b/tests/benchmarks/heat-equation-benchmark/HeatEquationBenchmarkProblem.h
index 25d6eb81ba931c38b222a81ff120b78d3ff07b6d..7eb8c3839cc7b88e06414c2aeba35a95a2594fe7 100644
--- a/tests/benchmarks/heat-equation-benchmark/HeatEquationBenchmarkProblem.h
+++ b/tests/benchmarks/heat-equation-benchmark/HeatEquationBenchmarkProblem.h
@@ -11,7 +11,8 @@ using namespace TNL::Problems;
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
- typename DifferentialOperator >
+ typename CommType,
+ typename DifferentialOperator >
class HeatEquationBenchmarkProblem:
public PDEProblem< Mesh,
typename DifferentialOperator::RealType,
@@ -30,6 +31,7 @@ class HeatEquationBenchmarkProblem:
typedef SharedPointer< BoundaryCondition > BoundaryConditionPointer;
typedef SharedPointer< RightHandSide, DeviceType > RightHandSidePointer;
+ typedef CommType CommunicatorType;
using typename BaseType::MeshType;
using typename BaseType::MeshPointer;
diff --git a/tests/benchmarks/heat-equation-benchmark/HeatEquationBenchmarkProblem_impl.h b/tests/benchmarks/heat-equation-benchmark/HeatEquationBenchmarkProblem_impl.h
index 573eab7885f83c8ffb2f26a6a4438b3129258099..128dd9efd05243e903863e0b38f121ef27dd4676 100644
--- a/tests/benchmarks/heat-equation-benchmark/HeatEquationBenchmarkProblem_impl.h
+++ b/tests/benchmarks/heat-equation-benchmark/HeatEquationBenchmarkProblem_impl.h
@@ -10,9 +10,10 @@
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
String
-HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getTypeStatic()
{
return String( "HeatEquationBenchmarkProblem< " ) + Mesh :: getTypeStatic() + " >";
@@ -21,8 +22,9 @@ getTypeStatic()
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
-HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
HeatEquationBenchmarkProblem()
: cudaMesh( 0 ),
cudaBoundaryConditions( 0 ),
@@ -34,9 +36,10 @@ HeatEquationBenchmarkProblem()
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
String
-HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getPrologHeader() const
{
return String( "Heat Equation Benchmark" );
@@ -45,9 +48,10 @@ getPrologHeader() const
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
void
-HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
writeProlog( Logger& logger, const Config::ParameterContainer& parameters ) const
{
/****
@@ -59,9 +63,10 @@ writeProlog( Logger& logger, const Config::ParameterContainer& parameters ) cons
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
bool
-HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
setup( const MeshPointer& meshPointer,
const Config::ParameterContainer& parameters,
const String& prefix )
@@ -84,9 +89,10 @@ setup( const MeshPointer& meshPointer,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
-typename HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::IndexType
-HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+typename HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::IndexType
+HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getDofs( const MeshPointer& meshPointer ) const
{
/****
@@ -99,9 +105,10 @@ getDofs( const MeshPointer& meshPointer ) const
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
void
-HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
bindDofs( const MeshPointer& meshPointer,
DofVectorPointer& dofsPointer )
{
@@ -110,9 +117,10 @@ bindDofs( const MeshPointer& meshPointer,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
bool
-HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
setInitialCondition( const Config::ParameterContainer& parameters,
const MeshPointer& meshPointer,
DofVectorPointer& dofsPointer,
@@ -131,10 +139,11 @@ setInitialCondition( const Config::ParameterContainer& parameters,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
template< typename Matrix >
bool
-HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
setupLinearSystem( const MeshType& mesh,
Matrix& matrix )
{
@@ -157,9 +166,10 @@ setupLinearSystem( const MeshType& mesh,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
bool
-HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
makeSnapshot( const RealType& time,
const IndexType& step,
const MeshPointer& meshPointer,
@@ -324,6 +334,7 @@ template< typename GridType,
typename GridEntity,
typename DifferentialOperator,
typename RightHandSide,
+ typename CommType,
typename MeshFunction >
__global__ void
heatEquationTemplatedCompact( const GridType* grid,
@@ -372,9 +383,10 @@ heatEquationTemplatedCompact( const GridType* grid,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
void
-HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
getExplicitUpdate( const RealType& time,
const RealType& tau,
const MeshPointer& mesh,
@@ -528,7 +540,7 @@ getExplicitUpdate( const RealType& time,
explicitUpdater.setBoundaryConditions( this->boundaryConditionPointer );
explicitUpdater.setRightHandSide( this->rightHandSidePointer );
- this->explicitUpdater.template update< typename Mesh::Cell >( time, tau, mesh, this->u, this->fu );
+ this->explicitUpdater.template update< typename Mesh::Cell, CommunicatorType >( time, tau, mesh, this->u, this->fu );
}
}
}
@@ -536,10 +548,11 @@ getExplicitUpdate( const RealType& time,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
template< typename MatrixPointer >
void
-HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
assemblyLinearSystem( const RealType& time,
const RealType& tau,
const MeshPointer& mesh,
@@ -569,8 +582,9 @@ assemblyLinearSystem( const RealType& time,
template< typename Mesh,
typename BoundaryCondition,
typename RightHandSide,
+ typename CommType,
typename DifferentialOperator >
-HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, DifferentialOperator >::
+HeatEquationBenchmarkProblem< Mesh, BoundaryCondition, RightHandSide, CommType, DifferentialOperator >::
~HeatEquationBenchmarkProblem()
{
if( this->cudaMesh ) Devices::Cuda::freeFromDevice( this->cudaMesh );
diff --git a/tests/benchmarks/heat-equation-benchmark/tnl-benchmark-heat-equation.h b/tests/benchmarks/heat-equation-benchmark/tnl-benchmark-heat-equation.h
index 3db5347a724356c11cf9afe8407ad7cacb5d4300..758003470a0167d52ccfe1b14c3654682a6240dd 100644
--- a/tests/benchmarks/heat-equation-benchmark/tnl-benchmark-heat-equation.h
+++ b/tests/benchmarks/heat-equation-benchmark/tnl-benchmark-heat-equation.h
@@ -48,7 +48,8 @@ template< typename Real,
typename Index,
typename MeshType,
typename ConfigTag,
- typename SolverStarter >
+ typename SolverStarter,
+ typename CommunicatorType >
class HeatEquationBenchmarkSetter
{
public:
@@ -76,12 +77,12 @@ class HeatEquationBenchmarkSetter
if( boundaryConditionsType == "dirichlet" )
{
typedef Operators::DirichletBoundaryConditions< MeshType, Constant, MeshType::getMeshDimension(), Real, Index > BoundaryConditions;
- typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;
+ typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, CommunicatorType, ApproximateOperator > Problem;
SolverStarter solverStarter;
return solverStarter.template run< Problem >( parameters );
}
typedef Operators::NeumannBoundaryConditions< MeshType, Constant, Real, Index > BoundaryConditions;
- typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;
+ typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, CommunicatorType, ApproximateOperator > Problem;
SolverStarter solverStarter;
return solverStarter.template run< Problem >( parameters );
}
@@ -89,12 +90,12 @@ class HeatEquationBenchmarkSetter
if( boundaryConditionsType == "dirichlet" )
{
typedef Operators::DirichletBoundaryConditions< MeshType, MeshFunction, MeshType::getMeshDimension(), Real, Index > BoundaryConditions;
- typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;
+ typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, CommunicatorType, ApproximateOperator > Problem;
SolverStarter solverStarter;
return solverStarter.template run< Problem >( parameters );
}
typedef Operators::NeumannBoundaryConditions< MeshType, MeshFunction, Real, Index > BoundaryConditions;
- typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, ApproximateOperator > Problem;
+ typedef HeatEquationBenchmarkProblem< MeshType, BoundaryConditions, RightHandSide, CommunicatorType, ApproximateOperator > Problem;
SolverStarter solverStarter;
return solverStarter.template run< Problem >( parameters );
}
diff --git a/tests/mpi/CMakeLists.txt b/tests/mpi/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0be5708088fcf742181c0ef3c0061484883841e5
--- /dev/null
+++ b/tests/mpi/CMakeLists.txt
@@ -0,0 +1,53 @@
+ ADD_EXECUTABLE( tnlMeshFuncttionEvaluateTest ${headers} MeshFunctionEvauateTest.cpp )
+ TARGET_LINK_LIBRARIES( tnlMeshFuncttionEvaluateTest ${CPPUNIT_LIBRARIES}
+ tnl )
+ TARGET_COMPILE_DEFINITIONS( tnlMeshFuncttionEvaluateTest PUBLIC "-DDIMENSION=2" )
+ TARGET_COMPILE_DEFINITIONS( tnlMeshFuncttionEvaluateTest PUBLIC "-DXDISTR -DYDISTR" )
+
+ ADD_EXECUTABLE( tnlMeshFuncttionEvaluateTestX ${headers} MeshFunctionEvauateTest.cpp )
+ TARGET_LINK_LIBRARIES( tnlMeshFuncttionEvaluateTestX ${CPPUNIT_LIBRARIES}
+ tnl )
+ TARGET_COMPILE_DEFINITIONS( tnlMeshFuncttionEvaluateTestX PUBLIC "-DDIMENSION=2" )
+ TARGET_COMPILE_DEFINITIONS( tnlMeshFuncttionEvaluateTestX PUBLIC "-DXDISTR" )
+
+ ADD_EXECUTABLE( tnlMeshFuncttionEvaluateTestY ${headers} MeshFunctionEvauateTest.cpp )
+ TARGET_LINK_LIBRARIES( tnlMeshFuncttionEvaluateTestY ${CPPUNIT_LIBRARIES}
+ tnl )
+ TARGET_COMPILE_DEFINITIONS( tnlMeshFuncttionEvaluateTestY PUBLIC "-DDIMENSION=2" )
+ TARGET_COMPILE_DEFINITIONS( tnlMeshFuncttionEvaluateTestY PUBLIC "-DYDISTR" )
+
+
+ ADD_EXECUTABLE( mpiio-save-test ${headers} mpiio-save-test.cpp )
+ TARGET_LINK_LIBRARIES( mpiio-save-test ${CPPUNIT_LIBRARIES}
+ tnl )
+
+ ADD_EXECUTABLE( mpiio-save-load-test ${headers} mpiio-save-load-test.cpp )
+ TARGET_LINK_LIBRARIES( mpiio-save-load-test ${CPPUNIT_LIBRARIES}
+ tnl )
+
+
+ #ADD_EXECUTABLE( tnlMeshFuncttionEvaluateTestZ ${headers} MeshFunctionEvauateTest.cpp )
+ #TARGET_LINK_LIBRARIES( tnlMeshFuncttionEvaluateTestZ ${CPPUNIT_LIBRARIES}
+ # tnl${mpiExt}-0.1 )
+ #TARGET_COMPILE_DEFINITIONS( tnlMeshFuncttionEvaluateTestZ PUBLIC "-DDIMENSION=3" )
+ #TARGET_COMPILE_DEFINITIONS( tnlMeshFuncttionEvaluateTestZ PUBLIC "-DZDISTR" )
+
+ #ADD_EXECUTABLE( tnlMeshFuncttionEvaluateTestXY ${headers} MeshFunctionEvauateTest.cpp )
+ #TARGET_LINK_LIBRARIES( tnlMeshFuncttionEvaluateTestXY ${CPPUNIT_LIBRARIES}
+ # tnl${mpiExt}-0.1 )
+ #TARGET_COMPILE_DEFINITIONS( tnlMeshFuncttionEvaluateTestXY PUBLIC "-DDIMENSION=3" )
+ #TARGET_COMPILE_DEFINITIONS( tnlMeshFuncttionEvaluateTestXY PUBLIC "-DXDISTR -DYDISTR" )
+
+IF( BUILD_CUDA )
+ CUDA_ADD_EXECUTABLE( mpi-gpu-test ${headers} mpi-gpu.cu )
+ TARGET_LINK_LIBRARIES( mpi-gpu-test ${CPPUNIT_LIBRARIES}
+ tnl )
+
+ CUDA_ADD_EXECUTABLE( GPUmeshFunctionEvaluateTest ${headers} GPUmeshFuncitonEvaluateTest.cu )
+ TARGET_LINK_LIBRARIES( GPUmeshFunctionEvaluateTest ${CPPUNIT_LIBRARIES}
+ tnl )
+
+ CUDA_ADD_EXECUTABLE( mpiio-save-test-gpu ${headers} mpiio-save-test.cu )
+ TARGET_LINK_LIBRARIES( mpiio-save-test-gpu ${CPPUNIT_LIBRARIES}
+ tnl )
+ENDIF( BUILD_CUDA )
diff --git a/tests/mpi/GPUmeshFuncitonEvaluateTest.cu b/tests/mpi/GPUmeshFuncitonEvaluateTest.cu
new file mode 100644
index 0000000000000000000000000000000000000000..b421dc9c6300074a0dc390be89888341b79fbc5d
--- /dev/null
+++ b/tests/mpi/GPUmeshFuncitonEvaluateTest.cu
@@ -0,0 +1,182 @@
+#include <iostream>
+
+
+#if defined(HAVE_MPI) && defined(HAVE_CUDA)
+
+#include <TNL/Containers/Array.h>
+#include <TNL/Meshes/Grid.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+#include <TNL/Communicators/MpiCommunicator.h>
+#include <TNL/Communicators/NoDistrCommunicator.h>
+#include <TNL/Functions/MeshFunction.h>
+
+#include <TNL/Timer.h>
+#include <TNL/SharedPointer.h>
+
+using namespace std;
+
+#define DIMENSION 3
+//#define OUTPUT
+#define XDISTR
+//#define YDISTR
+//#define ZDISTR
+
+#include "../../src/UnitTests/Mpi/Functions.h"
+
+using namespace TNL;
+using namespace TNL::Containers;
+using namespace TNL::Meshes;
+using namespace TNL::Meshes::DistributedMeshes;
+using namespace TNL::Communicators;
+using namespace TNL::Functions;
+using namespace TNL::Devices;
+
+
+int main ( int argc, char *argv[])
+{
+
+ Timer all,setup,eval,sync;
+
+ typedef Cuda Device;
+ //typedef Host Device;
+
+ typedef MpiCommunicator CommunicatorType;
+ //typedef NoDistrCommunicator CommType;
+ typedef Grid<DIMENSION, double,Device,int> MeshType;
+ typedef MeshFunction<MeshType> MeshFunctionType;
+ typedef Vector<double,Device,int> DofType;
+ typedef typename MeshType::Cell Cell;
+ typedef typename MeshType::IndexType IndexType;
+ typedef typename MeshType::PointType PointType;
+
+ typedef DistributedMesh<MeshType> DistributedMeshType;
+
+ typedef LinearFunction<double,DIMENSION> LinearFunctionType;
+ typedef ConstFunction<double,DIMENSION> ConstFunctionType;
+
+ CommunicatorType::Init(argc,argv);
+
+ int size=10;
+ int cycles=1;
+ if(argc==3)
+ {
+ size=strtol(argv[1],NULL,10);
+ cycles=strtol(argv[2],NULL,10);
+ //cout << "size: "<< size <<"cycles: "<< cycles <<endl;
+ }
+
+ all.start();
+ setup.start();
+
+ PointType globalOrigin;
+ globalOrigin.setValue(-0.5);
+
+ PointType globalProportions;
+ globalProportions.setValue(size);
+
+
+ MeshType globalGrid;
+ globalGrid.setDimensions(globalProportions);
+ globalGrid.setDomain(globalOrigin,globalProportions);
+
+
+ int distr[DIMENSION];
+ for(int i=0;i<DIMENSION;i++)
+ distr[i]=1;
+
+ #ifdef XDISTR
+ distr[0]=0;
+ #endif
+
+ #ifdef YDISTR
+ distr[1]=0;
+ #endif
+
+ #ifdef ZDISTR
+ distr[2]=0;
+ #endif
+
+ typename MeshType::CoordinatesType overlap;
+ overlap.setValue(1);
+ DistributedMeshType distrgrid;
+ distrgrid.template setGlobalGrid<CommunicatorType>(globalGrid,overlap, distr);
+
+ SharedPointer<MeshType> gridptr;
+ SharedPointer<MeshFunctionType> meshFunctionptr;
+ MeshFunctionEvaluator< MeshFunctionType, LinearFunctionType > linearFunctionEvaluator;
+ MeshFunctionEvaluator< MeshFunctionType, ConstFunctionType > constFunctionEvaluator;
+
+ distrgrid.SetupGrid(*gridptr);
+
+ DofType dof(gridptr->template getEntitiesCount< Cell >());
+
+ dof.setValue(0);
+
+ meshFunctionptr->bind(gridptr,dof);
+
+ SharedPointer< LinearFunctionType, Device > linearFunctionPtr;
+ SharedPointer< ConstFunctionType, Device > constFunctionPtr;
+
+ setup.stop();
+
+ double sum=0.0;
+
+ constFunctionPtr->Number=CommunicatorType::GetRank();
+
+ for(int i=0;i<cycles;i++)
+ {
+ eval.start();
+
+ constFunctionEvaluator.evaluateAllEntities( meshFunctionptr , constFunctionPtr );
+ //linearFunctionEvaluator.evaluateAllEntities(meshFunctionptr , linearFunctionPtr);
+ CommunicatorType::Barrier();
+ eval.stop();
+
+ sync.start();
+ meshFunctionptr->template Synchronize<CommunicatorType>();
+ CommunicatorType::Barrier();
+ sync.stop();
+
+ ///sum+=dof[gridptr->getDimensions().x()/2]; //dummy acces to array
+ }
+ all.stop();
+
+#ifdef OUTPUT
+ //print local dof
+ Printer<MeshType,DofType>::print_dof(CommunicatorType::GetRank(),*gridptr, dof);
+#endif
+
+ if(CommunicatorType::GetRank()==0)
+ {
+ cout << sum <<endl<<endl;
+
+ cout<<"distr: ";
+ cout << distrgrid.printProcessDistr().getString();
+ cout << endl;
+
+ cout<<"setup: "<<setup.getRealTime() <<endl;
+ cout<<"evalpercycle: "<<eval.getRealTime()/cycles<<endl;
+ cout<<"syncpercycle: "<<sync.getRealTime()/cycles<<endl;
+ cout <<"eval: "<<eval.getRealTime()<<endl;
+ cout <<"sync: "<<sync.getRealTime()<<endl;
+ cout<<"all: "<<all.getRealTime()<<endl<<endl;
+ }
+
+
+ CommunicatorType::Finalize();
+
+ return 0;
+
+}
+
+#else
+
+using namespace std;
+
+int main(void)
+{
+ cout << "MPI or Cuda missing...." <<endl;
+}
+#endif
+
+
diff --git a/tests/mpi/MPI-Test.cpp b/tests/mpi/MPI-Test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e6ef254d377921ca8546abb5cd0886e51a7906c9
--- /dev/null
+++ b/tests/mpi/MPI-Test.cpp
@@ -0,0 +1,32 @@
+/*
+* Experimentálnà test pro implementaci MPI do TNL
+ */
+
+#include <iostream>
+#include <mpi.h>
+
+unsigned int errors=0;
+unsigned int success=0;
+#define TEST_TEST(a) if((a)){cout << __LINE__ <<":\t OK " <<endl;success++;}else{cout << __LINE__<<":\t FAIL" <<endl;errors++;}
+#define TEST_RESULT cout<<"SUCCES: "<<success<<endl<<"ERRRORS: "<<errors<<endl;
+inline void Test_Say( const char * message)
+{
+#ifdef TEST_VERBOSE
+ cout << message <<endl;
+#endif
+}
+
+using namespace std;
+
+int main ( int argc, char *argv[])
+{
+ int rank, size;
+
+ MPI_Init (&argc, &argv); /* starts MPI */
+ MPI_Comm_rank (MPI_COMM_WORLD, &rank); /* get current process id */
+ MPI_Comm_size (MPI_COMM_WORLD, &size); /* get number of processes */
+ cout << "Hello world from process " << rank <<" of " << size << endl;
+ MPI_Finalize();
+ return 0;
+}
+
diff --git a/tests/mpi/MeshFunctionEvauateTest.cpp b/tests/mpi/MeshFunctionEvauateTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8aa6fee8ab142f7e79041405fca4ce517038dacc
--- /dev/null
+++ b/tests/mpi/MeshFunctionEvauateTest.cpp
@@ -0,0 +1,183 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+
+#include <iostream>
+#include <sstream>
+
+using namespace std;
+
+
+#ifdef HAVE_MPI
+
+#include <TNL/Containers/Array.h>
+#include <TNL/Meshes/Grid.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+#include <TNL/Communicators/MpiCommunicator.h>
+#include <TNL/Communicators/NoDistrCommunicator.h>
+#include <TNL/Functions/MeshFunction.h>
+
+#include <TNL/Timer.h>
+#include <TNL/SharedPointer.h>
+
+//#define DIMENSION 3
+//#define OUTPUT
+//#define XDISTR
+//#define YDISTR
+//#define ZDISTR
+
+#include "../../src/UnitTests/Mpi/Functions.h"
+
+using namespace TNL;
+using namespace TNL::Containers;
+using namespace TNL::Meshes;
+using namespace TNL::Meshes::DistributedMeshes;
+using namespace TNL::Communicators;
+using namespace TNL::Functions;
+using namespace TNL::Devices;
+#endif
+
+int main ( int argc, char *argv[])
+{
+
+#ifdef HAVE_MPI
+ Timer all,setup,eval,sync;
+
+
+ typedef MpiCommunicator CommunicatorType;
+ //typedef NoDistrCommunicator CommType;
+ typedef Grid<DIMENSION, double,Host,int> MeshType;
+ typedef MeshFunction<MeshType> MeshFunctionType;
+ typedef Vector<double,Host,int> DofType;
+ typedef typename MeshType::Cell Cell;
+ typedef typename MeshType::IndexType IndexType;
+ typedef typename MeshType::PointType PointType;
+
+ typedef DistributedMesh<MeshType> DistributedMeshType;
+
+ typedef LinearFunction<double,DIMENSION> LinearFunctionType;
+ typedef ConstFunction<double,DIMENSION> ConstFunctionType;
+
+ CommunicatorType::Init(argc,argv);
+
+ int size=9;
+ int cycles=1;
+ if(argc==3)
+ {
+ size=strtol(argv[1],NULL,10);
+ cycles=strtol(argv[2],NULL,10);
+ //cout << "size: "<< size <<"cycles: "<< cycles <<endl;
+ }
+
+ all.start();
+ setup.start();
+
+ PointType globalOrigin;
+ globalOrigin.setValue(-0.5);
+
+ PointType globalProportions;
+ globalProportions.setValue(size);
+
+
+ MeshType globalGrid;
+ globalGrid.setDimensions(globalProportions);
+ globalGrid.setDomain(globalOrigin,globalProportions);
+
+
+ int distr[DIMENSION];
+ for(int i=0;i<DIMENSION;i++)
+ distr[i]=1;
+
+ #ifdef XDISTR
+ distr[0]=0;
+ #endif
+
+ #ifdef YDISTR
+ distr[1]=0;
+ #endif
+
+ #ifdef ZDISTR
+ distr[2]=0;
+ #endif
+
+ typename MeshType::CoordinatesType overlap;
+ overlap.setValue(1);
+ DistributedMeshType distrgrid;
+ distrgrid.template setGlobalGrid<CommunicatorType>(globalGrid,overlap, distr);
+
+ SharedPointer<MeshType> gridptr;
+ SharedPointer<MeshFunctionType> meshFunctionptr;
+ MeshFunctionEvaluator< MeshFunctionType, LinearFunctionType > linearFunctionEvaluator;
+ MeshFunctionEvaluator< MeshFunctionType, ConstFunctionType > constFunctionEvaluator;
+
+ distrgrid.SetupGrid(*gridptr);
+
+ DofType dof(gridptr->template getEntitiesCount< Cell >());
+
+ dof.setValue(0);
+
+ meshFunctionptr->bind(gridptr,dof);
+
+ SharedPointer< LinearFunctionType, Host > linearFunctionPtr;
+ SharedPointer< ConstFunctionType, Host > constFunctionPtr;
+
+
+
+ setup.stop();
+
+ double sum=0.0;
+
+ constFunctionPtr->Number=CommunicatorType::GetRank();
+
+ for(int i=0;i<cycles;i++)
+ {
+ eval.start();
+
+ constFunctionEvaluator.evaluateAllEntities( meshFunctionptr , constFunctionPtr );
+ //linearFunctionEvaluator.evaluateAllEntities(meshFunctionptr , linearFunctionPtr);
+ CommunicatorType::Barrier();
+ eval.stop();
+
+ sync.start();
+ meshFunctionptr->template Synchronize<CommunicatorType>();
+ CommunicatorType::Barrier();
+ sync.stop();
+
+ sum+=dof[gridptr->getDimensions().x()/2]; //dummy acces to array
+ }
+ all.stop();
+
+#ifdef OUTPUT
+ //print local dof
+ Printer<MeshType,DofType>::print_dof(CommunicatorType::GetRank(),*gridptr, dof);
+#endif
+
+ if(CommunicatorType::GetRank()==0)
+ {
+ cout << sum <<endl<<endl;
+
+ cout<<"distr: ";
+ cout << distrgrid.printProcessDistr().getString();
+ cout << endl;
+
+ cout<<"setup: "<<setup.getRealTime() <<endl;
+ cout<<"evalpercycle: "<<eval.getRealTime()/cycles<<endl;
+ cout<<"syncpercycle: "<<sync.getRealTime()/cycles<<endl;
+ cout <<"eval: "<<eval.getRealTime()<<endl;
+ cout <<"sync: "<<sync.getRealTime()<<endl;
+ cout<<"all: "<<all.getRealTime()<<endl<<endl;
+ }
+
+
+ CommunicatorType::Finalize();
+
+
+
+#else
+ std::cout<<"MPI not Supported." << std::endl;
+#endif
+ return 0;
+
+}
diff --git a/tests/mpi/mpi-gpu.cu b/tests/mpi/mpi-gpu.cu
new file mode 100644
index 0000000000000000000000000000000000000000..e136e6bead8350ef8dedb1cab364310f117234cb
--- /dev/null
+++ b/tests/mpi/mpi-gpu.cu
@@ -0,0 +1,79 @@
+#include <iostream>
+
+using namespace std;
+
+
+#if defined(HAVE_MPI) && defined(HAVE_CUDA)
+
+#include <cuda_runtime.h>
+#include <mpi.h>
+
+__global__ void SetKernel(float *deviceData, float value)
+{
+ // Just a dummy kernel
+ const int idx = blockIdx.x * blockDim.x + threadIdx.x;
+ deviceData[idx] = value;
+}
+
+double sum(float * data, int count)
+{
+ double sum=0;
+ for(int i=0;i<count;i++)
+ sum+=data[i];
+
+ return sum;
+}
+
+
+int main(int argc, char **argv)
+{
+ MPI_Init(&argc, &argv);
+
+ int rank;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+ int blockSize = 256; //poÄet threadů v v bloku
+ int gridSize = 1000; //poÄet bloků v gridu -> musà být menÅ¡Ã než maxGridSize v cudeGetDeviceProperties
+
+ int dataCount=blockSize*gridSize;
+
+ float * deviceData=NULL;
+ cudaMalloc((void **)&deviceData, dataCount * sizeof(float));
+
+ if(rank==0)
+ {
+ cout << rank<<": "<<"Setup GPU alocated array to 1" << endl;
+ SetKernel<<< gridSize,blockSize >>>(deviceData,1.0f);
+ cout << rank<<": "<<" Sending GPU data " <<endl;
+ MPI_Send((void*)deviceData, dataCount, MPI_FLOAT, 1, 1, MPI_COMM_WORLD);
+ }
+
+ if(rank==1)
+ {
+ cout << rank<<": "<<" Reciving GPU data " <<endl;
+ MPI_Recv((void*) deviceData, dataCount, MPI_FLOAT, 0, 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+
+ float *data = new float[dataCount];
+ cout << rank<<": "<<" Copying data from GPU to CPU " <<endl;
+ cudaMemcpy( (void*) data, (void*)deviceData, dataCount*sizeof(float), cudaMemcpyDeviceToHost);
+ cout << rank<<": "<<" Computin Sum on CPU " <<endl;
+ cout << rank<<": "<< "sum:" << sum(data,dataCount) << endl;
+ delete [] data;
+ }
+
+ cudaFree(deviceData);
+
+ MPI_Finalize();
+return 0;
+}
+
+#else
+
+int main(void)
+{
+ cout << "CUDA or MPI missing...." <<endl;
+}
+
+#endif
+
+
diff --git a/tests/mpi/mpiio-save-load-test.cpp b/tests/mpi/mpiio-save-load-test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f96843d7e26fcf7c64ed8448dd4926a70b4f5ced
--- /dev/null
+++ b/tests/mpi/mpiio-save-load-test.cpp
@@ -0,0 +1,102 @@
+#include <TNL/Communicators/MpiCommunicator.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+#include <TNL/Functions/MeshFunction.h>
+
+#ifdef HAVE_MPI
+
+#define MPIIO
+#include <TNL/Meshes/DistributedMeshes/DistributedGridIO.h>
+
+
+#include "../../src/UnitTests/Mpi/Functions.h"
+
+#define DIM 1
+
+using namespace TNL::Containers;
+using namespace TNL::Meshes;
+using namespace TNL::Functions;
+using namespace TNL::Devices;
+using namespace TNL::Communicators;
+using namespace TNL::Meshes::DistributedMeshes;
+
+
+int main(int argc, char **argv)
+{
+
+ typedef Host Device;
+ typedef MpiCommunicator CommunicatorType;
+
+ typedef Grid<DIM,double,Device,int> MeshType;
+ typedef MeshFunction<MeshType> MeshFunctionType;
+ typedef Vector<double,Device,int> DofType;
+ typedef typename MeshType::Cell Cell;
+ typedef typename MeshType::IndexType IndexType;
+ typedef typename MeshType::PointType PointType;
+ typedef DistributedMesh<MeshType> DistributedGridType;
+
+ typedef typename DistributedGridType::CoordinatesType CoordinatesType;
+ typedef LinearFunction<double,DIM> LinearFunctionType;
+
+ CommunicatorType::Init(argc, argv);
+
+ SharedPointer< LinearFunctionType, Device > linearFunctionPtr;
+ MeshFunctionEvaluator< MeshFunctionType, LinearFunctionType > linearFunctionEvaluator;
+
+ //save distributed meshfunction into files
+ PointType globalOrigin;
+ globalOrigin.setValue(-0.5);
+
+ PointType globalProportions;
+ globalProportions.setValue(10);
+
+
+ MeshType globalGrid;
+ globalGrid.setDimensions(globalProportions);
+ globalGrid.setDomain(globalOrigin,globalProportions);
+
+ File meshFile;
+ meshFile.open( String("globalGrid.tnl"),IOMode::write);
+ globalGrid.save( meshFile );
+ meshFile.close();
+
+ CoordinatesType overlap;
+ overlap.setValue(1);
+ DistributedGridType distrgrid;
+ distrgrid.template setGlobalGrid<CommunicatorType>(globalGrid,overlap);
+
+ SharedPointer<MeshType> gridptr;
+ SharedPointer<MeshFunctionType> meshFunctionptr;
+ distrgrid.SetupGrid(*gridptr);
+
+ DofType dofsave(gridptr->template getEntitiesCount< Cell >());
+ dofsave.setValue(0);
+ meshFunctionptr->bind(gridptr,dofsave);
+
+ linearFunctionEvaluator.evaluateAllEntities(meshFunctionptr , linearFunctionPtr);
+
+ String fileName=String("./meshFunction.tnl");
+ DistributedGridIO<MeshFunctionType,MpiIO> ::save(fileName, *meshFunctionptr );
+
+ DofType dofload(gridptr->template getEntitiesCount< Cell >());
+ dofload.setValue(0);
+ meshFunctionptr->bind(gridptr,dofload);
+
+ DistributedGridIO<MeshFunctionType,MpiIO> ::load(fileName, *meshFunctionptr );
+
+ for(int i=0;i<dofload.getSize();i++)
+ {
+ if(dofsave[i]!=dofload[i])
+ std::cout<<"Chyba na pozici "<< i << " dofsave: "<< dofsave[i] << " dofload: "<<dofload[i] <<std::endl;
+ else
+ std::cout <<"Ok!"<<std::endl;
+ }
+
+ CommunicatorType::Finalize();
+
+}
+
+#else
+
+main(){}
+#endif
+
diff --git a/tests/mpi/mpiio-save-test.cpp b/tests/mpi/mpiio-save-test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c946a2d637e0e15a4d63a443a12ae055ddfcc6d7
--- /dev/null
+++ b/tests/mpi/mpiio-save-test.cpp
@@ -0,0 +1,3 @@
+#define DEVICE Host
+
+#include "mpiio-save-test.h"
diff --git a/tests/mpi/mpiio-save-test.cu b/tests/mpi/mpiio-save-test.cu
new file mode 100644
index 0000000000000000000000000000000000000000..24f98056dd37fe202f0e5809620d00a1ab323777
--- /dev/null
+++ b/tests/mpi/mpiio-save-test.cu
@@ -0,0 +1,3 @@
+#define DEVICE Cuda
+
+#include "mpiio-save-test.h"
diff --git a/tests/mpi/mpiio-save-test.h b/tests/mpi/mpiio-save-test.h
new file mode 100644
index 0000000000000000000000000000000000000000..4b03e497683f0b8c86de1694f9af6567fc87d890
--- /dev/null
+++ b/tests/mpi/mpiio-save-test.h
@@ -0,0 +1,91 @@
+#include <TNL/Communicators/MpiCommunicator.h>
+#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
+#include <TNL/Functions/MeshFunction.h>
+
+#ifdef HAVE_MPI
+
+#define MPIIO
+#include <TNL/Meshes/DistributedMeshes/DistributedGridIO.h>
+
+
+#include "../../src/UnitTests/Mpi/Functions.h"
+
+#define DIM 2
+
+using namespace TNL::Containers;
+using namespace TNL::Meshes;
+using namespace TNL::Functions;
+using namespace TNL::Devices;
+using namespace TNL::Communicators;
+using namespace TNL::Meshes::DistributedMeshes;
+
+
+int main(int argc, char **argv)
+{
+
+ typedef DEVICE Device;
+ typedef MpiCommunicator CommunicatorType;
+
+ typedef Grid<DIM,double,Device,int> MeshType;
+ typedef MeshFunction<MeshType> MeshFunctionType;
+ typedef Vector<double,Device,int> DofType;
+ typedef typename MeshType::Cell Cell;
+ typedef typename MeshType::IndexType IndexType;
+ typedef typename MeshType::PointType PointType;
+ typedef DistributedMesh<MeshType> DistributedGridType;
+
+ typedef typename DistributedGridType::CoordinatesType CoordinatesType;
+ typedef LinearFunction<double,DIM> LinearFunctionType;
+
+ CommunicatorType::Init(argc, argv);
+
+ SharedPointer< LinearFunctionType, Device > linearFunctionPtr;
+ MeshFunctionEvaluator< MeshFunctionType, LinearFunctionType > linearFunctionEvaluator;
+
+ //save distributed meshfunction into files
+ PointType globalOrigin;
+ globalOrigin.setValue(-0.5);
+
+ PointType globalProportions;
+ //globalProportions.setValue(10);
+ globalProportions.x()=10;
+ globalProportions.y()=20;
+
+
+
+ MeshType globalGrid;
+ globalGrid.setDimensions(globalProportions);
+ globalGrid.setDomain(globalOrigin,globalProportions);
+
+ File meshFile;
+ meshFile.open( String("globalGrid.tnl"),IOMode::write);
+ globalGrid.save( meshFile );
+ meshFile.close();
+
+ CoordinatesType overlap;
+ overlap.setValue(1);
+ DistributedGridType distrgrid;
+ distrgrid.template setGlobalGrid<CommunicatorType>(globalGrid,overlap);
+
+ SharedPointer<MeshType> gridptr;
+ SharedPointer<MeshFunctionType> meshFunctionptr;
+ distrgrid.SetupGrid(*gridptr);
+
+ DofType dof(gridptr->template getEntitiesCount< Cell >());
+ dof.setValue(0);
+ meshFunctionptr->bind(gridptr,dof);
+
+ linearFunctionEvaluator.evaluateAllEntities(meshFunctionptr , linearFunctionPtr);
+
+ String fileName=String("./meshFunction.tnl");
+ DistributedGridIO<MeshFunctionType,MpiIO> ::save(fileName, *meshFunctionptr );
+
+ CommunicatorType::Finalize();
+
+}
+
+#else
+
+main(){}
+
+#endif
diff --git a/tests/mpi/simple-mpiio.cu b/tests/mpi/simple-mpiio.cu
new file mode 100644
index 0000000000000000000000000000000000000000..1b8de26039b14b230a1b3a5e2d6d43cbc629d062
--- /dev/null
+++ b/tests/mpi/simple-mpiio.cu
@@ -0,0 +1,77 @@
+#include <stdio.h>
+#include <mpi.h>
+#include <stdlib.h>
+
+int main(int argc, char **argv)
+{
+
+ int localsize = 30;
+ int overlap=1;
+ int globalsize = localsize+2*overlap;
+
+ double * data;
+ //data=(double *)malloc(globalsize*globalsize*sizeof(double));
+ cudaMalloc((void **)&data, globalsize*globalsize*sizeof(double));
+
+ MPI_Init(&argc,&argv);
+
+ int rank;
+ MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+ int size;
+ MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+ printf("rank: %d, size %d\n",rank, size);
+
+ int fgsize[2],flsize[2],fstarts[2];
+ fgsize[0]=size*localsize;
+ fgsize[1]=localsize;
+ flsize[0]=localsize;
+ flsize[1]=localsize;
+ fstarts[0]=rank*localsize;
+ fstarts[1]=0;
+
+ MPI_Datatype ftype;
+
+ MPI_Type_create_subarray(2,
+ fgsize,flsize,fstarts,
+ MPI_ORDER_C,MPI_DOUBLE,&ftype);
+
+ MPI_Type_commit(&ftype);
+
+
+ int agsize[2],alsize[2],astarts[2];
+ agsize[0]=globalsize;
+ agsize[1]=globalsize;
+ alsize[0]=localsize;
+ alsize[1]=localsize;
+ astarts[0]=overlap;
+ astarts[1]=overlap;
+
+ MPI_Datatype atype;
+
+ MPI_Type_create_subarray(2,
+ agsize,alsize,astarts,
+ MPI_ORDER_C,MPI_DOUBLE,&atype);
+
+ MPI_Type_commit(&atype);
+
+
+ MPI_File file;
+ MPI_File_open(MPI_COMM_WORLD,"./pokus.file",
+ MPI_MODE_CREATE|MPI_MODE_WRONLY,
+ MPI_INFO_NULL, &file);
+
+ MPI_File_set_view(file,0,MPI_DOUBLE,ftype,"native",MPI_INFO_NULL);
+
+ MPI_Status wstatus;
+ MPI_File_write(file,data,1,atype,&wstatus);
+
+ MPI_File_close(&file);
+
+ MPI_Finalize();
+
+ free(data);
+
+return 0;
+}
+