Skip to content
Snippets Groups Projects
Commit eddfb5da authored by Tomáš Oberhuber's avatar Tomáš Oberhuber
Browse files

[WIP] Fixing distributed grid MPI tests.

parent 675024b2
No related branches found
No related tags found
1 merge request!26Lbm
Hide whitespace changes
Inline Side-by-side
src/TNL/Communicators/MPIPrint.h
+ 4
− 4
View file @ eddfb5da
......@@ -25,7 +25,7 @@ else
__tnl_mpi_print_stream_ << "Node " << TNL::Communicators::MpiCommunicator::GetRank() << " of " \
<< TNL::Communicators::MpiCommunicator::GetSize() << " : " << message << std::endl; \
TNL::String __tnl_mpi_print_string_( __tnl_mpi_print_stream_.str().c_str() ); \
__tnl_mpi_print_string_.send( 0 ); \
__tnl_mpi_print_string_.send( 0, std::numeric_limits< int >::max() ); \
} \
else \
{ \
......@@ -35,7 +35,7 @@ else
__tnl_mpi_print_j++ ) \
{ \
TNL::String __tnl_mpi_print_string_; \
__tnl_mpi_print_string_.receive( __tnl_mpi_print_j ); \
__tnl_mpi_print_string_.receive( __tnl_mpi_print_j, std::numeric_limits< int >::max() ); \
std::cerr << __tnl_mpi_print_string_; \
} \
} \
......@@ -78,7 +78,7 @@ else
__tnl_mpi_print_stream_ << "Node " << TNL::Communicators::MpiCommunicator::GetRank() << " of " \
<< TNL::Communicators::MpiCommunicator::GetSize() << " : " << message << std::endl; \
TNL::String __tnl_mpi_print_string_( __tnl_mpi_print_stream_.str().c_str() ); \
__tnl_mpi_print_string_.send( 0 ); \
__tnl_mpi_print_string_.send( 0, std::numeric_limits< int >::max() ); \
} \
} \
else \
......@@ -94,7 +94,7 @@ else
if( __tnl_mpi_print_cond ) \
{ \
TNL::String __tnl_mpi_print_string_; \
__tnl_mpi_print_string_.receive( __tnl_mpi_print_j ); \
__tnl_mpi_print_string_.receive( __tnl_mpi_print_j, std::numeric_limits< int >::max() ); \
std::cerr << __tnl_mpi_print_string_; \
} \
} \
......
src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h
+ 9
− 0
View file @ eddfb5da
......@@ -156,20 +156,29 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension,
//send everything, recieve everything
for( int i=0; i<this->getNeighborCount(); i++ )
{
TNL_MPI_PRINT( "Sending data... " << i << " sizes -> " << sendSizes[ i ] );
if( neighbors[ i ] != -1 )
{
TNL_MPI_PRINT( "Sending data to node " << neighbors[ i ] );
requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sendSizes[ i ], neighbors[ i ], 0, group );
TNL_MPI_PRINT( "Receiving data from node " << neighbors[ i ] );
requests[ requestsCount++ ] = CommunicatorType::IRecv( recieveBuffers[ i ].getData(), sendSizes[ i ], neighbors[ i ], 0, group );
}
else if( periodicBoundaries && sendSizes[ i ] !=0 )
{
TNL_MPI_PRINT( "Sending data to node " << periodicNeighbors[ i ] );
requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sendSizes[ i ], periodicNeighbors[ i ], 1, group );
TNL_MPI_PRINT( "Receiving data to node " << periodicNeighbors[ i ] );
requests[ requestsCount++ ] = CommunicatorType::IRecv( recieveBuffers[ i ].getData(), sendSizes[ i ], periodicNeighbors[ i ], 1, group );
}
}
//wait until send is done
TNL_MPI_PRINT( "Waiting for data ..." )
CommunicatorType::WaitAll( requests, requestsCount );
TNL_MPI_PRINT( "Copying data ..." )
//copy data from receive buffers
copyBuffers(meshFunction,
recieveBuffers,recieveBegin,sendDimensions ,
......
src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp
+ 13
− 7
View file @ eddfb5da
......@@ -97,6 +97,7 @@ typedef typename GridType::Cell Cell;
typedef typename GridType::IndexType IndexType;
typedef typename GridType::PointType PointType;
typedef DistributedMesh<GridType> DistributedGridType;
using Synchronizer = DistributedMeshSynchronizer< MeshFunctionType >;
class DistributedGridTest_1D : public ::testing::Test
{
......@@ -170,6 +171,7 @@ class DistributedGridTest_1D : public ::testing::Test
}
};
#ifdef UNDEF
TEST_F( DistributedGridTest_1D, isBoundaryDomainTest )
{
if( rank == 0 || rank == nproc - 1 )
......@@ -237,7 +239,7 @@ TEST_F(DistributedGridTest_1D, EvaluateLinearFunction )
entity2.refresh();
EXPECT_EQ(meshFunctionPtr->getValue(entity), (*linearFunctionPtr)(entity)) << "Linear function Overlap error on right Edge.";
}
#endif
TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithoutMask )
{
......@@ -255,18 +257,19 @@ TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithoutMask )
setDof_1D( dof, -rank-1 );
maskDofs.setValue( true );
constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr, constFunctionPtr );
using Synchronizer = decltype( meshFunctionPtr->getSynchronizer() );
//constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr, constFunctionPtr );
meshFunctionPtr->getSynchronizer().setPeriodicBoundariesCopyDirection( Synchronizer::OverlapToBoundary );
TNL_MPI_PRINT( ">>>>>>>>>>>>>> " << dof[ 1 ] << " : " << -rank - 1 );
meshFunctionPtr->template synchronize<CommunicatorType>( true );
if( rank == 0 )
TNL_MPI_PRINT( "#########" << dof[ 1 ] );
/*if( rank == 0 )
EXPECT_EQ( dof[ 1 ], -nproc ) << "Left Overlap was filled by wrong process.";
if( rank == nproc-1 )
EXPECT_EQ( dof[ dof.getSize() - 2 ], -1 )<< "Right Overlap was filled by wrong process.";
EXPECT_EQ( dof[ dof.getSize() - 2 ], -1 )<< "Right Overlap was filled by wrong process.";*/
}
#ifdef UNDEF
TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithActiveMask )
{
// Setup periodic boundaries
......@@ -284,6 +287,7 @@ TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithActiveMask )
setDof_1D( dof, -rank-1 );
maskDofs.setValue( true );
constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr, constFunctionPtr );
meshFunctionPtr->getSynchronizer().setPeriodicBoundariesCopyDirection( Synchronizer::OverlapToBoundary );
meshFunctionPtr->template synchronize<CommunicatorType>( true, maskPointer );
if( rank == 0 )
EXPECT_EQ( dof[ 1 ], -nproc ) << "Left Overlap was filled by wrong process.";
......@@ -309,6 +313,7 @@ TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithInactiveMaskOnLef
maskDofs.setValue( true );
maskDofs.setElement( 1, false );
constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr , constFunctionPtr );
meshFunctionPtr->getSynchronizer().setPeriodicBoundariesCopyDirection( Synchronizer::OverlapToBoundary );
meshFunctionPtr->template synchronize<CommunicatorType>( true, maskPointer );
if( rank == 0 )
......@@ -336,6 +341,7 @@ TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithInactiveMask )
maskDofs.setElement( 1, false );
maskDofs.setElement( dof.getSize() - 2, false );
constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr , constFunctionPtr );
meshFunctionPtr->getSynchronizer().setPeriodicBoundariesCopyDirection( Synchronizer::OverlapToBoundary );
meshFunctionPtr->template synchronize<CommunicatorType>( true, maskPointer );
if( rank == 0 )
......@@ -377,7 +383,7 @@ TEST_F(DistributedGridTest_1D, SynchronizePeriodicBoundariesLinearTest )
if( rank == nproc - 1 )
EXPECT_EQ( meshFunctionPtr->getValue(entity2), -1 ) << "Linear function Overlap error on right Edge.";
}
#endif
#else
......
src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp
+ 7
− 0
View file @ eddfb5da
......@@ -323,6 +323,7 @@ typedef typename GridType::Cell Cell;
typedef typename GridType::IndexType IndexType;
typedef typename GridType::PointType PointType;
typedef DistributedMesh<GridType> DistributedGridType;
using Synchronizer = DistributedMeshSynchronizer< MeshFunctionType >;
class DistributedGridTest_2D : public ::testing::Test
{
......@@ -541,6 +542,7 @@ TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithoutMask
//Expecting 9 processes
setDof_2D(*dof, -rank-1 );
constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr , constFunctionPtr );
meshFunctionPtr->getSynchronizer().setPeriodicBoundariesCopyDirection( Synchronizer::OverlapToBoundary );
meshFunctionPtr->template synchronize<CommunicatorType>( true );
if( rank == 0 )
......@@ -615,6 +617,7 @@ TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithActiveM
setDof_2D(*dof, -rank-1 );
maskDofs.setValue( true );
constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr , constFunctionPtr );
meshFunctionPtr->getSynchronizer().setPeriodicBoundariesCopyDirection( Synchronizer::OverlapToBoundary );
meshFunctionPtr->template synchronize<CommunicatorType>( true, maskPointer );
if( rank == 0 )
......@@ -699,6 +702,7 @@ TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithInactiv
}
}
constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr , constFunctionPtr );
meshFunctionPtr->getSynchronizer().setPeriodicBoundariesCopyDirection( Synchronizer::OverlapToBoundary );
meshFunctionPtr->template synchronize<CommunicatorType>( true, maskPointer );
if( rank == 0 )
......@@ -783,6 +787,7 @@ TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithInActiv
}
}
constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr , constFunctionPtr );
meshFunctionPtr->getSynchronizer().setPeriodicBoundariesCopyDirection( Synchronizer::OverlapToBoundary );
meshFunctionPtr->template synchronize<CommunicatorType>( true, maskPointer );
if( rank == 0 )
......@@ -867,6 +872,7 @@ TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithInActiv
}
}
constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr , constFunctionPtr );
meshFunctionPtr->getSynchronizer().setPeriodicBoundariesCopyDirection( Synchronizer::OverlapToBoundary );
meshFunctionPtr->template synchronize<CommunicatorType>( true, maskPointer );
if( rank == 0 )
......@@ -951,6 +957,7 @@ TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithInActiv
}
}
constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr , constFunctionPtr );
meshFunctionPtr->getSynchronizer().setPeriodicBoundariesCopyDirection( Synchronizer::OverlapToBoundary );
meshFunctionPtr->template synchronize<CommunicatorType>( true, maskPointer );
if( rank == 0 )
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment