Commit 5c15d04c authored by Matouš Fencl's avatar Matouš Fencl Committed by Tomáš Oberhuber

MPI implemented for CPU and GPU in 2D but meshFunction.template synchronize<...

MPI implemented for CPU and GPU in 2D but meshFunction.template synchronize< Communicator >(); doesn't copy overlaps.
parent 933cc22b
......@@ -36,6 +36,9 @@ class DirectEikonalSolverConfig
{
config.addDelimiter( "Direct eikonal equation solver settings:" );
config.addRequiredEntry< String >( "input-file", "Input file." );
config.addEntry< String >( "distributed-grid-io-type", "Choose Distributed Grid IO Type", "LocalCopy");
config.addEntryEnum< String >( "LocalCopy" );
config.addEntryEnum< String >( "MpiIO" );
};
};
......
......@@ -70,7 +70,7 @@ class tnlDirectEikonalMethodsBase< Meshes::Grid< 2, Real, Device, Index > >
InterfaceMapPointer& interfaceMap );
template< typename MeshEntity >
__cuda_callable__ void updateCell( MeshFunctionType& u,
__cuda_callable__ bool updateCell( MeshFunctionType& u,
const MeshEntity& cell,
const RealType velocity = 1.0 );
......@@ -147,7 +147,12 @@ __global__ void CudaUpdateCellCaller( tnlDirectEikonalMethodsBase< Meshes::Grid<
const Functions::MeshFunction< Meshes::Grid< 2, Real, Device, Index >, 2, bool >& interfaceMap,
const Functions::MeshFunction< Meshes::Grid< 2, Real, Device, Index > >& aux,
Functions::MeshFunction< Meshes::Grid< 2, Real, Device, Index > >& helpFunc,
TNL::Containers::ArrayView< int, Devices::Cuda, Index > BlockIterDevice, int oddEvenBlock =0);
TNL::Containers::Array< int, Devices::Cuda, Index > BlockIterDevice,
Containers::StaticVector< 2, Index > vLower, Containers::StaticVector< 2, Index > vUpper, int k,int oddEvenBlock =0);
template< typename Real, typename Device, typename Index >
__global__ void DeepCopy( const Functions::MeshFunction< Meshes::Grid< 2, Real, Device, Index > >& aux,
Functions::MeshFunction< Meshes::Grid< 2, Real, Device, Index > >& helpFunc );
template < typename Index >
__global__ void CudaParallelReduc( TNL::Containers::ArrayView< int, Devices::Cuda, Index > BlockIterDevice,
......@@ -160,7 +165,8 @@ __global__ void GetNeighbours( TNL::Containers::ArrayView< int, Devices::Cuda, I
template < typename Real, typename Device, typename Index >
__global__ void CudaInitCaller( const Functions::MeshFunction< Meshes::Grid< 2, Real, Device, Index > >& input,
Functions::MeshFunction< Meshes::Grid< 2, Real, Device, Index > >& output,
Functions::MeshFunction< Meshes::Grid< 2, Real, Device, Index >, 2, bool >& interfaceMap );
Functions::MeshFunction< Meshes::Grid< 2, Real, Device, Index >, 2, bool >& interfaceMap,
Containers::StaticVector< 2, Index > vLower, Containers::StaticVector< 2, Index > vUpper );
template < typename Real, typename Device, typename Index >
__global__ void CudaInitCaller3d( const Functions::MeshFunction< Meshes::Grid< 3, Real, Device, Index > >& input,
......
......@@ -71,6 +71,8 @@ class tnlDirectEikonalProblem
bool setInitialCondition( const Config::ParameterContainer& parameters,
DofVectorPointer& dofs );
bool makeSnapshot( );
bool solve( DofVectorPointer& dosf );
......
......@@ -12,6 +12,9 @@
*/
#pragma once
#include <TNL/FileName.h>
#include "tnlDirectEikonalProblem.h"
template< typename Mesh,
typename Communicator,
......@@ -76,6 +79,11 @@ tnlDirectEikonalProblem< Mesh, Communicator, Anisotropy, Real, Index >::
setup( const Config::ParameterContainer& parameters,
const String& prefix )
{
String param=parameters.getParameter< String >( "distributed-grid-io-type" );
if(param=="MpiIO")
distributedIOType=Meshes::DistributedMeshes::MpiIO;
if(param=="LocalCopy")
distributedIOType=Meshes::DistributedMeshes::LocalCopy;
return true;
}
......@@ -117,14 +125,14 @@ setInitialCondition( const Config::ParameterContainer& parameters,
String inputFile = parameters.getParameter< String >( "input-file" );
this->initialData->setMesh( this->getMesh() );
std::cout<<"setInitialCondition" <<std::endl;
if(CommunicatorType::isDistributed())
if( CommunicatorType::isDistributed() )
{
std::cout<<"Nodes Distribution: " << u->getMesh().getDistributedMesh()->printProcessDistr() << std::endl;
std::cout<<"Nodes Distribution: " << initialData->getMesh().getDistributedMesh()->printProcessDistr() << std::endl;
if(distributedIOType==Meshes::DistributedMeshes::MpiIO)
Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType,Meshes::DistributedMeshes::MpiIO> ::load(inputFile, *u );
Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType,Meshes::DistributedMeshes::MpiIO> ::load(inputFile, *initialData );
if(distributedIOType==Meshes::DistributedMeshes::LocalCopy)
Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType,Meshes::DistributedMeshes::LocalCopy> ::load(inputFile, *u );
u->template synchronize<CommunicatorType>();
Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType,Meshes::DistributedMeshes::LocalCopy> ::load(inputFile, *initialData );
initialData->template synchronize<CommunicatorType>();
}
else
{
......@@ -141,6 +149,38 @@ setInitialCondition( const Config::ParameterContainer& parameters,
return true;
}
template< typename Mesh,
typename Communicator,
typename Anisotropy,
typename Real,
typename Index >
bool
tnlDirectEikonalProblem< Mesh, Communicator, Anisotropy, Real, Index >::
makeSnapshot( )
{
std::cout << std::endl << "Writing output." << std::endl;
//this->bindDofs( dofs );
FileName fileName;
fileName.setFileNameBase( "u-" );
fileName.setExtension( "tnl" );
if(CommunicatorType::isDistributed())
{
if(distributedIOType==Meshes::DistributedMeshes::MpiIO)
Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType,Meshes::DistributedMeshes::MpiIO> ::save(fileName.getFileName(), *u );
if(distributedIOType==Meshes::DistributedMeshes::LocalCopy)
Meshes::DistributedMeshes::DistributedGridIO<MeshFunctionType,Meshes::DistributedMeshes::LocalCopy> ::save(fileName.getFileName(), *u );
}
else
{
if( ! this->u->save( fileName.getFileName() ) )
return false;
}
return true;
}
template< typename Mesh,
typename Communicator,
......@@ -151,7 +191,19 @@ bool
tnlDirectEikonalProblem< Mesh, Communicator, Anisotropy, Real, Index >::
solve( DofVectorPointer& dofs )
{
FastSweepingMethod< MeshType, AnisotropyType > fsm;
fsm.solve( this->getMesh(), anisotropy, initialData );
FastSweepingMethod< MeshType, Communicator,AnisotropyType > fsm;
fsm.solve( this->getMesh(), u, anisotropy, initialData );
/*int i = Communicators::MpiCommunicator::GetRank( Communicators::MpiCommunicator::AllGroup );
const MeshPointer msh = this->getMesh();
if( i == 0 && msh->getMeshDimension() == 2 )
{
for( int k = 0; k < 9; k++ ){
for( int l = 0; l < msh->getDimensions().x(); l++ )
printf("%.2f\t",(*initialData)[ k * msh->getDimensions().x() + l ] );
printf("\n");
}
}*/
makeSnapshot();
return true;
}
......@@ -17,6 +17,7 @@
template< typename Mesh,
typename Communicator,
typename Anisotropy = Functions::Analytic::Constant< Mesh::getMeshDimension(), typename Mesh::RealType > >
class FastSweepingMethod
{
......@@ -25,8 +26,9 @@ class FastSweepingMethod
template< typename Real,
typename Device,
typename Index,
typename Communicator,
typename Anisotropy >
class FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Anisotropy >
class FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Communicator, Anisotropy >
: public tnlDirectEikonalMethodsBase< Meshes::Grid< 1, Real, Device, Index > >
{
//static_assert( std::is_same< Device, TNL::Devices::Host >::value, "The fast sweeping method works only on CPU." );
......@@ -47,7 +49,7 @@ class FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Anisotropy >
using typename BaseType::MeshFunctionType;
using typename BaseType::InterfaceMapPointer;
using typename BaseType::MeshFunctionPointer;
FastSweepingMethod();
......@@ -56,6 +58,7 @@ class FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Anisotropy >
void setMaxIterations( const IndexType& maxIterations );
void solve( const MeshPointer& mesh,
MeshFunctionPointer& Aux,
const AnisotropyPointer& anisotropy,
MeshFunctionPointer& u );
......@@ -68,8 +71,9 @@ class FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Anisotropy >
template< typename Real,
typename Device,
typename Index,
typename Communicator,
typename Anisotropy >
class FastSweepingMethod< Meshes::Grid< 2, Real, Device, Index >, Anisotropy >
class FastSweepingMethod< Meshes::Grid< 2, Real, Device, Index >, Communicator, Anisotropy >
: public tnlDirectEikonalMethodsBase< Meshes::Grid< 2, Real, Device, Index > >
{
//static_assert( std::is_same< Device, TNL::Devices::Host >::value, "The fast sweeping method works only on CPU." );
......@@ -84,13 +88,14 @@ class FastSweepingMethod< Meshes::Grid< 2, Real, Device, Index >, Anisotropy >
typedef tnlDirectEikonalMethodsBase< Meshes::Grid< 2, Real, Device, Index > > BaseType;
using MeshPointer = Pointers::SharedPointer< MeshType >;
using AnisotropyPointer = Pointers::SharedPointer< AnisotropyType, DeviceType >;
using MPI = Communicators::MpiCommunicator;
using typename BaseType::InterfaceMapType;
using typename BaseType::MeshFunctionType;
using typename BaseType::InterfaceMapPointer;
using typename BaseType::MeshFunctionPointer;
using typename BaseType::ArrayContainer;
FastSweepingMethod();
const IndexType& getMaxIterations() const;
......@@ -98,8 +103,9 @@ class FastSweepingMethod< Meshes::Grid< 2, Real, Device, Index >, Anisotropy >
void setMaxIterations( const IndexType& maxIterations );
void solve( const MeshPointer& mesh,
MeshFunctionPointer& Aux,
const AnisotropyPointer& anisotropy,
MeshFunctionPointer& u );
const MeshFunctionPointer& u );
protected:
......@@ -109,8 +115,9 @@ class FastSweepingMethod< Meshes::Grid< 2, Real, Device, Index >, Anisotropy >
template< typename Real,
typename Device,
typename Index,
typename Communicator,
typename Anisotropy >
class FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Anisotropy >
class FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Communicator, Anisotropy >
: public tnlDirectEikonalMethodsBase< Meshes::Grid< 3, Real, Device, Index > >
{
//static_assert( std::is_same< Device, TNL::Devices::Host >::value, "The fast sweeping method works only on CPU." );
......@@ -140,6 +147,7 @@ class FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Anisotropy >
void setMaxIterations( const IndexType& maxIterations );
void solve( const MeshPointer& mesh,
MeshFunctionPointer& Aux,
const AnisotropyPointer& anisotropy,
MeshFunctionPointer& u );
......
......@@ -18,8 +18,9 @@
template< typename Real,
typename Device,
typename Index,
typename Communicator,
typename Anisotropy >
FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Anisotropy >::
FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Communicator, Anisotropy >::
FastSweepingMethod()
: maxIterations( 1 )
{
......@@ -29,9 +30,10 @@ FastSweepingMethod()
template< typename Real,
typename Device,
typename Index,
typename Communicator,
typename Anisotropy >
const Index&
FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Anisotropy >::
FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Communicator, Anisotropy >::
getMaxIterations() const
{
......@@ -40,9 +42,10 @@ getMaxIterations() const
template< typename Real,
typename Device,
typename Index,
typename Communicator,
typename Anisotropy >
void
FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Anisotropy >::
FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Communicator, Anisotropy >::
setMaxIterations( const IndexType& maxIterations )
{
......@@ -51,10 +54,12 @@ setMaxIterations( const IndexType& maxIterations )
template< typename Real,
typename Device,
typename Index,
typename Communicator,
typename Anisotropy >
void
FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Anisotropy >::
FastSweepingMethod< Meshes::Grid< 1, Real, Device, Index >, Communicator, Anisotropy >::
solve( const MeshPointer& mesh,
MeshFunctionPointer& Aux,
const AnisotropyPointer& anisotropy,
MeshFunctionPointer& u )
{
......
......@@ -18,8 +18,9 @@
template< typename Real,
typename Device,
typename Index,
typename Communicator,
typename Anisotropy >
FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Anisotropy >::
FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Communicator, Anisotropy >::
FastSweepingMethod()
: maxIterations( 1 )
{
......@@ -29,9 +30,10 @@ FastSweepingMethod()
template< typename Real,
typename Device,
typename Index,
typename Communicator,
typename Anisotropy >
const Index&
FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Anisotropy >::
FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Communicator, Anisotropy >::
getMaxIterations() const
{
......@@ -40,9 +42,10 @@ getMaxIterations() const
template< typename Real,
typename Device,
typename Index,
typename Communicator,
typename Anisotropy >
void
FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Anisotropy >::
FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Communicator, Anisotropy >::
setMaxIterations( const IndexType& maxIterations )
{
......@@ -51,10 +54,12 @@ setMaxIterations( const IndexType& maxIterations )
template< typename Real,
typename Device,
typename Index,
typename Communicator,
typename Anisotropy >
void
FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Anisotropy >::
FastSweepingMethod< Meshes::Grid< 3, Real, Device, Index >, Communicator, Anisotropy >::
solve( const MeshPointer& mesh,
MeshFunctionPointer& Aux,
const AnisotropyPointer& anisotropy,
MeshFunctionPointer& u )
{
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment