From a1f8dd044e6fc42e770505f798b4c6af6d1be18f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Hanousek=20-=20vz?= Date: Wed, 21 Nov 2018 08:55:12 +0100 Subject: [PATCH 1/7] issue #15 - Between commit -rename directions to ZvYvXv, where v is m,z,p -correct y direction in 2D --- src/TNL/Meshes/DistributedMeshes/Directions.h | 37 ++- .../DistributedGridSynchronizer_1D.h | 56 ++-- .../DistributedGridSynchronizer_2D.h | 120 ++++---- .../DistributedGridSynchronizer_3D.h | 287 +++++++++--------- .../GridDetails/Traverser_Grid1D_impl.h | 8 +- .../GridDetails/Traverser_Grid2D_impl.h | 17 +- .../GridDetails/Traverser_Grid3D_impl.h | 24 +- .../DistributedMeshes/DirectionsTest.cpp | 146 ++++----- .../DistributedGridTest_2D.cpp | 9 +- 9 files changed, 360 insertions(+), 344 deletions(-) diff --git a/src/TNL/Meshes/DistributedMeshes/Directions.h b/src/TNL/Meshes/DistributedMeshes/Directions.h index 8813f3c9f..84a5afa91 100644 --- a/src/TNL/Meshes/DistributedMeshes/Directions.h +++ b/src/TNL/Meshes/DistributedMeshes/Directions.h @@ -14,7 +14,7 @@ namespace DistributedMeshes { // -> 0 - not used, 1 negative direction, 2 positive direction //finaly we subtrackt 1 because we dont need (0,0,0) aka 0 aka no direction -enum Directions2D { Left = 0 , Right = 1 , Up = 2, UpLeft =3, UpRight=4, Down=5, DownLeft=6, DownRight=7 }; +//enum Directions2D { Left = 0 , Right = 1 , Up = 2, UpLeft =3, UpRight=4, Down=5, DownLeft=6, DownRight=7 }; /*MEH - osa zed je zdola nahoru, asi--- enum Directions3D { West = 0 , East = 1 , @@ -28,16 +28,33 @@ enum Directions3D { West = 0 , East = 1 , BottomSouth = 23, BottomSouthWest = 24, BottomSouthEast = 25, };*/ -enum Directions3D { West = 0 , East = 1 , - North = 2, NorthWest = 3, NorthEast = 4, - South = 5, SouthWest = 6, SouthEast = 7, - Bottom = 8 ,BottomWest = 9 , BottomEast = 10 , - BottomNorth = 11, BottomNorthWest = 12, BottomNorthEast = 13, - BottomSouth = 14, BottomSouthWest = 15, BottomSouthEast = 16, - Top = 17, TopWest = 18, TopEast =19, - TopNorth = 20, TopNorthWest = 21, TopNorthEast = 22, - TopSouth = 23, TopSouthWest = 24,TopSouthEast = 25, +/* +with self +enum Directions3D { + ZzYzXz = 0, ZzYzXm = 1, ZzYzXp = 2, + ZzYmXz = 3, ZzYmXm = 4, ZzYmXp = 5, + ZzYpXz = 6, ZzYpXm = 7, ZzYpXp = 8, + ZmYzXz = 9, ZmYzXm = 10, ZmYzXp = 11, + ZmYmXz = 12, ZmYmXm = 13, ZmYmXp = 14, + ZmYpXz = 15, ZmYpXm = 16, ZmYpXp = 17, + ZpYzXz = 18, ZpYzXm = 19, ZpYzXp = 20, + ZpYmXz = 21, ZpYmXm = 22, ZpYmXp = 23, + ZpYpXz = 24, ZpYpXm = 25, ZpYpXp = 26 }; +*/ + +enum Directions3D { + ZzYzXm = 0, ZzYzXp = 1, + ZzYmXz = 2, ZzYmXm = 3, ZzYmXp = 4, + ZzYpXz = 5, ZzYpXm = 6, ZzYpXp = 7, + ZmYzXz = 8, ZmYzXm = 9, ZmYzXp = 10, + ZmYmXz = 11, ZmYmXm = 12, ZmYmXp = 13, + ZmYpXz = 14, ZmYpXm = 15, ZmYpXp = 16, + ZpYzXz = 17, ZpYzXm = 18, ZpYzXp = 19, + ZpYmXz = 20, ZpYmXm = 21, ZpYmXp = 22, + ZpYpXz = 23, ZpYpXm = 24, ZpYpXp = 25 + }; + class Directions { diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_1D.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_1D.h index 2aaa47bab..ffc700556 100644 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_1D.h +++ b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_1D.h @@ -68,10 +68,10 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 1, GridReal, D const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); - sendBuffers[ Left ].setSize( lowerOverlap.x() ); - sendBuffers[ Right ].setSize( upperOverlap.x() ); - receiveBuffers[ Left ].setSize( lowerOverlap.x() ); - receiveBuffers[ Right ].setSize( upperOverlap.x() ); + sendBuffers[ ZzYzXm ].setSize( lowerOverlap.x() ); + sendBuffers[ ZzYzXp ].setSize( upperOverlap.x() ); + receiveBuffers[ ZzYzXm ].setSize( lowerOverlap.x() ); + receiveBuffers[ ZzYzXp ].setSize( upperOverlap.x() ); }; @@ -102,9 +102,9 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 1, GridReal, D if( periodicBoundaries ) { - if( neighbors[ Left ] == -1 ) + if( neighbors[ ZzYzXm ] == -1 ) swap( leftSource, leftDestination ); - if( neighbors[ Right ] == -1 ) + if( neighbors[ ZzYzXp ] == -1 ) swap( rightSource, rightDestination ); } @@ -122,34 +122,34 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 1, GridReal, D int requestsCount( 0 ); //send everything, recieve everything - if( neighbors[ Left ] != -1 ) + if( neighbors[ ZzYzXm ] != -1 ) { - TNL_ASSERT_GE( sendBuffers[ Left ].getSize(), lowerOverlap.x(), "" ); - TNL_ASSERT_GE( receiveBuffers[ Left ].getSize(), lowerOverlap.x(), "" ); - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ Left ].getData(), lowerOverlap.x(), neighbors[ Left ], 0, group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ Left ].getData(), lowerOverlap.x(), neighbors[ Left ], 0, group ); + TNL_ASSERT_GE( sendBuffers[ ZzYzXm ].getSize(), lowerOverlap.x(), "" ); + TNL_ASSERT_GE( receiveBuffers[ ZzYzXm ].getSize(), lowerOverlap.x(), "" ); + requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ ZzYzXm ].getData(), lowerOverlap.x(), neighbors[ ZzYzXm ],0, group ); + requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ ZzYzXm ].getData(), lowerOverlap.x(), neighbors[ ZzYzXm ],0, group ); } else if( periodicBoundaries ) { - TNL_ASSERT_GE( sendBuffers[ Left ].getSize(), lowerOverlap.x(), "" ); - TNL_ASSERT_GE( receiveBuffers[ Left ].getSize(), lowerOverlap.x(), "" ); - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ Left ].getData(), lowerOverlap.x(), periodicNeighbors[ Left ], 1, group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ Left ].getData(), lowerOverlap.x(), periodicNeighbors[ Left ], 1, group ); + TNL_ASSERT_GE( sendBuffers[ ZzYzXm ].getSize(), lowerOverlap.x(), "" ); + TNL_ASSERT_GE( receiveBuffers[ ZzYzXm ].getSize(), lowerOverlap.x(), "" ); + requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ ZzYzXm ].getData(), lowerOverlap.x(), periodicNeighbors[ ZzYzXm ],1, group ); + requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ ZzYzXm ].getData(), lowerOverlap.x(), periodicNeighbors[ ZzYzXm ],1, group ); } - if( neighbors[ Right ] != -1 ) + if( neighbors[ ZzYzXp ] != -1 ) { - TNL_ASSERT_GE( sendBuffers[ Right ].getSize(), upperOverlap.x(), "" ); - TNL_ASSERT_GE( receiveBuffers[ Right ].getSize(), upperOverlap.x(), "" ); - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ Right ].getData(), upperOverlap.x(), neighbors[ Right ], 0, group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ Right ].getData(), upperOverlap.x(), neighbors[ Right ], 0, group ); + TNL_ASSERT_GE( sendBuffers[ ZzYzXp ].getSize(), upperOverlap.x(), "" ); + TNL_ASSERT_GE( receiveBuffers[ ZzYzXp ].getSize(), upperOverlap.x(), "" ); + requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ ZzYzXp ].getData(), upperOverlap.x(), neighbors[ ZzYzXp ], 0, group ); + requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ ZzYzXp ].getData(), upperOverlap.x(), neighbors[ ZzYzXp ], 0, group ); } else if( periodicBoundaries ) { - TNL_ASSERT_GE( sendBuffers[ Right ].getSize(), upperOverlap.x(), "" ); - TNL_ASSERT_GE( receiveBuffers[ Right ].getSize(), upperOverlap.x(), "" ); - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ Right ].getData(), upperOverlap.x(), periodicNeighbors[ Right ], 1, group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ Right ].getData(), upperOverlap.x(), periodicNeighbors[ Right ], 1, group ); + TNL_ASSERT_GE( sendBuffers[ ZzYzXp ].getSize(), upperOverlap.x(), "" ); + TNL_ASSERT_GE( receiveBuffers[ ZzYzXp ].getSize(), upperOverlap.x(), "" ); + requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ ZzYzXp ].getData(), upperOverlap.x(), periodicNeighbors[ ZzYzXp ], 1, group ); + requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ ZzYzXp ].getData(), upperOverlap.x(), periodicNeighbors[ ZzYzXp ], 1, group ); } //wait until send and receive is done @@ -181,12 +181,12 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 1, GridReal, D { typedef BufferEntitiesHelper< MeshFunctionType, PeriodicBoundariesMaskPointer, 1, Real_, Device > Helper; - bool leftIsBoundary = ( neighbors[ Left ] == -1 ); - bool rightIsBoundary = ( neighbors[ Right ] == -1 ); + bool leftIsBoundary = ( neighbors[ ZzYzXm ] == -1 ); + bool rightIsBoundary = ( neighbors[ ZzYzXp ] == -1 ); if( ! leftIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ Left ].getData(), leftIsBoundary, left, lowerOverlap.x(), toBuffer ); + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYzXm ].getData(), leftIsBoundary, left, lowerOverlap.x(), toBuffer ); if( ! rightIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ Right ].getData(), rightIsBoundary, right, upperOverlap.x(), toBuffer ); + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYzXp ].getData(), rightIsBoundary, right, upperOverlap.x(), toBuffer ); } Containers::Array sendBuffers[ 2 ], receiveBuffers[ 2 ]; diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_2D.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_2D.h index 51c2aef56..047e81748 100644 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_2D.h +++ b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_2D.h @@ -68,15 +68,14 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 2, GridReal, D const CoordinatesType& localSize = distributedGrid->getLocalSize(); const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); - // TODO: SWAP up and down - sizes[ Left ] = localSize.y() * lowerOverlap.x(); - sizes[ Right ] = localSize.y() * upperOverlap.x(); - sizes[ Up ] = localSize.x() * lowerOverlap.y(); - sizes[ Down ] = localSize.x() * upperOverlap.y(); - sizes[ UpLeft ] = lowerOverlap.x() * lowerOverlap.y(); - sizes[ DownLeft ] = lowerOverlap.x() * upperOverlap.y(); - sizes[ UpRight ] = upperOverlap.x() * lowerOverlap.y(); - sizes[ DownRight ] = upperOverlap.x() * upperOverlap.y(); + sizes[ ZzYzXm ] = localSize.y() * lowerOverlap.x(); + sizes[ ZzYzXp ] = localSize.y() * upperOverlap.x(); + sizes[ ZzYmXz ] = localSize.x() * lowerOverlap.y(); + sizes[ ZzYpXz ] = localSize.x() * upperOverlap.y(); + sizes[ ZzYmXm ] = lowerOverlap.x() * lowerOverlap.y(); + sizes[ ZzYpXm ] = lowerOverlap.x() * upperOverlap.y(); + sizes[ ZzYmXp ] = upperOverlap.x() * lowerOverlap.y(); + sizes[ ZzYpXp ] = upperOverlap.x() * upperOverlap.y(); for(int i=0;i<8;i++) { @@ -104,36 +103,36 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 2, GridReal, D const CoordinatesType& localSize = distributedGrid->getLocalSize(); const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); - leftSource = lowerOverlap.x(); - rightSource = localGridSize.x() - 2 * upperOverlap.x(); - upSource = lowerOverlap.y(); // TODO: SWAP up and down - downSource = localGridSize.y() - 2 * upperOverlap.y(); // TODO: SWAP up and down + ZzYzXm_Source = lowerOverlap.x(); + ZzYzXp_Source = localGridSize.x() - 2 * upperOverlap.x(); + ZzYmXz_Source = lowerOverlap.y(); + ZzYpXz_Source = localGridSize.y() - 2 * upperOverlap.y(); xCenter = lowerOverlap.x(); yCenter = lowerOverlap.y(); - leftDestination = 0; - rightDestination = localGridSize.x() - upperOverlap.x(); - upDestination = 0; - downDestination = localGridSize.y() - upperOverlap.y(); + ZzYzXm_Destination = 0; + ZzYzXp_Destination = localGridSize.x() - upperOverlap.x(); + ZzYmXz_Destination = 0; + ZzYpXz_Destination = localGridSize.y() - upperOverlap.y(); const int *neighbors = distributedGrid->getNeighbors(); const int *periodicNeighbors = distributedGrid->getPeriodicNeighbors(); if( periodicBoundaries ) { - if( neighbors[ Left ] == -1 ) - swap( leftSource, leftDestination ); - if( neighbors[ Right ] == -1 ) - swap( rightSource, rightDestination ); - if( neighbors[ Up ] == -1 ) - swap( upSource, upDestination ); - if( neighbors[ Down ] == -1 ) - swap( downSource, downDestination ); + if( neighbors[ ZzYzXm ] == -1 ) + swap( ZzYzXm_Source, ZzYzXm_Destination ); + if( neighbors[ ZzYzXp ] == -1 ) + swap( ZzYzXp_Source, ZzYzXp_Destination ); + if( neighbors[ ZzYmXz ] == -1 ) + swap( ZzYmXz_Source, ZzYmXz_Destination ); + if( neighbors[ ZzYpXz ] == -1 ) + swap( ZzYpXz_Source, ZzYpXz_Destination ); } copyBuffers(meshFunction, sendBuffers, true, - leftSource, rightSource, upSource, downSource, + ZzYzXm_Source, ZzYzXp_Source, ZzYmXz_Source, ZzYpXz_Source, xCenter, yCenter, lowerOverlap, upperOverlap, localSize, neighbors, @@ -166,7 +165,7 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 2, GridReal, D //copy data from receive buffers copyBuffers(meshFunction, receiveBuffers, false, - leftDestination, rightDestination, upDestination, downDestination, + ZzYzXm_Destination, ZzYzXp_Destination, ZzYmXz_Destination, ZzYpXz_Destination, xCenter, yCenter, lowerOverlap, upperOverlap, localSize, neighbors, @@ -183,7 +182,7 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 2, GridReal, D MeshFunctionType& meshFunction, Containers::Array* buffers, bool toBuffer, - int left, int right, int up, int down, + int ZzYzXm_Position, int ZzYzXp_Position, int ZzYmXz_Position, int ZzYpXz_Position, int xcenter, int ycenter, const CoordinatesType& lowerOverlap, const CoordinatesType& upperOverlap, @@ -192,33 +191,32 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 2, GridReal, D bool periodicBoundaries, const PeriodicBoundariesMaskPointer& mask ) { - // TODO: SWAP up and down - bool leftIsBoundary = ( neighbors[ Left ] == -1 ); - bool rightIsBoundary = ( neighbors[ Right ] == -1 ); - bool upIsBoundary = ( neighbors[ Up ] == -1 ); - bool downIsBoundary = ( neighbors[ Down ] == -1 ); - bool upLeftIsBoundary = ( neighbors[ UpLeft ] == -1 ); - bool upRightIsBoundary = ( neighbors[ UpRight ] == -1 ); - bool downLeftIsBoundary = ( neighbors[ DownLeft ] == -1 ); - bool downRightIsBoundary = ( neighbors[ DownRight ] == -1 ); + bool ZzYzXm_IsBoundary = ( neighbors[ ZzYzXm ] == -1 ); + bool ZzYzXp_IsBoundary = ( neighbors[ ZzYzXp ] == -1 ); + bool ZzYmXz_IsBoundary = ( neighbors[ ZzYmXz ] == -1 ); + bool ZzYpXz_IsBoundary = ( neighbors[ ZzYpXz ] == -1 ); + bool ZzYmXm_IsBoundary = ( neighbors[ ZzYmXm ] == -1 ); + bool ZzYmXp_IsBoundary = ( neighbors[ ZzYmXp ] == -1 ); + bool ZzYpXm_IsBoundary = ( neighbors[ ZzYpXm ] == -1 ); + bool ZzYpXp_IsBoundary = ( neighbors[ ZzYpXp ] == -1 ); using Helper = BufferEntitiesHelper< MeshFunctionType, PeriodicBoundariesMaskPointer, 2, Real_, Device >; - if( ! leftIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ Left ].getData(), leftIsBoundary, left, ycenter, lowerOverlap.x(), localSize.y(), toBuffer ); - if( ! rightIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ Right ].getData(), rightIsBoundary, right, ycenter, upperOverlap.x(), localSize.y(), toBuffer ); - if( ! upIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ Up ].getData(), upIsBoundary, xcenter, up, localSize.x(), lowerOverlap.y(), toBuffer ); - if( ! downIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ Down ].getData(), downIsBoundary, xcenter, down, localSize.x(), upperOverlap.y(), toBuffer ); - if( ! upLeftIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ UpLeft ].getData(), upLeftIsBoundary, left, up, lowerOverlap.x(), lowerOverlap.y(), toBuffer ); - if( ! upRightIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ UpRight ].getData(), upRightIsBoundary, right, up, upperOverlap.x(), lowerOverlap.y(), toBuffer ); - if( ! downLeftIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ DownLeft ].getData(), downLeftIsBoundary, left, down, lowerOverlap.x(), upperOverlap.y(), toBuffer ); - if( ! downRightIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ DownRight ].getData(), downRightIsBoundary, right, down, upperOverlap.x(), upperOverlap.y(), toBuffer ); + if( ! ZzYzXm_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYzXm ].getData(), ZzYzXm_IsBoundary, ZzYzXm_Position, ycenter, lowerOverlap.x(), localSize.y(), toBuffer ); + if( ! ZzYzXp_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYzXp ].getData(), ZzYzXp_IsBoundary, ZzYzXp_Position, ycenter, upperOverlap.x(), localSize.y(), toBuffer ); + if( ! ZzYmXz_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYmXz ].getData(), ZzYmXz_IsBoundary, xcenter, ZzYmXz_Position, localSize.x(), lowerOverlap.y(), toBuffer ); + if( ! ZzYpXz_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYpXz ].getData(), ZzYpXz_IsBoundary, xcenter, ZzYpXz_Position, localSize.x(), upperOverlap.y(), toBuffer ); + if( ! ZzYmXm_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYmXm ].getData(), ZzYmXm_IsBoundary, ZzYzXm_Position, ZzYmXz_Position, lowerOverlap.x(), lowerOverlap.y(), toBuffer ); + if( ! ZzYmXp_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYmXp ].getData(), ZzYmXp_IsBoundary, ZzYzXp_Position, ZzYmXz_Position, upperOverlap.x(), lowerOverlap.y(), toBuffer ); + if( ! ZzYpXm_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYpXm ].getData(), ZzYpXm_IsBoundary, ZzYzXm_Position, ZzYpXz_Position, lowerOverlap.x(), upperOverlap.y(), toBuffer ); + if( ! ZzYpXp_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYpXp ].getData(), ZzYpXp_IsBoundary, ZzYzXp_Position, ZzYpXz_Position, upperOverlap.x(), upperOverlap.y(), toBuffer ); } DistributedGridType *distributedGrid; @@ -227,16 +225,16 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 2, GridReal, D Containers::Array receiveBuffers[8]; Containers::StaticArray< 8, int > sizes; - int leftSource; - int rightSource; - int upSource; - int downSource; + int ZzYzXm_Source; + int ZzYzXp_Source; + int ZzYmXz_Source; + int ZzYpXz_Source; int xCenter; int yCenter; - int leftDestination; - int rightDestination; - int upDestination; - int downDestination; + int ZzYzXm_Destination; + int ZzYzXp_Destination; + int ZzYmXz_Destination; + int ZzYpXz_Destination; bool isSet; }; diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h index 047cf9822..811a12a66 100644 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h +++ b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h @@ -66,34 +66,34 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 3, GridReal, D const CoordinatesType& localSize = distributedGrid->getLocalSize(); const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); - sizes[ West ] = localSize.y() * localSize.z() * lowerOverlap.x(); - sizes[ East ] = localSize.y() * localSize.z() * upperOverlap.x(); - sizes[ North ] = localSize.x() * localSize.z() * lowerOverlap.y(); - sizes[ South ] = localSize.x() * localSize.z() * upperOverlap.y(); - sizes[ Bottom ] = localSize.x() * localSize.y() * lowerOverlap.z(); - sizes[ Top ] = localSize.x() * localSize.y() * upperOverlap.z(); + sizes[ ZzYzXm ] = localSize.y() * localSize.z() * lowerOverlap.x(); + sizes[ ZzYzXp ] = localSize.y() * localSize.z() * upperOverlap.x(); + sizes[ ZzYmXz ] = localSize.x() * localSize.z() * lowerOverlap.y(); + sizes[ ZzYpXz ] = localSize.x() * localSize.z() * upperOverlap.y(); + sizes[ ZmYzXz ] = localSize.x() * localSize.y() * lowerOverlap.z(); + sizes[ ZpYzXz ] = localSize.x() * localSize.y() * upperOverlap.z(); - sizes[ NorthWest ] = localSize.z() * lowerOverlap.x() * lowerOverlap.y(); - sizes[ NorthEast ] = localSize.z() * upperOverlap.x() * lowerOverlap.y(); - sizes[ SouthWest ] = localSize.z() * lowerOverlap.x() * upperOverlap.y(); - sizes[ SouthEast ] = localSize.z() * upperOverlap.x() * upperOverlap.y(); - sizes[ BottomWest ] = localSize.y() * lowerOverlap.x() * lowerOverlap.z(); - sizes[ BottomEast ] = localSize.y() * upperOverlap.x() * lowerOverlap.z(); - sizes[ TopWest ] = localSize.y() * lowerOverlap.x() * upperOverlap.z(); - sizes[ TopEast ] = localSize.y() * upperOverlap.x() * upperOverlap.z(); - sizes[ BottomNorth ] = localSize.x() * lowerOverlap.y() * lowerOverlap.z(); - sizes[ BottomSouth ] = localSize.x() * upperOverlap.y() * lowerOverlap.z(); - sizes[ TopNorth ] = localSize.x() * lowerOverlap.y() * upperOverlap.z(); - sizes[ TopSouth ] = localSize.x() * upperOverlap.y() * upperOverlap.z(); + sizes[ ZzYmXm ] = localSize.z() * lowerOverlap.x() * lowerOverlap.y(); + sizes[ ZzYmXp ] = localSize.z() * upperOverlap.x() * lowerOverlap.y(); + sizes[ ZzYpXm ] = localSize.z() * lowerOverlap.x() * upperOverlap.y(); + sizes[ ZzYpXp ] = localSize.z() * upperOverlap.x() * upperOverlap.y(); + sizes[ ZmYzXp ] = localSize.y() * lowerOverlap.x() * lowerOverlap.z(); + sizes[ ZmYzXm ] = localSize.y() * upperOverlap.x() * lowerOverlap.z(); + sizes[ ZpYzXp ] = localSize.y() * lowerOverlap.x() * upperOverlap.z(); + sizes[ ZpYzXm ] = localSize.y() * upperOverlap.x() * upperOverlap.z(); + sizes[ ZmYmXz ] = localSize.x() * lowerOverlap.y() * lowerOverlap.z(); + sizes[ ZmYpXz ] = localSize.x() * upperOverlap.y() * lowerOverlap.z(); + sizes[ ZpYmXz ] = localSize.x() * lowerOverlap.y() * upperOverlap.z(); + sizes[ ZpYpXz ] = localSize.x() * upperOverlap.y() * upperOverlap.z(); - sizes[ BottomNorthWest ] = lowerOverlap.x() * lowerOverlap.y() * lowerOverlap.z(); - sizes[ BottomNorthEast ] = upperOverlap.x() * lowerOverlap.y() * lowerOverlap.z(); - sizes[ BottomSouthWest ] = lowerOverlap.x() * upperOverlap.y() * lowerOverlap.z(); - sizes[ BottomSouthEast ] = upperOverlap.x() * upperOverlap.y() * lowerOverlap.z(); - sizes[ TopNorthWest ] = lowerOverlap.x() * lowerOverlap.y() * upperOverlap.z(); - sizes[ TopNorthEast ] = upperOverlap.x() * lowerOverlap.y() * upperOverlap.z(); - sizes[ TopSouthWest ] = lowerOverlap.x() * upperOverlap.y() * upperOverlap.z(); - sizes[ TopSouthEast ] = upperOverlap.x() * upperOverlap.y() * upperOverlap.z(); + sizes[ ZmYmXm ] = lowerOverlap.x() * lowerOverlap.y() * lowerOverlap.z(); + sizes[ ZmYmXp ] = upperOverlap.x() * lowerOverlap.y() * lowerOverlap.z(); + sizes[ ZmYpXm ] = lowerOverlap.x() * upperOverlap.y() * lowerOverlap.z(); + sizes[ ZmYpXp ] = upperOverlap.x() * upperOverlap.y() * lowerOverlap.z(); + sizes[ ZpYmXm ] = lowerOverlap.x() * lowerOverlap.y() * upperOverlap.z(); + sizes[ ZpYmXp ] = upperOverlap.x() * lowerOverlap.y() * upperOverlap.z(); + sizes[ ZpYpXm ] = lowerOverlap.x() * upperOverlap.y() * upperOverlap.z(); + sizes[ ZpYpXp ] = upperOverlap.x() * upperOverlap.y() * upperOverlap.z(); for( int i=0; i<26; i++ ) { @@ -120,46 +120,46 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 3, GridReal, D const CoordinatesType& localSize = distributedGrid->getLocalSize(); const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); - westSource = lowerOverlap.x(); - eastSource = localGridSize.x() - 2 * upperOverlap.x(); - northSource = lowerOverlap.y(); - southSource = localGridSize.y() - 2 * upperOverlap.y(); - bottomSource = lowerOverlap.z(); - topSource = localGridSize.z() - 2 * upperOverlap.z(); + ZzYzXm_Source = lowerOverlap.x(); + ZzYzXp_Source = localGridSize.x() - 2 * upperOverlap.x(); + ZzYmXz_Source = lowerOverlap.y(); + ZzYpXz_Source = localGridSize.y() - 2 * upperOverlap.y(); + ZmYzXz_Source = lowerOverlap.z(); + ZpYzXz_Source = localGridSize.z() - 2 * upperOverlap.z(); xCenter = lowerOverlap.x(); yCenter = lowerOverlap.y(); zCenter = lowerOverlap.z(); - westDestination = 0; - eastDestination = localGridSize.x() - upperOverlap.x(); - northDestination = 0; - southDestination = localGridSize.y() - upperOverlap.y(); - bottomDestination = 0; - topDestination = localGridSize.z() - upperOverlap.z(); + ZzYzXm_Destination = 0; + ZzYzXp_Destination = localGridSize.x() - upperOverlap.x(); + ZzYmXz_Destination = 0; + ZzYpXz_Destination = localGridSize.y() - upperOverlap.y(); + ZmYzXz_Destination = 0; + ZpYzXz_Destination = localGridSize.z() - upperOverlap.z(); const int *neighbors = distributedGrid->getNeighbors(); const int *periodicNeighbors = distributedGrid->getPeriodicNeighbors(); if( periodicBoundaries ) { - if( neighbors[ West ] == -1 ) - swap( westSource, westDestination ); - if( neighbors[ East ] == -1 ) - swap( eastSource, eastDestination ); - if( neighbors[ South ] == -1 ) - swap( southSource, southDestination ); - if( neighbors[ North ] == -1 ) - swap( northSource, northDestination ); - if( neighbors[ Bottom ] == -1 ) - swap( bottomSource, bottomDestination ); - if( neighbors[ Top ] == -1 ) - swap( topSource, topDestination ); + if( neighbors[ ZzYzXm ] == -1 ) + swap( ZzYzXm_Source, ZzYzXm_Destination ); + if( neighbors[ ZzYzXp ] == -1 ) + swap( ZzYzXp_Source, ZzYzXp_Destination ); + if( neighbors[ ZzYpXz ] == -1 ) + swap( ZzYpXz_Source, ZzYpXz_Destination ); + if( neighbors[ ZzYmXz ] == -1 ) + swap( ZzYmXz_Source, ZzYmXz_Destination ); + if( neighbors[ ZmYzXz ] == -1 ) + swap( ZmYzXz_Source, ZmYzXz_Destination ); + if( neighbors[ ZpYzXz ] == -1 ) + swap( ZpYzXz_Source, ZpYzXz_Destination ); } //fill send buffers copyBuffers( meshFunction, sendBuffers, true, - westSource, eastSource, northSource, southSource, bottomSource, topSource, + ZzYzXm_Source, ZzYzXp_Source, ZzYmXz_Source, ZzYpXz_Source, ZmYzXz_Source, ZpYzXz_Source, xCenter, yCenter, zCenter, lowerOverlap, upperOverlap, localSize, neighbors, @@ -190,7 +190,7 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 3, GridReal, D //copy data from receive buffers copyBuffers(meshFunction, receiveBuffers, false, - westDestination, eastDestination, northDestination, southDestination, bottomDestination, topDestination, + ZzYzXm_Destination, ZzYzXp_Destination, ZzYmXz_Destination, ZzYpXz_Destination, ZmYzXz_Destination, ZpYzXz_Destination, xCenter, yCenter, zCenter, lowerOverlap, upperOverlap, localSize, neighbors, @@ -207,7 +207,7 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 3, GridReal, D MeshFunctionType& meshFunction, Containers::Array* buffers, bool toBuffer, - int west, int east, int north, int south, int bottom, int top, + int ZzYzXm, int ZzYzXp, int ZzYmXz, int ZzYpXz, int ZmYzXz, int ZpYzXz, int xcenter, int ycenter, int zcenter, const CoordinatesType& lowerOverlap, const CoordinatesType& upperOverlap, @@ -216,100 +216,101 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 3, GridReal, D bool periodicBoundaries, const PeriodicBoundariesMaskPointer& mask ) { - bool westIsBoundary = ( neighbor[ West ] == -1 ); - bool eastIsBoundary = ( neighbor[ East ] == -1 ); - bool northIsBoundary = ( neighbor[ North ] == -1 ); - bool southIsBoundary = ( neighbor[ South ] == -1 ); - bool bottomIsBoundary = ( neighbor[ Bottom ] == -1 ); - bool topIsBoundary = ( neighbor[ Top ] == -1 ); + //TODO: refactor to array IsBoundary[number of neighbor] + bool ZzYzXm_IsBoundary = ( neighbor[ ZzYzXm ] == -1 ); + bool ZzYzXp_IsBoundary = ( neighbor[ ZzYzXp ] == -1 ); + bool ZzYmXz_IsBoundary = ( neighbor[ ZzYmXz ] == -1 ); + bool ZzYpXz_IsBoundary = ( neighbor[ ZzYpXz ] == -1 ); + bool ZmYzXz_IsBoundary = ( neighbor[ ZmYzXz ] == -1 ); + bool ZpYzXz_IsBoundary = ( neighbor[ ZpYzXz ] == -1 ); - bool northWestIsBoundary = ( neighbor[ NorthWest ] == -1 ); - bool northEastIsBoundary = ( neighbor[ NorthEast ] == -1 ); - bool southWestIsBoundary = ( neighbor[ SouthWest ] == -1 ); - bool southEastIsBoundary = ( neighbor[ SouthEast ] == -1 ); + bool ZzYmXm_IsBoundary = ( neighbor[ ZzYmXm ] == -1 ); + bool ZzYmXp_IsBoundary = ( neighbor[ ZzYmXp ] == -1 ); + bool ZzYpXm_IsBoundary = ( neighbor[ ZzYpXm ] == -1 ); + bool ZzYpXp_IsBoundary = ( neighbor[ ZzYpXp ] == -1 ); - bool bottomWestIsBoundary = ( neighbor[ BottomWest ] == -1 ); - bool bottomEastIsBoundary = ( neighbor[ BottomEast ] == -1 ); - bool bottomNorthIsBoundary = ( neighbor[ BottomNorth ] == -1 ); - bool bottomSouthIsBoundary = ( neighbor[ BottomSouth ] == -1 ); + bool ZmYzXp_IsBoundary = ( neighbor[ ZmYzXp ] == -1 ); + bool ZmYzXm_IsBoundary = ( neighbor[ ZmYzXm ] == -1 ); + bool ZmYmXz_IsBoundary = ( neighbor[ ZmYmXz ] == -1 ); + bool ZmYpXz_IsBoundary = ( neighbor[ ZmYpXz ] == -1 ); - bool topWestIsBoundary = ( neighbor[ TopWest ] == -1 ); - bool topEastIsBoundary = ( neighbor[ TopEast ] == -1 ); - bool topNorthIsBoundary = ( neighbor[ TopNorth ] == -1 ); - bool topSouthIsBoundary = ( neighbor[ TopSouth ] == -1 ); + bool ZpYzXp_IsBoundary = ( neighbor[ ZpYzXp ] == -1 ); + bool ZpYzXm_IsBoundary = ( neighbor[ ZpYzXm ] == -1 ); + bool ZpYmXz_IsBoundary = ( neighbor[ ZpYmXz ] == -1 ); + bool ZpYpXz_IsBoundary = ( neighbor[ ZpYpXz ] == -1 ); - bool bottomNorthWestIsBoundary = ( neighbor[ BottomNorthWest ] == -1 ); - bool bottomNorthEastIsBoundary = ( neighbor[ BottomNorthEast ] == -1 ); - bool bottomSouthWestIsBoundary = ( neighbor[ BottomSouthWest ] == -1 ); - bool bottomSouthEastIsBoundary = ( neighbor[ BottomSouthEast ] == -1 ); + bool ZmYmXm_IsBoundary = ( neighbor[ ZmYmXm ] == -1 ); + bool ZmYmXp_IsBoundary = ( neighbor[ ZmYmXp ] == -1 ); + bool ZmYpXm_IsBoundary = ( neighbor[ ZmYpXm ] == -1 ); + bool ZmYpXp_IsBoundary = ( neighbor[ ZmYpXp ] == -1 ); - bool topNorthWestIsBoundary = ( neighbor[ TopNorthWest ] == -1 ); - bool topNorthEastIsBoundary = ( neighbor[ TopNorthEast ] == -1 ); - bool topSouthWestIsBoundary = ( neighbor[ TopSouthWest ] == -1 ); - bool topSouthEastIsBoundary = ( neighbor[ TopSouthEast ] == -1 ); + bool ZpYmXm_IsBoundary = ( neighbor[ ZpYmXm ] == -1 ); + bool ZpYmXp_IsBoundary = ( neighbor[ ZpYmXp ] == -1 ); + bool ZpYpXm_IsBoundary = ( neighbor[ ZpYpXm ] == -1 ); + bool ZpYpXp_IsBoundary = ( neighbor[ ZpYpXp ] == -1 ); using Helper = BufferEntitiesHelper< MeshFunctionType, PeriodicBoundariesMaskPointer, 3, Real_, Device >; //X-Y-Z - if( ! westIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ West ].getData(), westIsBoundary, west, ycenter, zcenter, lowerOverlap.x(), localSize.y(), localSize.z(), toBuffer ); - if( ! eastIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ East ].getData(), eastIsBoundary, east, ycenter, zcenter, upperOverlap.x(), localSize.y(), localSize.z(), toBuffer ); - if( ! northIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ North ].getData(), northIsBoundary, xcenter, north, zcenter, localSize.x(), lowerOverlap.y(), localSize.z(), toBuffer ); - if( ! southIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ South ].getData(), southIsBoundary, xcenter, south, zcenter, localSize.x(), upperOverlap.y(), localSize.z(), toBuffer ); - if( ! bottomIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ Bottom ].getData(), bottomIsBoundary, xcenter, ycenter, bottom, localSize.x(), localSize.y(), lowerOverlap.z(), toBuffer ); - if( ! topIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ Top ].getData(), topIsBoundary, xcenter, ycenter, top, localSize.x(), localSize.y(), upperOverlap.z(), toBuffer ); + if( ! ZzYzXm_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYzXm ].getData(), ZzYzXm_IsBoundary, ZzYzXm, ycenter, zcenter, lowerOverlap.x(), localSize.y(), localSize.z(), toBuffer ); + if( ! ZzYzXp_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYzXp ].getData(), ZzYzXp_IsBoundary, ZzYzXp, ycenter, zcenter, upperOverlap.x(), localSize.y(), localSize.z(), toBuffer ); + if( ! ZzYmXz_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYmXz ].getData(), ZzYmXz_IsBoundary, xcenter, ZzYmXz, zcenter, localSize.x(), lowerOverlap.y(), localSize.z(), toBuffer ); + if( ! ZzYpXz_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYpXz ].getData(), ZzYpXz_IsBoundary, xcenter, ZzYpXz, zcenter, localSize.x(), upperOverlap.y(), localSize.z(), toBuffer ); + if( ! ZmYzXz_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZmYzXz ].getData(), ZmYzXz_IsBoundary, xcenter, ycenter, ZmYzXz, localSize.x(), localSize.y(), lowerOverlap.z(), toBuffer ); + if( ! ZpYzXz_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZpYzXz ].getData(), ZpYzXz_IsBoundary, xcenter, ycenter, ZpYzXz, localSize.x(), localSize.y(), upperOverlap.z(), toBuffer ); //XY - if( ! northWestIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ NorthWest ].getData(), northWestIsBoundary, west, north, zcenter, lowerOverlap.x(), lowerOverlap.y(), localSize.z(), toBuffer ); - if( ! northEastIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ NorthEast ].getData(), northEastIsBoundary, east, north, zcenter, upperOverlap.x(), lowerOverlap.y(), localSize.z(), toBuffer ); - if( ! southWestIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ SouthWest ].getData(), southWestIsBoundary, west, south, zcenter, lowerOverlap.x(), upperOverlap.y(), localSize.z(), toBuffer ); - if( ! southEastIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ SouthEast ].getData(), southEastIsBoundary, east, south, zcenter, upperOverlap.x(), upperOverlap.y(), localSize.z(), toBuffer ); + if( ! ZzYmXm_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYmXm ].getData(), ZzYmXm_IsBoundary, ZzYzXm, ZzYmXz, zcenter, lowerOverlap.x(), lowerOverlap.y(), localSize.z(), toBuffer ); + if( ! ZzYmXp_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYmXp ].getData(), ZzYmXp_IsBoundary, ZzYzXp, ZzYmXz, zcenter, upperOverlap.x(), lowerOverlap.y(), localSize.z(), toBuffer ); + if( ! ZzYpXm_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYpXm ].getData(), ZzYpXm_IsBoundary, ZzYzXm, ZzYpXz, zcenter, lowerOverlap.x(), upperOverlap.y(), localSize.z(), toBuffer ); + if( ! ZzYpXp_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZzYpXp ].getData(), ZzYpXp_IsBoundary, ZzYzXp, ZzYpXz, zcenter, upperOverlap.x(), upperOverlap.y(), localSize.z(), toBuffer ); //XZ - if( ! bottomWestIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ BottomWest ].getData(), bottomWestIsBoundary, west, ycenter, bottom, lowerOverlap.x(), localSize.y(), lowerOverlap.z(), toBuffer ); - if( ! bottomEastIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ BottomEast ].getData(), bottomEastIsBoundary, east, ycenter, bottom, upperOverlap.x(), localSize.y(), lowerOverlap.z(), toBuffer ); - if( ! topWestIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ TopWest ].getData(), topWestIsBoundary, west, ycenter, top, lowerOverlap.x(), localSize.y(), upperOverlap.z(), toBuffer ); - if( ! topEastIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ TopEast ].getData(), topEastIsBoundary, east, ycenter, top, upperOverlap.x(), localSize.y(), upperOverlap.z(), toBuffer ); + if( ! ZmYzXp_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZmYzXp ].getData(), ZmYzXp_IsBoundary, ZzYzXm, ycenter, ZmYzXz, lowerOverlap.x(), localSize.y(), lowerOverlap.z(), toBuffer ); + if( ! ZmYzXm_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZmYzXm ].getData(), ZmYzXm_IsBoundary, ZzYzXp, ycenter, ZmYzXz, upperOverlap.x(), localSize.y(), lowerOverlap.z(), toBuffer ); + if( ! ZpYzXp_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZpYzXp ].getData(), ZpYzXp_IsBoundary, ZzYzXm, ycenter, ZpYzXz, lowerOverlap.x(), localSize.y(), upperOverlap.z(), toBuffer ); + if( ! ZpYzXm_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZpYzXm ].getData(), ZpYzXm_IsBoundary, ZzYzXp, ycenter, ZpYzXz, upperOverlap.x(), localSize.y(), upperOverlap.z(), toBuffer ); //YZ - if( ! bottomNorthIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ BottomNorth ].getData(), bottomNorthIsBoundary, xcenter, north, bottom, localSize.x(), lowerOverlap.y(), lowerOverlap.z(), toBuffer ); - if( ! bottomSouthIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ BottomSouth ].getData(), bottomSouthIsBoundary, xcenter, south, bottom, localSize.x(), upperOverlap.y(), lowerOverlap.z(), toBuffer ); - if( ! topNorthIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ TopNorth ].getData(), topNorthIsBoundary, xcenter, north, top, localSize.x(), lowerOverlap.y(), upperOverlap.z(), toBuffer ); - if( ! topSouthIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ TopSouth ].getData(), topSouthIsBoundary, xcenter, south, top, localSize.x(), upperOverlap.y(), upperOverlap.z(), toBuffer ); + if( ! ZmYmXz_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZmYmXz ].getData(), ZmYmXz_IsBoundary, xcenter, ZzYmXz, ZmYzXz, localSize.x(), lowerOverlap.y(), lowerOverlap.z(), toBuffer ); + if( ! ZmYpXz_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZmYpXz ].getData(), ZmYpXz_IsBoundary, xcenter, ZzYpXz, ZmYzXz, localSize.x(), upperOverlap.y(), lowerOverlap.z(), toBuffer ); + if( ! ZpYmXz_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZpYmXz ].getData(), ZpYmXz_IsBoundary, xcenter, ZzYmXz, ZpYzXz, localSize.x(), lowerOverlap.y(), upperOverlap.z(), toBuffer ); + if( ! ZpYpXz_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZpYpXz ].getData(), ZpYpXz_IsBoundary, xcenter, ZzYpXz, ZpYzXz, localSize.x(), upperOverlap.y(), upperOverlap.z(), toBuffer ); //XYZ - if( ! bottomNorthWestIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ BottomNorthWest ].getData(), bottomNorthWestIsBoundary, west, north, bottom, lowerOverlap.x(), lowerOverlap.y(), lowerOverlap.z(), toBuffer ); - if( ! bottomNorthEastIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ BottomNorthEast ].getData(), bottomNorthEastIsBoundary, east, north, bottom, upperOverlap.x(), lowerOverlap.y(), lowerOverlap.z(), toBuffer ); - if( ! bottomSouthWestIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ BottomSouthWest ].getData(), bottomSouthWestIsBoundary, west, south, bottom, lowerOverlap.x(), upperOverlap.y(), lowerOverlap.z(), toBuffer ); - if( ! bottomSouthEastIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ BottomSouthEast ].getData(), bottomSouthEastIsBoundary, east, south, bottom, upperOverlap.x(), upperOverlap.y(), lowerOverlap.z(), toBuffer ); - if( ! topNorthWestIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ TopNorthWest ].getData(), topNorthWestIsBoundary, west, north, top, lowerOverlap.x(), lowerOverlap.y(), upperOverlap.z(), toBuffer ); - if( ! topNorthEastIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ TopNorthEast ].getData(), topNorthEastIsBoundary, east, north, top, upperOverlap.x(), lowerOverlap.y(), upperOverlap.z(), toBuffer ); - if( ! topSouthWestIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ TopSouthWest ].getData(), topSouthEastIsBoundary, west, south, top, lowerOverlap.x(), upperOverlap.y(), upperOverlap.z(), toBuffer ); - if( ! topSouthEastIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ TopSouthEast ].getData(), topSouthEastIsBoundary, east, south, top, upperOverlap.x(), upperOverlap.y(), upperOverlap.z(), toBuffer ); + if( ! ZmYmXm_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZmYmXm ].getData(), ZmYmXm_IsBoundary, ZzYzXm, ZzYmXz, ZmYzXz, lowerOverlap.x(), lowerOverlap.y(), lowerOverlap.z(), toBuffer ); + if( ! ZmYmXp_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZmYmXp ].getData(), ZmYmXp_IsBoundary, ZzYzXp, ZzYmXz, ZmYzXz, upperOverlap.x(), lowerOverlap.y(), lowerOverlap.z(), toBuffer ); + if( ! ZmYpXm_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZmYpXm ].getData(), ZmYpXm_IsBoundary, ZzYzXm, ZzYpXz, ZmYzXz, lowerOverlap.x(), upperOverlap.y(), lowerOverlap.z(), toBuffer ); + if( ! ZmYpXp_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZmYpXp ].getData(), ZmYpXp_IsBoundary, ZzYzXp, ZzYpXz, ZmYzXz, upperOverlap.x(), upperOverlap.y(), lowerOverlap.z(), toBuffer ); + if( ! ZpYmXm_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZpYmXm ].getData(), ZpYmXm_IsBoundary, ZzYzXm, ZzYmXz, ZpYzXz, lowerOverlap.x(), lowerOverlap.y(), upperOverlap.z(), toBuffer ); + if( ! ZpYmXp_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZpYmXp ].getData(), ZpYmXp_IsBoundary, ZzYzXp, ZzYmXz, ZpYzXz, upperOverlap.x(), lowerOverlap.y(), upperOverlap.z(), toBuffer ); + if( ! ZpYpXm_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZpYpXm ].getData(), ZpYpXp_IsBoundary, ZzYzXm, ZzYpXz, ZpYzXz, lowerOverlap.x(), upperOverlap.y(), upperOverlap.z(), toBuffer ); + if( ! ZpYpXp_IsBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ ZpYpXp ].getData(), ZpYpXp_IsBoundary, ZzYzXp, ZzYpXz, ZpYzXz, upperOverlap.x(), upperOverlap.y(), upperOverlap.z(), toBuffer ); } private: @@ -320,21 +321,21 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 3, GridReal, D DistributedGridType *distributedGrid; - int westSource; - int eastSource; - int northSource; - int southSource; - int bottomSource; - int topSource; + int ZzYzXm_Source; + int ZzYzXp_Source; + int ZzYmXz_Source; + int ZzYpXz_Source; + int ZmYzXz_Source; + int ZpYzXz_Source; int xCenter; int yCenter; int zCenter; - int westDestination; - int eastDestination; - int northDestination; - int southDestination; - int bottomDestination; - int topDestination; + int ZzYzXm_Destination; + int ZzYzXp_Destination; + int ZzYmXz_Destination; + int ZzYpXz_Destination; + int ZmYzXz_Destination; + int ZpYzXz_Destination; CoordinatesType overlap; CoordinatesType localSize; diff --git a/src/TNL/Meshes/GridDetails/Traverser_Grid1D_impl.h b/src/TNL/Meshes/GridDetails/Traverser_Grid1D_impl.h index d3d2a129c..c0ddcf2da 100644 --- a/src/TNL/Meshes/GridDetails/Traverser_Grid1D_impl.h +++ b/src/TNL/Meshes/GridDetails/Traverser_Grid1D_impl.h @@ -48,7 +48,7 @@ processBoundaryEntities( const GridPointer& gridPointer, else //Distributed { const int* neighbors=distributedGrid->getNeighbors(); - if( neighbors[ Meshes::DistributedMeshes::Left ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYzXm ] == -1 ) { GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >( gridPointer, @@ -57,7 +57,7 @@ processBoundaryEntities( const GridPointer& gridPointer, userData ); } - if( neighbors[ Meshes::DistributedMeshes::Right ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYzXp ] == -1 ) { GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >( gridPointer, @@ -100,10 +100,10 @@ processInteriorEntities( const GridPointer& gridPointer, CoordinatesType end( gridPointer->getDimensions() - distributedGrid->getUpperOverlap() - CoordinatesType( 1 ) ); const int* neighbors = distributedGrid->getNeighbors(); - if( neighbors[ Meshes::DistributedMeshes::Left ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYzXm ] == -1 ) begin += CoordinatesType( 1 ); - if( neighbors[ Meshes::DistributedMeshes::Right ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYzXp ] == -1 ) end -= CoordinatesType( 1 ); /* diff --git a/src/TNL/Meshes/GridDetails/Traverser_Grid2D_impl.h b/src/TNL/Meshes/GridDetails/Traverser_Grid2D_impl.h index 23d93d7e0..41e161256 100644 --- a/src/TNL/Meshes/GridDetails/Traverser_Grid2D_impl.h +++ b/src/TNL/Meshes/GridDetails/Traverser_Grid2D_impl.h @@ -50,7 +50,7 @@ processBoundaryEntities( const GridPointer& gridPointer, const CoordinatesType end = gridPointer->getDimensions() - distributedGrid->getUpperOverlap() - CoordinatesType( 1, 1 ); const int* neighbors=distributedGrid->getNeighbors(); - if( neighbors[ Meshes::DistributedMeshes::Left ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYzXm ] == -1 ) { GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >( gridPointer, @@ -60,7 +60,7 @@ processBoundaryEntities( const GridPointer& gridPointer, 0 ); } - if(neighbors[Meshes::DistributedMeshes::Right]==-1) + if(neighbors[Meshes::DistributedMeshes::ZzYzXp]==-1) { GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >( gridPointer, @@ -70,8 +70,7 @@ processBoundaryEntities( const GridPointer& gridPointer, 0 ); } - // TODO: Up and Down should by swapped, I think - if( neighbors[ Meshes::DistributedMeshes::Up ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYmXz ] == -1 ) { GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >( gridPointer, @@ -81,7 +80,7 @@ processBoundaryEntities( const GridPointer& gridPointer, 0 ); } - if( neighbors[ Meshes::DistributedMeshes::Down ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYpXz ] == -1 ) { GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >( gridPointer, @@ -124,16 +123,16 @@ processInteriorEntities( const GridPointer& gridPointer, const int* neighbors = distributedGrid->getNeighbors(); CoordinatesType begin( distributedGrid->getLowerOverlap()); CoordinatesType end( gridPointer->getDimensions() - distributedGrid->getUpperOverlap()- CoordinatesType(1,1) ); - if( neighbors[ Meshes::DistributedMeshes::Left ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYzXm ] == -1 ) begin.x() += 1 ; - if( neighbors[ Meshes::DistributedMeshes::Right ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYzXp ] == -1 ) end.x() -= 1; - if( neighbors[ Meshes::DistributedMeshes::Up ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYmXz ] == -1 ) begin.y() += 1 ; - if( neighbors[ Meshes::DistributedMeshes::Down ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYpXz ] == -1 ) end.y() -= 1; diff --git a/src/TNL/Meshes/GridDetails/Traverser_Grid3D_impl.h b/src/TNL/Meshes/GridDetails/Traverser_Grid3D_impl.h index 3c9fffd81..e32c5a12e 100644 --- a/src/TNL/Meshes/GridDetails/Traverser_Grid3D_impl.h +++ b/src/TNL/Meshes/GridDetails/Traverser_Grid3D_impl.h @@ -52,7 +52,7 @@ processBoundaryEntities( const GridPointer& gridPointer, const CoordinatesType end = gridPointer->getDimensions() - distributedGrid->getUpperOverlap() - CoordinatesType( 1, 1, 1 ); const int* neighbors = distributedGrid->getNeighbors(); - if( neighbors[ Meshes::DistributedMeshes::West ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYzXm ] == -1 ) { GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >( gridPointer, @@ -62,7 +62,7 @@ processBoundaryEntities( const GridPointer& gridPointer, 0 ); } - if( neighbors[ Meshes::DistributedMeshes::East ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYzXp ] == -1 ) { GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >( gridPointer, @@ -72,7 +72,7 @@ processBoundaryEntities( const GridPointer& gridPointer, 0 ); } - if( neighbors[ Meshes::DistributedMeshes::North ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYmXz ] == -1 ) { GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >( gridPointer, @@ -82,7 +82,7 @@ processBoundaryEntities( const GridPointer& gridPointer, 0 ); } - if( neighbors[ Meshes::DistributedMeshes::South ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYpXz ] == -1 ) { GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >( gridPointer, @@ -92,7 +92,7 @@ processBoundaryEntities( const GridPointer& gridPointer, 0 ); } - if( neighbors[ Meshes::DistributedMeshes::Bottom ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZmYzXz ] == -1 ) { GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >( gridPointer, @@ -102,7 +102,7 @@ processBoundaryEntities( const GridPointer& gridPointer, 0 ); } - if( neighbors[ Meshes::DistributedMeshes::Top ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZpYzXz ] == -1 ) { GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >( gridPointer, @@ -146,22 +146,22 @@ processInteriorEntities( const GridPointer& gridPointer, CoordinatesType begin( distributedGrid->getLowerOverlap()); CoordinatesType end( gridPointer->getDimensions() - distributedGrid->getUpperOverlap()- CoordinatesType(1,1,1) ); - if( neighbors[ Meshes::DistributedMeshes::West ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYzXm ] == -1 ) begin.x() += 1 ; - if( neighbors[ Meshes::DistributedMeshes::East ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYzXp ] == -1 ) end.x() -= 1; - if( neighbors[ Meshes::DistributedMeshes::North ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYmXz ] == -1 ) begin.y() += 1; - if( neighbors[ Meshes::DistributedMeshes::South ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZzYpXz ] == -1 ) end.y() -= 1; - if( neighbors[ Meshes::DistributedMeshes::Bottom ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZmYzXz ] == -1 ) begin.z() += 1 ; - if( neighbors[ Meshes::DistributedMeshes::Top ] == -1 ) + if( neighbors[ Meshes::DistributedMeshes::ZpYzXz ] == -1 ) end.z() -= 1; GridTraverser< GridType >::template processEntities< GridEntity, EntitiesProcessor, UserData, false >( diff --git a/src/UnitTests/Meshes/DistributedMeshes/DirectionsTest.cpp b/src/UnitTests/Meshes/DistributedMeshes/DirectionsTest.cpp index 0d2bc77ed..b772ecda0 100644 --- a/src/UnitTests/Meshes/DistributedMeshes/DirectionsTest.cpp +++ b/src/UnitTests/Meshes/DistributedMeshes/DirectionsTest.cpp @@ -10,111 +10,111 @@ using namespace TNL; TEST(Direction1D, Conners) { - EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(-1)),Left) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(1)),Right) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(-1)),ZzYzXm) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(1)),ZzYzXp) << "Failed"; } TEST(Direction2D, Edge) { - EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(-1)),Left) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(1)),Right) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(-2)),Up) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(2)),Down) << "Failed"; - - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-1,0)),Left) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(1,0)),Right) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(0,-2)),Up) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(0,2)),Down) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(-1)),ZzYzXm) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(1)),ZzYzXp) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(-2)),ZzYmXz) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(2)),ZzYpXz) << "Failed"; + + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-1,0)),ZzYzXm) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(1,0)),ZzYzXp) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(0,-2)),ZzYmXz) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(0,2)),ZzYpXz) << "Failed"; } TEST(Direction2D, Conners) { - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-2,-1)),UpLeft) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-2, 1)),UpRight) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(2,-1)),DownLeft) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(2,1)),DownRight) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-2,-1)),ZzYmXm) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-2, 1)),ZzYmXp) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(2,-1)),ZzYpXm) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(2,1)),ZzYpXp) << "Failed"; } TEST(Direction3D, Faces) { - EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(-1)),West) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(1)),East) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(-2)),North) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(2)),South) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(-3)),Bottom) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(3)),Top) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(-1)),ZzYzXm) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(1)),ZzYzXp) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(-2)),ZzYmXz) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(2)),ZzYpXz) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(-3)),ZmYzXz) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<1,int>(3)),ZpYzXz) << "Failed"; } TEST(Direction3D, Edges) { - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-2,-1)),NorthWest) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-2, 1)),NorthEast) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>( 2,-1)),SouthWest) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>( 2, 1)),SouthEast) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-2,-1)),ZzYmXm) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-2, 1)),ZzYmXp) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>( 2,-1)),ZzYpXm) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>( 2, 1)),ZzYpXp) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-3,-1)),BottomWest) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-3, 1)),BottomEast) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-3,-2)),BottomNorth) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-3, 2)),BottomSouth) << "Failed"; - - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>( 3,-1)),TopWest) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>( 3, 1)),TopEast) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>( 3,-2)),TopNorth) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<2,int>( 3, 2)),TopSouth) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-3,-1)),ZmYzXm) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-3, 1)),ZmYzXp) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-3,-2)),ZmYmXz) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>(-3, 2)),ZmYpXz) << "Failed"; + + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>( 3,-1)),ZpYzXm) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>( 3, 1)),ZpYzXp) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>( 3,-2)),ZpYmXz) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<2,int>( 3, 2)),ZpYpXz) << "Failed"; } TEST(Direction3D, Conners) { - EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(-3,-2,-1)),BottomNorthWest) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(-3,-2, 1)),BottomNorthEast) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(-3, 2,-1)),BottomSouthWest) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(-3, 2, 1)),BottomSouthEast) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(-3,-2,-1)),ZmYmXm) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(-3,-2, 1)),ZmYmXp) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(-3, 2,-1)),ZmYpXm) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(-3, 2, 1)),ZmYpXp) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(3,-2,-1)),TopNorthWest) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(3,-2, 1)),TopNorthEast) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(3, 2,-1)),TopSouthWest) << "Failed"; - EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(3, 2, 1)),TopSouthEast) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(3,-2,-1)),ZpYmXm) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(3,-2, 1)),ZpYmXp) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(3, 2,-1)),ZpYpXm) << "Failed"; + EXPECT_EQ(Directions::getDirection(StaticVector<3,int>(3, 2, 1)),ZpYpXp) << "Failed"; } TEST(XYZ, 2D ) { - EXPECT_EQ( Directions::template getXYZ<2>(Left), (StaticVector<2,int>(-1,0)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<2>(Right), (StaticVector<2,int>(1,0)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<2>(Up), (StaticVector<2,int>(0,-1)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<2>(Down), (StaticVector<2,int>(0,1)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<2>(ZzYzXm), (StaticVector<2,int>(-1,0)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<2>(ZzYzXp), (StaticVector<2,int>(1,0)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<2>(ZzYmXz), (StaticVector<2,int>(0,-1)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<2>(ZzYpXz), (StaticVector<2,int>(0,1)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<2>(UpLeft), (StaticVector<2,int>(-1,-1)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<2>(UpRight), (StaticVector<2,int>(1,-1)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<2>(DownLeft), (StaticVector<2,int>(-1,1)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<2>(DownRight), (StaticVector<2,int>(1,1)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<2>(ZzYmXm), (StaticVector<2,int>(-1,-1)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<2>(ZzYmXp), (StaticVector<2,int>(1,-1)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<2>(ZzYpXm), (StaticVector<2,int>(-1,1)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<2>(ZzYpXp), (StaticVector<2,int>(1,1)) ) << "Failed"; } TEST(XYZ, 3D ) { - EXPECT_EQ( Directions::template getXYZ<3>(West), (StaticVector<3,int>(-1,0,0)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<3>(East), (StaticVector<3,int>(1,0,0)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<3>(North), (StaticVector<3,int>(0,-1,0)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<3>(South), (StaticVector<3,int>(0,1,0)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<3>(Bottom), (StaticVector<3,int>(0,0,-1)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<3>(Top), (StaticVector<3,int>(0,0,1)) ) << "Failed"; - - EXPECT_EQ( Directions::template getXYZ<3>(NorthWest), (StaticVector<3,int>(-1,-1,0)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<3>(NorthEast), (StaticVector<3,int>(1,-1,0)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<3>(SouthWest), (StaticVector<3,int>(-1,1,0)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<3>(SouthEast), (StaticVector<3,int>(1,1,0)) ) << "Failed"; - - EXPECT_EQ( Directions::template getXYZ<3>(BottomWest), (StaticVector<3,int>(-1,0,-1)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<3>(BottomEast), (StaticVector<3,int>(1,0,-1)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<3>(TopWest), (StaticVector<3,int>(-1,0,1)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<3>(TopEast), (StaticVector<3,int>(1,0,1)) ) << "Failed"; - - EXPECT_EQ( Directions::template getXYZ<3>(BottomNorth), (StaticVector<3,int>(0,-1,-1)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<3>(BottomSouth), (StaticVector<3,int>(0,1,-1)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<3>(TopNorth), (StaticVector<3,int>(0,-1,1)) ) << "Failed"; - EXPECT_EQ( Directions::template getXYZ<3>(TopSouth), (StaticVector<3,int>(0,1,1)) ) << "Failed"; - - EXPECT_EQ( Directions::template getXYZ<3>(TopSouthWest), (StaticVector<3,int>(-1,1,1)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZzYzXm), (StaticVector<3,int>(-1,0,0)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZzYzXp), (StaticVector<3,int>(1,0,0)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZzYmXz), (StaticVector<3,int>(0,-1,0)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZzYpXz), (StaticVector<3,int>(0,1,0)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZmYzXz), (StaticVector<3,int>(0,0,-1)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZpYzXz), (StaticVector<3,int>(0,0,1)) ) << "Failed"; + + EXPECT_EQ( Directions::template getXYZ<3>(ZzYmXm), (StaticVector<3,int>(-1,-1,0)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZzYmXp), (StaticVector<3,int>(1,-1,0)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZzYpXm), (StaticVector<3,int>(-1,1,0)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZzYpXp), (StaticVector<3,int>(1,1,0)) ) << "Failed"; + + EXPECT_EQ( Directions::template getXYZ<3>(ZmYzXm), (StaticVector<3,int>(-1,0,-1)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZmYzXp), (StaticVector<3,int>(1,0,-1)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZpYzXm), (StaticVector<3,int>(-1,0,1)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZpYzXp), (StaticVector<3,int>(1,0,1)) ) << "Failed"; + + EXPECT_EQ( Directions::template getXYZ<3>(ZmYmXz), (StaticVector<3,int>(0,-1,-1)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZmYpXz), (StaticVector<3,int>(0,1,-1)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZpYmXz), (StaticVector<3,int>(0,-1,1)) ) << "Failed"; + EXPECT_EQ( Directions::template getXYZ<3>(ZpYpXz), (StaticVector<3,int>(0,1,1)) ) << "Failed"; + + EXPECT_EQ( Directions::template getXYZ<3>(ZpYpXm), (StaticVector<3,int>(-1,1,1)) ) << "Failed"; } diff --git a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp index b19b0f13d..09250da03 100644 --- a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp +++ b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp @@ -400,6 +400,7 @@ TEST_F(DistributedGridTest_2D, evaluateAllEntities) check_Inner_2D(rank, *gridPtr, *dof, rank); } + TEST_F(DistributedGridTest_2D, evaluateBoundaryEntities) { //Boundary entities, without overlap @@ -686,7 +687,7 @@ TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithInactiv //Expecting 9 processes setDof_2D(*dof, -rank-1 ); maskDofs.setValue( true ); - if( distributedGrid->getNeighbors()[ Left ] == -1 ) + if( distributedGrid->getNeighbors()[ ZzYzXm ] == -1 ) { for( IndexType i = 0; i < gridPtr->getDimensions().y(); i++ ) { @@ -770,7 +771,7 @@ TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithInActiv //Expecting 9 processes setDof_2D(*dof, -rank-1 ); maskDofs.setValue( true ); - if( distributedGrid->getNeighbors()[ Right ] == -1 ) + if( distributedGrid->getNeighbors()[ ZzYzXp ] == -1 ) { for( IndexType i = 0; i < gridPtr->getDimensions().y(); i++ ) { @@ -854,7 +855,7 @@ TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithInActiv //Expecting 9 processes setDof_2D(*dof, -rank-1 ); maskDofs.setValue( true ); - if( distributedGrid->getNeighbors()[ Up ] == -1 ) + if( distributedGrid->getNeighbors()[ ZzYmXz ] == -1 ) { for( IndexType i = 0; i < gridPtr->getDimensions().x(); i++ ) { @@ -938,7 +939,7 @@ TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithInActiv //Expecting 9 processes setDof_2D(*dof, -rank-1 ); maskDofs.setValue( true ); - if( distributedGrid->getNeighbors()[ Down ] == -1 ) + if( distributedGrid->getNeighbors()[ ZzYpXz ] == -1 ) { for( IndexType i = 0; i < gridPtr->getDimensions().x(); i++ ) { -- GitLab From e21b28af13307a9fb6599596213289733bd734ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Hanousek=20-=20vz?= Date: Wed, 21 Nov 2018 20:34:14 +0100 Subject: [PATCH 2/7] Issue #15 - FIX synchronizer by refactorization - doesnt work, do to bad interpretation of lower and upper overlap. --- .../DistributedMeshes/BufferEntitiesHelper.h | 36 +- .../DistributedGridSynchronizer.h | 4 +- .../DistributedGridSynchronizer_3D.h | 316 ++++++------------ .../DistributedGridTest_1D.cpp | 15 +- .../DistributedGridTest_3D.cpp | 4 +- 5 files changed, 147 insertions(+), 228 deletions(-) diff --git a/src/TNL/Meshes/DistributedMeshes/BufferEntitiesHelper.h b/src/TNL/Meshes/DistributedMeshes/BufferEntitiesHelper.h index 0b3c7b363..599ed9e8e 100644 --- a/src/TNL/Meshes/DistributedMeshes/BufferEntitiesHelper.h +++ b/src/TNL/Meshes/DistributedMeshes/BufferEntitiesHelper.h @@ -13,6 +13,7 @@ #include #include #include +#include namespace TNL { namespace Meshes { @@ -43,10 +44,14 @@ class BufferEntitiesHelper< MeshFunctionType, MaskPointer, 1, RealType, Device, const MaskPointer& maskPointer, RealType* buffer, bool isBoundary, - const Index& beginx, - const Index& sizex, + const Containers::StaticVector<1,Index>& begin, + const Containers::StaticVector<1,Index>& size, bool tobuffer ) { + + Index beginx=begin.x(); + Index sizex=size.x(); + auto mesh = meshFunction.getMesh(); RealType* meshFunctionData = meshFunction.getData().getData(); const typename MaskPointer::ObjectType* mask( nullptr ); @@ -83,12 +88,16 @@ class BufferEntitiesHelper< MeshFunctionType, MaskPointer, 2, RealType, Device, const MaskPointer& maskPointer, RealType* buffer, bool isBoundary, - const Index& beginx, - const Index& beginy, - const Index& sizex, - const Index& sizey, + const Containers::StaticVector<2,Index>& begin, + const Containers::StaticVector<2,Index>& size, bool tobuffer) { + + Index beginx=begin.x(); + Index beginy=begin.y(); + Index sizex=size.x(); + Index sizey=size.y(); + auto mesh=meshFunction.getMesh(); RealType* meshFunctionData = meshFunction.getData().getData(); const typename MaskPointer::ObjectType* mask( nullptr ); @@ -127,15 +136,18 @@ class BufferEntitiesHelper< MeshFunctionType, MaskPointer, 3, RealType, Device, const MaskPointer& maskPointer, RealType* buffer, bool isBoundary, - const Index& beginx, - const Index& beginy, - const Index& beginz, - const Index& sizex, - const Index& sizey, - const Index& sizez, + const Containers::StaticVector<3,Index>& begin, + const Containers::StaticVector<3,Index>& size, bool tobuffer) { + Index beginx=begin.x(); + Index beginy=begin.y(); + Index beginz=begin.z(); + Index sizex=size.x(); + Index sizey=size.y(); + Index sizez=size.z(); + auto mesh=meshFunction.getMesh(); RealType * meshFunctionData=meshFunction.getData().getData(); const typename MaskPointer::ObjectType* mask( nullptr ); diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h index 989f7419a..7171a9655 100644 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h +++ b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h @@ -10,6 +10,6 @@ #pragma once -#include -#include +//#include +//#include #include diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h index 811a12a66..ca7d50097 100644 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h +++ b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h @@ -13,7 +13,7 @@ #include #include #include - +#include namespace TNL { namespace Functions{ @@ -30,17 +30,21 @@ namespace DistributedMeshes { template -class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 3, GridReal, Device, Index >,EntityDimension, RealType>> +class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension, GridReal, Device, Index >,EntityDimension, RealType>> { public: - typedef typename Grid< 3, GridReal, Device, Index >::Cell Cell; + static constexpr int getMeshDimension() { return MeshDimension; }; + static constexpr int getNeighborCount() {return DirectionCount::get();}; + + typedef typename Grid< MeshDimension, GridReal, Device, Index >::Cell Cell; // FIXME: clang does not like this (incomplete type error) // typedef typename Functions::MeshFunction< Grid< 3, GridReal, Device, Index >,EntityDimension, RealType> MeshFunctionType; - typedef typename Grid< 3, GridReal, Device, Index >::DistributedMeshType DistributedGridType; + typedef typename Grid< MeshDimension, GridReal, Device, Index >::DistributedMeshType DistributedGridType; typedef typename DistributedGridType::CoordinatesType CoordinatesType; using SubdomainOverlapsType = typename DistributedGridType::SubdomainOverlapsType; @@ -61,44 +65,43 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 3, GridReal, D this->distributedGrid = distributedGrid; - const SubdomainOverlapsType& lowerOverlap = distributedGrid->getLowerOverlap(); - const SubdomainOverlapsType& upperOverlap = distributedGrid->getUpperOverlap(); - const CoordinatesType& localSize = distributedGrid->getLocalSize(); + const SubdomainOverlapsType& lowerOverlap = this->distributedGrid->getLowerOverlap(); + const SubdomainOverlapsType& upperOverlap = this->distributedGrid->getUpperOverlap(); + const CoordinatesType& localSize = this->distributedGrid->getLocalSize(); const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); - sizes[ ZzYzXm ] = localSize.y() * localSize.z() * lowerOverlap.x(); - sizes[ ZzYzXp ] = localSize.y() * localSize.z() * upperOverlap.x(); - sizes[ ZzYmXz ] = localSize.x() * localSize.z() * lowerOverlap.y(); - sizes[ ZzYpXz ] = localSize.x() * localSize.z() * upperOverlap.y(); - sizes[ ZmYzXz ] = localSize.x() * localSize.y() * lowerOverlap.z(); - sizes[ ZpYzXz ] = localSize.x() * localSize.y() * upperOverlap.z(); - - sizes[ ZzYmXm ] = localSize.z() * lowerOverlap.x() * lowerOverlap.y(); - sizes[ ZzYmXp ] = localSize.z() * upperOverlap.x() * lowerOverlap.y(); - sizes[ ZzYpXm ] = localSize.z() * lowerOverlap.x() * upperOverlap.y(); - sizes[ ZzYpXp ] = localSize.z() * upperOverlap.x() * upperOverlap.y(); - sizes[ ZmYzXp ] = localSize.y() * lowerOverlap.x() * lowerOverlap.z(); - sizes[ ZmYzXm ] = localSize.y() * upperOverlap.x() * lowerOverlap.z(); - sizes[ ZpYzXp ] = localSize.y() * lowerOverlap.x() * upperOverlap.z(); - sizes[ ZpYzXm ] = localSize.y() * upperOverlap.x() * upperOverlap.z(); - sizes[ ZmYmXz ] = localSize.x() * lowerOverlap.y() * lowerOverlap.z(); - sizes[ ZmYpXz ] = localSize.x() * upperOverlap.y() * lowerOverlap.z(); - sizes[ ZpYmXz ] = localSize.x() * lowerOverlap.y() * upperOverlap.z(); - sizes[ ZpYpXz ] = localSize.x() * upperOverlap.y() * upperOverlap.z(); - - sizes[ ZmYmXm ] = lowerOverlap.x() * lowerOverlap.y() * lowerOverlap.z(); - sizes[ ZmYmXp ] = upperOverlap.x() * lowerOverlap.y() * lowerOverlap.z(); - sizes[ ZmYpXm ] = lowerOverlap.x() * upperOverlap.y() * lowerOverlap.z(); - sizes[ ZmYpXp ] = upperOverlap.x() * upperOverlap.y() * lowerOverlap.z(); - sizes[ ZpYmXm ] = lowerOverlap.x() * lowerOverlap.y() * upperOverlap.z(); - sizes[ ZpYmXp ] = upperOverlap.x() * lowerOverlap.y() * upperOverlap.z(); - sizes[ ZpYpXm ] = lowerOverlap.x() * upperOverlap.y() * upperOverlap.z(); - sizes[ ZpYpXp ] = upperOverlap.x() * upperOverlap.y() * upperOverlap.z(); - - for( int i=0; i<26; i++ ) + for( int i=0; igetNeighborCount(); i++ ) { - sendBuffers[ i ].setSize( sizes[ i ] ); - receiveBuffers[ i ].setSize( sizes[ i ] ); + Index sendSize=1; + Index rcvSize=1; + auto directions=Directions::template getXYZgetMeshDimension()>(i); + for(int j=0;jgetMeshDimension();j++) + { + if(directions[j]==-1) + { + sendSize*=upperOverlap[j]; + rcvSize*=lowerOverlap[j]; + } + if(directions[j]==0) + { + sendSize*=localSize[j]; + rcvSize*=localSize[j]; + } + if(directions[j]==1) + { + sendSize*=lowerOverlap[j]; + rcvSize*=upperOverlap[j]; + } + } + + sendSizes[ i ] = sendSize; + recieveSizes[ i ] = rcvSize; + sendBuffers[ i ].setSize( sendSize ); + recieveBuffers[ i ].setSize( rcvSize); + + int world_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + std::cout<< world_rank<<": " << " "<getLowerOverlap(); const SubdomainOverlapsType& upperOverlap = distributedGrid->getUpperOverlap(); - const CoordinatesType& localSize = distributedGrid->getLocalSize(); - const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); - - ZzYzXm_Source = lowerOverlap.x(); - ZzYzXp_Source = localGridSize.x() - 2 * upperOverlap.x(); - ZzYmXz_Source = lowerOverlap.y(); - ZzYpXz_Source = localGridSize.y() - 2 * upperOverlap.y(); - ZmYzXz_Source = lowerOverlap.z(); - ZpYzXz_Source = localGridSize.z() - 2 * upperOverlap.z(); - - xCenter = lowerOverlap.x(); - yCenter = lowerOverlap.y(); - zCenter = lowerOverlap.z(); - - ZzYzXm_Destination = 0; - ZzYzXp_Destination = localGridSize.x() - upperOverlap.x(); - ZzYmXz_Destination = 0; - ZzYpXz_Destination = localGridSize.y() - upperOverlap.y(); - ZmYzXz_Destination = 0; - ZpYzXz_Destination = localGridSize.z() - upperOverlap.z(); + const CoordinatesType& localSize = this->distributedGrid->getLocalSize(); + const CoordinatesType& localBegin = this->distributedGrid->getLocalBegin(); const int *neighbors = distributedGrid->getNeighbors(); const int *periodicNeighbors = distributedGrid->getPeriodicNeighbors(); if( periodicBoundaries ) { - if( neighbors[ ZzYzXm ] == -1 ) - swap( ZzYzXm_Source, ZzYzXm_Destination ); - if( neighbors[ ZzYzXp ] == -1 ) - swap( ZzYzXp_Source, ZzYzXp_Destination ); - if( neighbors[ ZzYpXz ] == -1 ) - swap( ZzYpXz_Source, ZzYpXz_Destination ); - if( neighbors[ ZzYmXz ] == -1 ) - swap( ZzYmXz_Source, ZzYmXz_Destination ); - if( neighbors[ ZmYzXz ] == -1 ) - swap( ZmYzXz_Source, ZmYzXz_Destination ); - if( neighbors[ ZpYzXz ] == -1 ) - swap( ZpYzXz_Source, ZpYzXz_Destination ); + std::cerr<<"TOTALY DEMAGED by refactorization" << std::endl; } //fill send buffers copyBuffers( meshFunction, sendBuffers, true, - ZzYzXm_Source, ZzYzXp_Source, ZzYmXz_Source, ZzYpXz_Source, ZmYzXz_Source, ZpYzXz_Source, - xCenter, yCenter, zCenter, - lowerOverlap, upperOverlap, localSize, + localBegin,localSize, + lowerOverlap, upperOverlap, neighbors, periodicBoundaries, PeriodicBoundariesMaskPointer( nullptr ) ); // the mask is used only when receiving data ); //async send and receive - typename CommunicatorType::Request requests[52]; + typename CommunicatorType::Request requests[2*this->getNeighborCount()]; typename CommunicatorType::CommunicationGroup group; group=*((typename CommunicatorType::CommunicationGroup *)(distributedGrid->getCommunicationGroup())); int requestsCount( 0 ); - //send everything, receive everything - for( int i = 0; i < 26; i++ ) + //send everything, recieve everything + for( int i=0; igetNeighborCount(); i++ ) if( neighbors[ i ] != -1 ) { - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sizes[ i ], neighbors[ i ], 0, group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ i ].getData(), sizes[ i ], neighbors[ i ], 0, group ); + requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sendSizes[ i ], neighbors[ i ],0, group ); + requests[ requestsCount++ ] = CommunicatorType::IRecv( recieveBuffers[ i ].getData(), recieveSizes[ i ], neighbors[ i ], 0, group ); } else if( periodicBoundaries ) { - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sizes[ i ], periodicNeighbors[ i ], 1, group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ i ].getData(), sizes[ i ], periodicNeighbors[ i ], 1, group ); + requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sendSizes[ i ], periodicNeighbors[ i ], 1, group ); + requests[ requestsCount++ ] = CommunicatorType::IRecv( recieveBuffers[ i ].getData(), recieveSizes[ i ], periodicNeighbors[ i ],1, group ); } //wait until send is done CommunicatorType::WaitAll( requests, requestsCount ); + int world_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + for(int i=0;igetNeighborCount();i++) + std::cout<< world_rank<<": " << i << " send:"< @@ -207,138 +183,68 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 3, GridReal, D MeshFunctionType& meshFunction, Containers::Array* buffers, bool toBuffer, - int ZzYzXm, int ZzYzXp, int ZzYmXz, int ZzYpXz, int ZmYzXz, int ZpYzXz, - int xcenter, int ycenter, int zcenter, + const CoordinatesType& localBegin, + const CoordinatesType& localSize, const CoordinatesType& lowerOverlap, const CoordinatesType& upperOverlap, - const CoordinatesType& localSize, const int* neighbor, bool periodicBoundaries, const PeriodicBoundariesMaskPointer& mask ) { - //TODO: refactor to array IsBoundary[number of neighbor] - bool ZzYzXm_IsBoundary = ( neighbor[ ZzYzXm ] == -1 ); - bool ZzYzXp_IsBoundary = ( neighbor[ ZzYzXp ] == -1 ); - bool ZzYmXz_IsBoundary = ( neighbor[ ZzYmXz ] == -1 ); - bool ZzYpXz_IsBoundary = ( neighbor[ ZzYpXz ] == -1 ); - bool ZmYzXz_IsBoundary = ( neighbor[ ZmYzXz ] == -1 ); - bool ZpYzXz_IsBoundary = ( neighbor[ ZpYzXz ] == -1 ); - - bool ZzYmXm_IsBoundary = ( neighbor[ ZzYmXm ] == -1 ); - bool ZzYmXp_IsBoundary = ( neighbor[ ZzYmXp ] == -1 ); - bool ZzYpXm_IsBoundary = ( neighbor[ ZzYpXm ] == -1 ); - bool ZzYpXp_IsBoundary = ( neighbor[ ZzYpXp ] == -1 ); - - bool ZmYzXp_IsBoundary = ( neighbor[ ZmYzXp ] == -1 ); - bool ZmYzXm_IsBoundary = ( neighbor[ ZmYzXm ] == -1 ); - bool ZmYmXz_IsBoundary = ( neighbor[ ZmYmXz ] == -1 ); - bool ZmYpXz_IsBoundary = ( neighbor[ ZmYpXz ] == -1 ); - - bool ZpYzXp_IsBoundary = ( neighbor[ ZpYzXp ] == -1 ); - bool ZpYzXm_IsBoundary = ( neighbor[ ZpYzXm ] == -1 ); - bool ZpYmXz_IsBoundary = ( neighbor[ ZpYmXz ] == -1 ); - bool ZpYpXz_IsBoundary = ( neighbor[ ZpYpXz ] == -1 ); - bool ZmYmXm_IsBoundary = ( neighbor[ ZmYmXm ] == -1 ); - bool ZmYmXp_IsBoundary = ( neighbor[ ZmYmXp ] == -1 ); - bool ZmYpXm_IsBoundary = ( neighbor[ ZmYpXm ] == -1 ); - bool ZmYpXp_IsBoundary = ( neighbor[ ZmYpXp ] == -1 ); + using Helper = BufferEntitiesHelper< MeshFunctionType, PeriodicBoundariesMaskPointer, this->getMeshDimension(), Real_, Device >; + + for(int i=0;igetNeighborCount();i++)//performace isssue - this should be buffered when Synchronizer is created + { + bool isBoundary=( neighbor[ i ] == -1 ); + + CoordinatesType begin=localBegin; + CoordinatesType size=localSize; + auto directions=Directions::template getXYZgetMeshDimension()>(i); + for(int j=0;jgetMeshDimension();j++) + { + if(toBuffer) + { + if(directions[j]==-1) + { + size[j]=upperOverlap[j]; + } + if(directions[j]==1) + { + begin[j]=localBegin[j]+localSize[j]-lowerOverlap[j]; + size[j]=lowerOverlap[j]; + } + } + else + { + if(directions[j]==-1) + { + //tady se asi bude řešit periodic boundary + begin[j]=0; + size[j]=lowerOverlap[j]; + } + if(directions[j]==1) + { + begin[j]=localBegin[j]+localSize[j]; + size[j]=upperOverlap[j]; + } + } + } - bool ZpYmXm_IsBoundary = ( neighbor[ ZpYmXm ] == -1 ); - bool ZpYmXp_IsBoundary = ( neighbor[ ZpYmXp ] == -1 ); - bool ZpYpXm_IsBoundary = ( neighbor[ ZpYpXm ] == -1 ); - bool ZpYpXp_IsBoundary = ( neighbor[ ZpYpXp ] == -1 ); - - using Helper = BufferEntitiesHelper< MeshFunctionType, PeriodicBoundariesMaskPointer, 3, Real_, Device >; - //X-Y-Z - if( ! ZzYzXm_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYzXm ].getData(), ZzYzXm_IsBoundary, ZzYzXm, ycenter, zcenter, lowerOverlap.x(), localSize.y(), localSize.z(), toBuffer ); - if( ! ZzYzXp_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYzXp ].getData(), ZzYzXp_IsBoundary, ZzYzXp, ycenter, zcenter, upperOverlap.x(), localSize.y(), localSize.z(), toBuffer ); - if( ! ZzYmXz_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYmXz ].getData(), ZzYmXz_IsBoundary, xcenter, ZzYmXz, zcenter, localSize.x(), lowerOverlap.y(), localSize.z(), toBuffer ); - if( ! ZzYpXz_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYpXz ].getData(), ZzYpXz_IsBoundary, xcenter, ZzYpXz, zcenter, localSize.x(), upperOverlap.y(), localSize.z(), toBuffer ); - if( ! ZmYzXz_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZmYzXz ].getData(), ZmYzXz_IsBoundary, xcenter, ycenter, ZmYzXz, localSize.x(), localSize.y(), lowerOverlap.z(), toBuffer ); - if( ! ZpYzXz_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZpYzXz ].getData(), ZpYzXz_IsBoundary, xcenter, ycenter, ZpYzXz, localSize.x(), localSize.y(), upperOverlap.z(), toBuffer ); - - //XY - if( ! ZzYmXm_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYmXm ].getData(), ZzYmXm_IsBoundary, ZzYzXm, ZzYmXz, zcenter, lowerOverlap.x(), lowerOverlap.y(), localSize.z(), toBuffer ); - if( ! ZzYmXp_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYmXp ].getData(), ZzYmXp_IsBoundary, ZzYzXp, ZzYmXz, zcenter, upperOverlap.x(), lowerOverlap.y(), localSize.z(), toBuffer ); - if( ! ZzYpXm_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYpXm ].getData(), ZzYpXm_IsBoundary, ZzYzXm, ZzYpXz, zcenter, lowerOverlap.x(), upperOverlap.y(), localSize.z(), toBuffer ); - if( ! ZzYpXp_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYpXp ].getData(), ZzYpXp_IsBoundary, ZzYzXp, ZzYpXz, zcenter, upperOverlap.x(), upperOverlap.y(), localSize.z(), toBuffer ); - - //XZ - if( ! ZmYzXp_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZmYzXp ].getData(), ZmYzXp_IsBoundary, ZzYzXm, ycenter, ZmYzXz, lowerOverlap.x(), localSize.y(), lowerOverlap.z(), toBuffer ); - if( ! ZmYzXm_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZmYzXm ].getData(), ZmYzXm_IsBoundary, ZzYzXp, ycenter, ZmYzXz, upperOverlap.x(), localSize.y(), lowerOverlap.z(), toBuffer ); - if( ! ZpYzXp_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZpYzXp ].getData(), ZpYzXp_IsBoundary, ZzYzXm, ycenter, ZpYzXz, lowerOverlap.x(), localSize.y(), upperOverlap.z(), toBuffer ); - if( ! ZpYzXm_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZpYzXm ].getData(), ZpYzXm_IsBoundary, ZzYzXp, ycenter, ZpYzXz, upperOverlap.x(), localSize.y(), upperOverlap.z(), toBuffer ); - - //YZ - if( ! ZmYmXz_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZmYmXz ].getData(), ZmYmXz_IsBoundary, xcenter, ZzYmXz, ZmYzXz, localSize.x(), lowerOverlap.y(), lowerOverlap.z(), toBuffer ); - if( ! ZmYpXz_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZmYpXz ].getData(), ZmYpXz_IsBoundary, xcenter, ZzYpXz, ZmYzXz, localSize.x(), upperOverlap.y(), lowerOverlap.z(), toBuffer ); - if( ! ZpYmXz_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZpYmXz ].getData(), ZpYmXz_IsBoundary, xcenter, ZzYmXz, ZpYzXz, localSize.x(), lowerOverlap.y(), upperOverlap.z(), toBuffer ); - if( ! ZpYpXz_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZpYpXz ].getData(), ZpYpXz_IsBoundary, xcenter, ZzYpXz, ZpYzXz, localSize.x(), upperOverlap.y(), upperOverlap.z(), toBuffer ); - - //XYZ - if( ! ZmYmXm_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZmYmXm ].getData(), ZmYmXm_IsBoundary, ZzYzXm, ZzYmXz, ZmYzXz, lowerOverlap.x(), lowerOverlap.y(), lowerOverlap.z(), toBuffer ); - if( ! ZmYmXp_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZmYmXp ].getData(), ZmYmXp_IsBoundary, ZzYzXp, ZzYmXz, ZmYzXz, upperOverlap.x(), lowerOverlap.y(), lowerOverlap.z(), toBuffer ); - if( ! ZmYpXm_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZmYpXm ].getData(), ZmYpXm_IsBoundary, ZzYzXm, ZzYpXz, ZmYzXz, lowerOverlap.x(), upperOverlap.y(), lowerOverlap.z(), toBuffer ); - if( ! ZmYpXp_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZmYpXp ].getData(), ZmYpXp_IsBoundary, ZzYzXp, ZzYpXz, ZmYzXz, upperOverlap.x(), upperOverlap.y(), lowerOverlap.z(), toBuffer ); - if( ! ZpYmXm_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZpYmXm ].getData(), ZpYmXm_IsBoundary, ZzYzXm, ZzYmXz, ZpYzXz, lowerOverlap.x(), lowerOverlap.y(), upperOverlap.z(), toBuffer ); - if( ! ZpYmXp_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZpYmXp ].getData(), ZpYmXp_IsBoundary, ZzYzXp, ZzYmXz, ZpYzXz, upperOverlap.x(), lowerOverlap.y(), upperOverlap.z(), toBuffer ); - if( ! ZpYpXm_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZpYpXm ].getData(), ZpYpXp_IsBoundary, ZzYzXm, ZzYpXz, ZpYzXz, lowerOverlap.x(), upperOverlap.y(), upperOverlap.z(), toBuffer ); - if( ! ZpYpXp_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZpYpXp ].getData(), ZpYpXp_IsBoundary, ZzYzXp, ZzYpXz, ZpYzXz, upperOverlap.x(), upperOverlap.y(), upperOverlap.z(), toBuffer ); + if( ! isBoundary || periodicBoundaries ) + Helper::BufferEntities( meshFunction, mask, buffers[ i ].getData(), isBoundary, begin, size, toBuffer ); + + } } private: - Containers::Array sendBuffers[26]; - Containers::Array receiveBuffers[26]; - Containers::StaticArray< 26, int > sizes; + Containers::Array sendBuffers[DirectionCount::get()]; + Containers::Array recieveBuffers[DirectionCount::get()]; + Containers::StaticArray< DirectionCount::get(), int > sendSizes; + Containers::StaticArray< DirectionCount::get(), int > recieveSizes; DistributedGridType *distributedGrid; - - int ZzYzXm_Source; - int ZzYzXp_Source; - int ZzYmXz_Source; - int ZzYpXz_Source; - int ZmYzXz_Source; - int ZpYzXz_Source; - int xCenter; - int yCenter; - int zCenter; - int ZzYzXm_Destination; - int ZzYzXp_Destination; - int ZzYmXz_Destination; - int ZzYpXz_Destination; - int ZmYzXz_Destination; - int ZpYzXz_Destination; - - CoordinatesType overlap; - CoordinatesType localSize; bool isSet; diff --git a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp index 251b9f553..927d2fe91 100644 --- a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp +++ b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp @@ -169,7 +169,7 @@ class DistributedGridTest_1D : public ::testing::Test delete distributedGrid; } }; - +/* TEST_F( DistributedGridTest_1D, isBoundaryDomainTest ) { if( rank == 0 || rank == nproc - 1 ) @@ -209,20 +209,21 @@ TEST_F(DistributedGridTest_1D, evaluateInteriorEntities) check_Overlap_1D(rank, nproc, dof, -1); check_Inner_1D(rank, nproc, dof, rank); } - +*/ TEST_F(DistributedGridTest_1D, SynchronizerNeighborsTest ) { setDof_1D(dof,-1); constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr , constFunctionPtr ); meshFunctionPtr->template synchronize(); + std::cout << rank << dof << std::endl; + if(rank!=0) EXPECT_EQ((dof)[0],rank-1)<< "Left Overlap was filled by wrong process."; if(rank!=nproc-1) EXPECT_EQ((dof)[dof.getSize()-1],rank+1)<< "Right Overlap was filled by wrong process."; } - - +/* TEST_F(DistributedGridTest_1D, EvaluateLinearFunction ) { //fill mesh function with linear function (physical center of cell corresponds with its coordinates in grid) @@ -237,9 +238,9 @@ TEST_F(DistributedGridTest_1D, EvaluateLinearFunction ) auto entity2= gridptr->template getEntity< Cell >((dof).getSize()-1); entity2.refresh(); EXPECT_EQ(meshFunctionPtr->getValue(entity), (*linearFunctionPtr)(entity)) << "Linear function Overlap error on right Edge."; -} - +}*/ +/* TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithoutMask ) { // Setup periodic boundaries @@ -374,7 +375,7 @@ TEST_F(DistributedGridTest_1D, SynchronizePeriodicBoundariesLinearTest ) if( rank == nproc - 1 ) EXPECT_EQ( meshFunctionPtr->getValue(entity2), -1 ) << "Linear function Overlap error on right Edge."; } - +*/ #else diff --git a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_3D.cpp b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_3D.cpp index 6bbd7ad25..574d14a74 100644 --- a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_3D.cpp +++ b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_3D.cpp @@ -663,7 +663,7 @@ class DistributedGirdTest_3D : public ::testing::Test delete distributedGrid; } }; - +/* TEST_F(DistributedGirdTest_3D, evaluateAllEntities) { @@ -695,7 +695,7 @@ TEST_F(DistributedGirdTest_3D, evaluateInteriorEntities) check_Boundary_3D(rank, *gridptr, *dof, -1); check_Overlap_3D(rank, *gridptr, *dof, -1); check_Inner_3D(rank, *gridptr, *dof, rank); -} +} */ TEST_F(DistributedGirdTest_3D, LinearFunctionTest) { -- GitLab From 1f622ebbd52bb6e48eec06088bde1030c91c7121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Hanousek=20-=20vz?= Date: Wed, 21 Nov 2018 22:35:39 +0100 Subject: [PATCH 3/7] Issue #15 - working synchronizer - without PeriodicBC implementation - SubdomainOverlapGetter has changed interface -> need more updates, only basic grid tests work --- .../DistributedMeshes/DistributedGrid.h | 12 +- .../DistributedMeshes/DistributedGrid.hpp | 23 +++- .../DistributedGridSynchronizer_3D.h | 128 +++++++----------- .../SubdomainOverlapsGetter.h | 2 + .../SubdomainOverlapsGetter.hpp | 5 + .../DistributedGridTest_1D.cpp | 22 ++- .../DistributedGridTest_2D.cpp | 11 +- .../DistributedGridTest_3D.cpp | 11 +- 8 files changed, 110 insertions(+), 104 deletions(-) diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid.h b/src/TNL/Meshes/DistributedMeshes/DistributedGrid.h index 35297485b..4234e20cd 100644 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGrid.h +++ b/src/TNL/Meshes/DistributedMeshes/DistributedGrid.h @@ -62,7 +62,9 @@ class DistributedMesh< Grid< Dimension, Real, Device, Index > > const GridType& getGlobalGrid() const; void setOverlaps( const SubdomainOverlapsType& lower, - const SubdomainOverlapsType& upper ); + const SubdomainOverlapsType& upper, + const SubdomainOverlapsType& globalLower, + const SubdomainOverlapsType& globalUpper ); void setupGrid( GridType& grid); @@ -74,10 +76,16 @@ class DistributedMesh< Grid< Dimension, Real, Device, Index > > // It is still being used in cuts set-up const CoordinatesType& getOverlap() const { return this->overlap;}; + //currently used overlaps at this subdomain const SubdomainOverlapsType& getLowerOverlap() const; const SubdomainOverlapsType& getUpperOverlap() const; + //original overlaps set by user - same values return all subdomains.. + const SubdomainOverlapsType& getGlobalLowerOverlap() const; + + const SubdomainOverlapsType& getGlobalUpperOverlap() const; + //number of elements of local sub domain WITHOUT overlap // TODO: getSubdomainDimensions const CoordinatesType& getLocalSize() const; @@ -147,7 +155,7 @@ class DistributedMesh< Grid< Dimension, Real, Device, Index > > CoordinatesType globalBegin; PointType spaceSteps; - SubdomainOverlapsType lowerOverlap, upperOverlap; + SubdomainOverlapsType lowerOverlap, upperOverlap, globalLowerOverlap, globalUpperOverlap; CoordinatesType domainDecomposition; CoordinatesType subdomainCoordinates; diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid.hpp b/src/TNL/Meshes/DistributedMeshes/DistributedGrid.hpp index 1fbbe12aa..fd506c648 100644 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGrid.hpp +++ b/src/TNL/Meshes/DistributedMeshes/DistributedGrid.hpp @@ -170,8 +170,13 @@ template< int Dimension, typename Real, typename Device, typename Index > void DistributedMesh< Grid< Dimension, Real, Device, Index > >:: setOverlaps( const SubdomainOverlapsType& lower, - const SubdomainOverlapsType& upper ) + const SubdomainOverlapsType& upper, + const SubdomainOverlapsType& globalLower, + const SubdomainOverlapsType& globalUpper ) { + this->globalLowerOverlap = globalLower; + this->globalUpperOverlap = globalUpper; + this->lowerOverlap = lower; this->upperOverlap = upper; @@ -258,6 +263,22 @@ getUpperOverlap() const return this->upperOverlap; }; +template< int Dimension, typename Real, typename Device, typename Index > +const typename DistributedMesh< Grid< Dimension, Real, Device, Index > >::CoordinatesType& +DistributedMesh< Grid< Dimension, Real, Device, Index > >:: +getGlobalLowerOverlap() const +{ + return this->globalLowerOverlap; +}; + +template< int Dimension, typename Real, typename Device, typename Index > +const typename DistributedMesh< Grid< Dimension, Real, Device, Index > >::CoordinatesType& +DistributedMesh< Grid< Dimension, Real, Device, Index > >:: +getGlobalUpperOverlap() const +{ + return this->globalUpperOverlap; +}; + template< int Dimension, typename Real, typename Device, typename Index > const typename DistributedMesh< Grid< Dimension, Real, Device, Index > >::CoordinatesType& DistributedMesh< Grid< Dimension, Real, Device, Index > >:: diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h index ca7d50097..80f35b5a7 100644 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h +++ b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h @@ -65,8 +65,10 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension, this->distributedGrid = distributedGrid; - const SubdomainOverlapsType& lowerOverlap = this->distributedGrid->getLowerOverlap(); - const SubdomainOverlapsType& upperOverlap = this->distributedGrid->getUpperOverlap(); + const SubdomainOverlapsType& globalLowerOverlap = this->distributedGrid->getGlobalLowerOverlap(); + const SubdomainOverlapsType& globalUpperOverlap = this->distributedGrid->getGlobalUpperOverlap(); + // const SubdomainOverlapsType& globalPeriodicBCOverlap = this->distributedGrid->getGlobalPeriodicBCOverlap(); + const CoordinatesType& localBegin = this->distributedGrid->getLocalBegin(); const CoordinatesType& localSize = this->distributedGrid->getLocalSize(); const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); @@ -74,34 +76,42 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension, { Index sendSize=1; Index rcvSize=1; + + //bool isBoundary=( neighbor[ i ] == -1 ); auto directions=Directions::template getXYZgetMeshDimension()>(i); + + sendDimensions[i]=localSize; + recieveDimensions[i]=localSize; + sendBegin[i]=localBegin; + recieveBegin[i]=localBegin; + for(int j=0;jgetMeshDimension();j++) { if(directions[j]==-1) { - sendSize*=upperOverlap[j]; - rcvSize*=lowerOverlap[j]; - } - if(directions[j]==0) - { - sendSize*=localSize[j]; - rcvSize*=localSize[j]; + //TODO:periodicBC + sendDimensions[i][j]=globalUpperOverlap[j]; + recieveDimensions[i][j]=globalLowerOverlap[j]; + recieveBegin[i][j]=0; } + if(directions[j]==1) { - sendSize*=lowerOverlap[j]; - rcvSize*=upperOverlap[j]; + //TODO:periodicBC + sendDimensions[i][j]=globalLowerOverlap[j]; + recieveDimensions[i][j]=globalUpperOverlap[j]; + sendBegin[i][j]=localBegin[j]+localSize[j]-globalLowerOverlap[j]; + recieveBegin[i][j]=localBegin[j]+localSize[j]; } + + sendSize*=sendDimensions[i][j]; + rcvSize*=recieveDimensions[i][j]; } sendSizes[ i ] = sendSize; recieveSizes[ i ] = rcvSize; sendBuffers[ i ].setSize( sendSize ); recieveBuffers[ i ].setSize( rcvSize); - - int world_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); - std::cout<< world_rank<<": " << " "<isDistributed() ) return; - const SubdomainOverlapsType& lowerOverlap = distributedGrid->getLowerOverlap(); - const SubdomainOverlapsType& upperOverlap = distributedGrid->getUpperOverlap(); - const CoordinatesType& localSize = this->distributedGrid->getLocalSize(); - const CoordinatesType& localBegin = this->distributedGrid->getLocalBegin(); - const int *neighbors = distributedGrid->getNeighbors(); const int *periodicNeighbors = distributedGrid->getPeriodicNeighbors(); @@ -132,9 +137,9 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension, } //fill send buffers - copyBuffers( meshFunction, sendBuffers, true, - localBegin,localSize, - lowerOverlap, upperOverlap, + copyBuffers( meshFunction, + sendBuffers, sendBegin,sendDimensions, + true, neighbors, periodicBoundaries, PeriodicBoundariesMaskPointer( nullptr ) ); // the mask is used only when receiving data ); @@ -161,15 +166,10 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension, //wait until send is done CommunicatorType::WaitAll( requests, requestsCount ); - int world_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); - for(int i=0;igetNeighborCount();i++) - std::cout<< world_rank<<": " << i << " send:"<* buffers, + CoordinatesType* begins, + CoordinatesType* sizes, bool toBuffer, - const CoordinatesType& localBegin, - const CoordinatesType& localSize, - const CoordinatesType& lowerOverlap, - const CoordinatesType& upperOverlap, const int* neighbor, bool periodicBoundaries, const PeriodicBoundariesMaskPointer& mask ) { - using Helper = BufferEntitiesHelper< MeshFunctionType, PeriodicBoundariesMaskPointer, this->getMeshDimension(), Real_, Device >; - for(int i=0;igetNeighborCount();i++)//performace isssue - this should be buffered when Synchronizer is created + for(int i=0;igetNeighborCount();i++) { - bool isBoundary=( neighbor[ i ] == -1 ); - - CoordinatesType begin=localBegin; - CoordinatesType size=localSize; - auto directions=Directions::template getXYZgetMeshDimension()>(i); - for(int j=0;jgetMeshDimension();j++) - { - if(toBuffer) - { - if(directions[j]==-1) - { - size[j]=upperOverlap[j]; - } - if(directions[j]==1) - { - begin[j]=localBegin[j]+localSize[j]-lowerOverlap[j]; - size[j]=lowerOverlap[j]; - } - } - else - { - if(directions[j]==-1) - { - //tady se asi bude řešit periodic boundary - begin[j]=0; - size[j]=lowerOverlap[j]; - } - if(directions[j]==1) - { - begin[j]=localBegin[j]+localSize[j]; - size[j]=upperOverlap[j]; - } - } - } - + bool isBoundary=( neighbor[ i ] == -1 ); if( ! isBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ i ].getData(), isBoundary, begin, size, toBuffer ); - - } + { + Helper::BufferEntities( meshFunction, mask, buffers[ i ].getData(), isBoundary, begins[i], sizes[i], toBuffer ); + } + } } private: - Containers::Array sendBuffers[DirectionCount::get()]; - Containers::Array recieveBuffers[DirectionCount::get()]; - Containers::StaticArray< DirectionCount::get(), int > sendSizes; - Containers::StaticArray< DirectionCount::get(), int > recieveSizes; + Containers::Array sendBuffers[getNeighborCount()]; + Containers::Array recieveBuffers[getNeighborCount()]; + Containers::StaticArray< getNeighborCount(), int > sendSizes; + Containers::StaticArray< getNeighborCount(), int > recieveSizes; + + + CoordinatesType sendDimensions[getNeighborCount()]; + CoordinatesType recieveDimensions[getNeighborCount()]; + CoordinatesType sendBegin[getNeighborCount()]; + CoordinatesType recieveBegin[getNeighborCount()]; DistributedGridType *distributedGrid; diff --git a/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.h b/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.h index 774f5c24e..d5e7edd2a 100644 --- a/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.h +++ b/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.h @@ -73,6 +73,8 @@ class SubdomainOverlapsGetter< Grid< Dimension, Real, Device, Index >, Communica static void getOverlaps( const DistributedMeshType* distributedMesh, SubdomainOverlapsType& lower, SubdomainOverlapsType& upper, + SubdomainOverlapsType& globalLower, + SubdomainOverlapsType& globalUpper, IndexType subdomainOverlapSize, const SubdomainOverlapsType& periodicBoundariesOverlapSize = 0 ); diff --git a/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.hpp b/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.hpp index abf6243ab..5ff558d40 100644 --- a/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.hpp +++ b/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.hpp @@ -27,6 +27,8 @@ SubdomainOverlapsGetter< Grid< Dimension, Real, Device, Index >, Communicator >: getOverlaps( const DistributedMeshType* distributedMesh, SubdomainOverlapsType& lower, SubdomainOverlapsType& upper, + SubdomainOverlapsType& globalLower, + SubdomainOverlapsType& globalUpper, IndexType subdomainOverlapSize, const SubdomainOverlapsType& periodicBoundariesOverlapSize ) { @@ -38,6 +40,9 @@ getOverlaps( const DistributedMeshType* distributedMesh, for( int i = 0; i < Dimension; i++ ) { + globalLower[i]=subdomainOverlapSize; + globalUpper[i]=subdomainOverlapSize; + if( subdomainCoordinates[ i ] > 0 ) lower[ i ] = subdomainOverlapSize; else diff --git a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp index 927d2fe91..2e5cdb4b2 100644 --- a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp +++ b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp @@ -140,12 +140,12 @@ class DistributedGridTest_1D : public ::testing::Test overlap.setValue(1); distributedGrid=new DistributedGridType(); - typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap; + typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap; distributedGrid->template setGlobalGrid( globalGrid ); //distributedGrid->setupGrid(*gridptr); SubdomainOverlapsGetter< GridType, CommunicatorType >:: - getOverlaps( distributedGrid, lowerOverlap, upperOverlap, 1 ); - distributedGrid->setOverlaps( lowerOverlap, upperOverlap ); + getOverlaps( distributedGrid, lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap, 1 ); + distributedGrid->setOverlaps( lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap ); distributedGrid->setupGrid(*gridptr); dof.setSize( gridptr->template getEntitiesCount< Cell >() ); @@ -157,10 +157,10 @@ class DistributedGridTest_1D : public ::testing::Test void SetUpPeriodicBoundaries() { - typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap; + typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap; SubdomainOverlapsGetter< GridType, CommunicatorType >:: - getOverlaps( distributedGrid, lowerOverlap, upperOverlap, 1 ); - distributedGrid->setOverlaps( lowerOverlap, upperOverlap ); + getOverlaps( distributedGrid, lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap, 1 ); + distributedGrid->setOverlaps( lowerOverlap, upperOverlap, globalLowerOverlap, globalUpperOverlap ); distributedGrid->setupGrid(*gridptr); } @@ -169,7 +169,7 @@ class DistributedGridTest_1D : public ::testing::Test delete distributedGrid; } }; -/* + TEST_F( DistributedGridTest_1D, isBoundaryDomainTest ) { if( rank == 0 || rank == nproc - 1 ) @@ -209,21 +209,19 @@ TEST_F(DistributedGridTest_1D, evaluateInteriorEntities) check_Overlap_1D(rank, nproc, dof, -1); check_Inner_1D(rank, nproc, dof, rank); } -*/ + TEST_F(DistributedGridTest_1D, SynchronizerNeighborsTest ) { setDof_1D(dof,-1); constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr , constFunctionPtr ); meshFunctionPtr->template synchronize(); - std::cout << rank << dof << std::endl; - if(rank!=0) EXPECT_EQ((dof)[0],rank-1)<< "Left Overlap was filled by wrong process."; if(rank!=nproc-1) EXPECT_EQ((dof)[dof.getSize()-1],rank+1)<< "Right Overlap was filled by wrong process."; } -/* + TEST_F(DistributedGridTest_1D, EvaluateLinearFunction ) { //fill mesh function with linear function (physical center of cell corresponds with its coordinates in grid) @@ -238,7 +236,7 @@ TEST_F(DistributedGridTest_1D, EvaluateLinearFunction ) auto entity2= gridptr->template getEntity< Cell >((dof).getSize()-1); entity2.refresh(); EXPECT_EQ(meshFunctionPtr->getValue(entity), (*linearFunctionPtr)(entity)) << "Linear function Overlap error on right Edge."; -}*/ +} /* TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithoutMask ) diff --git a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp index 09250da03..3f6c67c2b 100644 --- a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp +++ b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp @@ -369,9 +369,10 @@ class DistributedGridTest_2D : public ::testing::Test distributedGrid=new DistributedGridType(); distributedGrid->setDomainDecomposition( typename DistributedGridType::CoordinatesType( 3, 3 ) ); distributedGrid->template setGlobalGrid( globalGrid ); - typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap; - SubdomainOverlapsGetter< GridType, CommunicatorType >::getOverlaps( distributedGrid, lowerOverlap, upperOverlap, 1 ); - distributedGrid->setOverlaps( lowerOverlap, upperOverlap ); + typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap; + SubdomainOverlapsGetter< GridType, CommunicatorType >:: + getOverlaps( distributedGrid, lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap, 1 ); + distributedGrid->setOverlaps( lowerOverlap, upperOverlap, globalLowerOverlap, globalUpperOverlap ); distributedGrid->setupGrid(*gridPtr); dof=new DofType(gridPtr->template getEntitiesCount< Cell >()); @@ -524,7 +525,7 @@ TEST_F(DistributedGridTest_2D, SynchronizerNeighborTest ) } } -TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithoutMask ) +/*TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithoutMask ) { // Setup periodic boundaries // TODO: I do not know how to do it better with GTEST - additional setup @@ -1003,7 +1004,7 @@ TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithInActiv checkDownBoundary( *gridPtr, *dof, true, false, 8 ); checkRightBoundary( *gridPtr, *dof, true, false, -7 ); } -} +}*/ #else TEST(NoMPI, NoTest) diff --git a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_3D.cpp b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_3D.cpp index 574d14a74..dab290247 100644 --- a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_3D.cpp +++ b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_3D.cpp @@ -642,13 +642,14 @@ class DistributedGirdTest_3D : public ::testing::Test globalGrid.setDimensions(size,size,size); globalGrid.setDomain(globalOrigin,globalProportions); - typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap; distributedGrid=new DistributedGridType(); distributedGrid->setDomainDecomposition( typename DistributedGridType::CoordinatesType( 3, 3, 3 ) ); distributedGrid->template setGlobalGrid( globalGrid ); distributedGrid->setupGrid(*gridptr); - SubdomainOverlapsGetter< GridType, CommunicatorType >::getOverlaps( distributedGrid, lowerOverlap, upperOverlap, 1 ); - distributedGrid->setOverlaps( lowerOverlap, upperOverlap ); + typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap; + SubdomainOverlapsGetter< GridType, CommunicatorType >:: + getOverlaps( distributedGrid, lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap, 1 ); + distributedGrid->setOverlaps( lowerOverlap, upperOverlap, globalLowerOverlap, globalUpperOverlap ); distributedGrid->setupGrid(*gridptr); dof=new DofType(gridptr->template getEntitiesCount< Cell >()); @@ -663,7 +664,7 @@ class DistributedGirdTest_3D : public ::testing::Test delete distributedGrid; } }; -/* + TEST_F(DistributedGirdTest_3D, evaluateAllEntities) { @@ -695,7 +696,7 @@ TEST_F(DistributedGirdTest_3D, evaluateInteriorEntities) check_Boundary_3D(rank, *gridptr, *dof, -1); check_Overlap_3D(rank, *gridptr, *dof, -1); check_Inner_3D(rank, *gridptr, *dof, rank); -} */ +} TEST_F(DistributedGirdTest_3D, LinearFunctionTest) { -- GitLab From e1326cacb75d775609384e2df3997226d9f648f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Hanousek=20-=20vz?= Date: Thu, 22 Nov 2018 10:44:10 +0100 Subject: [PATCH 4/7] Issue #15 - distributed grid modified to expect upper and lower overlap have same sizes at standard communication. -distributedGridSynchronizer was moved. --- .../Meshes/DistributedMeshes/CMakeLists.txt | 3 - .../DistributedMeshes/DistributedGrid.h | 9 +- .../DistributedMeshes/DistributedGrid.hpp | 23 +- .../DistributedGridSynchronizer.h | 210 ++++++++++++++- .../DistributedGridSynchronizer_1D.h | 206 --------------- .../DistributedGridSynchronizer_2D.h | 246 ------------------ .../DistributedGridSynchronizer_3D.h | 227 ---------------- .../SubdomainOverlapsGetter.h | 2 - .../SubdomainOverlapsGetter.hpp | 4 - .../DistributedGridTest_1D.cpp | 22 +- .../DistributedGridTest_2D.cpp | 6 +- .../DistributedGridTest_3D.cpp | 6 +- 12 files changed, 228 insertions(+), 736 deletions(-) delete mode 100644 src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_1D.h delete mode 100644 src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_2D.h delete mode 100644 src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h diff --git a/src/TNL/Meshes/DistributedMeshes/CMakeLists.txt b/src/TNL/Meshes/DistributedMeshes/CMakeLists.txt index 7785de6c4..89a64d74c 100644 --- a/src/TNL/Meshes/DistributedMeshes/CMakeLists.txt +++ b/src/TNL/Meshes/DistributedMeshes/CMakeLists.txt @@ -5,9 +5,6 @@ SET( headers BufferEntitiesHelper.h DistributedGrid.h DistributedGrid.hpp DistributedGridSynchronizer.h - DistributedGridSynchronizer_1D.h - DistributedGridSynchronizer_2D.h - DistributedGridSynchronizer_3D.h DistributedGridIO.h DistributedGridIO_MeshFunction.h DistributedGridIO_VectorField.h diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid.h b/src/TNL/Meshes/DistributedMeshes/DistributedGrid.h index 4234e20cd..6a4fdf7df 100644 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGrid.h +++ b/src/TNL/Meshes/DistributedMeshes/DistributedGrid.h @@ -62,9 +62,7 @@ class DistributedMesh< Grid< Dimension, Real, Device, Index > > const GridType& getGlobalGrid() const; void setOverlaps( const SubdomainOverlapsType& lower, - const SubdomainOverlapsType& upper, - const SubdomainOverlapsType& globalLower, - const SubdomainOverlapsType& globalUpper ); + const SubdomainOverlapsType& upper); void setupGrid( GridType& grid); @@ -81,11 +79,6 @@ class DistributedMesh< Grid< Dimension, Real, Device, Index > > const SubdomainOverlapsType& getUpperOverlap() const; - //original overlaps set by user - same values return all subdomains.. - const SubdomainOverlapsType& getGlobalLowerOverlap() const; - - const SubdomainOverlapsType& getGlobalUpperOverlap() const; - //number of elements of local sub domain WITHOUT overlap // TODO: getSubdomainDimensions const CoordinatesType& getLocalSize() const; diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGrid.hpp b/src/TNL/Meshes/DistributedMeshes/DistributedGrid.hpp index fd506c648..1f68fe289 100644 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGrid.hpp +++ b/src/TNL/Meshes/DistributedMeshes/DistributedGrid.hpp @@ -170,13 +170,8 @@ template< int Dimension, typename Real, typename Device, typename Index > void DistributedMesh< Grid< Dimension, Real, Device, Index > >:: setOverlaps( const SubdomainOverlapsType& lower, - const SubdomainOverlapsType& upper, - const SubdomainOverlapsType& globalLower, - const SubdomainOverlapsType& globalUpper ) + const SubdomainOverlapsType& upper) { - this->globalLowerOverlap = globalLower; - this->globalUpperOverlap = globalUpper; - this->lowerOverlap = lower; this->upperOverlap = upper; @@ -263,22 +258,6 @@ getUpperOverlap() const return this->upperOverlap; }; -template< int Dimension, typename Real, typename Device, typename Index > -const typename DistributedMesh< Grid< Dimension, Real, Device, Index > >::CoordinatesType& -DistributedMesh< Grid< Dimension, Real, Device, Index > >:: -getGlobalLowerOverlap() const -{ - return this->globalLowerOverlap; -}; - -template< int Dimension, typename Real, typename Device, typename Index > -const typename DistributedMesh< Grid< Dimension, Real, Device, Index > >::CoordinatesType& -DistributedMesh< Grid< Dimension, Real, Device, Index > >:: -getGlobalUpperOverlap() const -{ - return this->globalUpperOverlap; -}; - template< int Dimension, typename Real, typename Device, typename Index > const typename DistributedMesh< Grid< Dimension, Real, Device, Index > >::CoordinatesType& DistributedMesh< Grid< Dimension, Real, Device, Index > >:: diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h index 7171a9655..e07eec29b 100644 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h +++ b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h @@ -10,6 +10,210 @@ #pragma once -//#include -//#include -#include +#include +#include +#include +#include + +namespace TNL { +namespace Functions{ +template< typename Mesh, + int MeshEntityDimension, + typename Real > +class MeshFunction; +}//Functions +}//TNL + +namespace TNL { +namespace Meshes { +namespace DistributedMeshes { + +template +class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension, GridReal, Device, Index >,EntityDimension, RealType>> +{ + + public: + static constexpr int getMeshDimension() { return MeshDimension; }; + static constexpr int getNeighborCount() {return DirectionCount::get();}; + + typedef typename Grid< MeshDimension, GridReal, Device, Index >::Cell Cell; + // FIXME: clang does not like this (incomplete type error) +// typedef typename Functions::MeshFunction< Grid< 3, GridReal, Device, Index >,EntityDimension, RealType> MeshFunctionType; + typedef typename Grid< MeshDimension, GridReal, Device, Index >::DistributedMeshType DistributedGridType; + typedef typename DistributedGridType::CoordinatesType CoordinatesType; + using SubdomainOverlapsType = typename DistributedGridType::SubdomainOverlapsType; + + DistributedMeshSynchronizer() + { + isSet = false; + }; + + DistributedMeshSynchronizer( DistributedGridType *distributedGrid ) + { + isSet = false; + setDistributedGrid( distributedGrid ); + }; + + void setDistributedGrid( DistributedGridType *distributedGrid ) + { + isSet = true; + + this->distributedGrid = distributedGrid; + + const SubdomainOverlapsType& lowerOverlap = this->distributedGrid->getLowerOverlap(); + const SubdomainOverlapsType& upperOverlap = this->distributedGrid->getUpperOverlap(); + + const CoordinatesType& localBegin = this->distributedGrid->getLocalBegin(); + const CoordinatesType& localSize = this->distributedGrid->getLocalSize(); + const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); + + for( int i=0; igetNeighborCount(); i++ ) + { + Index sendSize=1;//sended and recieve areas has same size + + //bool isBoundary=( neighbor[ i ] == -1 ); + auto directions=Directions::template getXYZgetMeshDimension()>(i); + + sendDimensions[i]=localSize;//sended and recieve areas has same dimensions + sendBegin[i]=localBegin; + recieveBegin[i]=localBegin; + + for(int j=0;jgetMeshDimension();j++) + { + if(directions[j]==-1) + { + sendDimensions[i][j]=lowerOverlap[j]; + recieveBegin[i][j]=0; + } + + if(directions[j]==1) + { + sendDimensions[i][j]=upperOverlap[j]; + sendBegin[i][j]=localBegin[j]+localSize[j]-upperOverlap[j]; + recieveBegin[i][j]=localBegin[j]+localSize[j]; + } + + sendSize*=sendDimensions[i][j]; + } + + sendSizes[ i ] = sendSize; + sendBuffers[ i ].setSize( sendSize ); + recieveBuffers[ i ].setSize( sendSize); + } + + } + + template< typename CommunicatorType, + typename MeshFunctionType, + typename PeriodicBoundariesMaskPointer = Pointers::SharedPointer< MeshFunctionType > > + void synchronize( MeshFunctionType &meshFunction, + bool periodicBoundaries = false, + const PeriodicBoundariesMaskPointer& mask = PeriodicBoundariesMaskPointer( nullptr ) ) + { + + TNL_ASSERT_TRUE( isSet, "Synchronizer is not set, but used to synchronize" ); + + if( !distributedGrid->isDistributed() ) return; + + const int *neighbors = distributedGrid->getNeighbors(); + const int *periodicNeighbors = distributedGrid->getPeriodicNeighbors(); + + if( periodicBoundaries ) + { + std::cerr<<"TOTALY DEMAGED by refactorization" << std::endl; + } + + //fill send buffers + copyBuffers( meshFunction, + sendBuffers, sendBegin,sendDimensions, + true, + neighbors, + periodicBoundaries, + PeriodicBoundariesMaskPointer( nullptr ) ); // the mask is used only when receiving data ); + + //async send and receive + typename CommunicatorType::Request requests[2*this->getNeighborCount()]; + typename CommunicatorType::CommunicationGroup group; + group=*((typename CommunicatorType::CommunicationGroup *)(distributedGrid->getCommunicationGroup())); + int requestsCount( 0 ); + + //send everything, recieve everything + for( int i=0; igetNeighborCount(); i++ ) + if( neighbors[ i ] != -1 ) + { + requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sendSizes[ i ], neighbors[ i ], group ); + requests[ requestsCount++ ] = CommunicatorType::IRecv( recieveBuffers[ i ].getData(), sendSizes[ i ], neighbors[ i ], group ); + } + else if( periodicBoundaries && sendSizes[ i ] !=0 ) + { + requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sendSizes[ i ], periodicNeighbors[ i ], group ); + requests[ requestsCount++ ] = CommunicatorType::IRecv( recieveBuffers[ i ].getData(), sendSizes[ i ], periodicNeighbors[ i ], group ); + } + + //wait until send is done + CommunicatorType::WaitAll( requests, requestsCount ); + + //copy data from receive buffers + copyBuffers(meshFunction, + recieveBuffers,recieveBegin,sendDimensions , + false, + neighbors, + periodicBoundaries, + mask ); + } + + private: + template< typename Real_, + typename MeshFunctionType, + typename PeriodicBoundariesMaskPointer > + void copyBuffers( + MeshFunctionType& meshFunction, + Containers::Array* buffers, + CoordinatesType* begins, + CoordinatesType* sizes, + bool toBuffer, + const int* neighbor, + bool periodicBoundaries, + const PeriodicBoundariesMaskPointer& mask ) + { + using Helper = BufferEntitiesHelper< MeshFunctionType, PeriodicBoundariesMaskPointer, this->getMeshDimension(), Real_, Device >; + + for(int i=0;igetNeighborCount();i++) + { + bool isBoundary=( neighbor[ i ] == -1 ); + if( ! isBoundary || periodicBoundaries ) + { + Helper::BufferEntities( meshFunction, mask, buffers[ i ].getData(), isBoundary, begins[i], sizes[i], toBuffer ); + } + } + } + + private: + + Containers::Array sendBuffers[getNeighborCount()]; + Containers::Array recieveBuffers[getNeighborCount()]; + Containers::StaticArray< getNeighborCount(), int > sendSizes; + Containers::StaticArray< getNeighborCount(), int > recieveSizes; + + + CoordinatesType sendDimensions[getNeighborCount()]; + CoordinatesType recieveDimensions[getNeighborCount()]; + CoordinatesType sendBegin[getNeighborCount()]; + CoordinatesType recieveBegin[getNeighborCount()]; + + DistributedGridType *distributedGrid; + + bool isSet; + +}; + + +} // namespace DistributedMeshes +} // namespace Meshes +} // namespace TNL + diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_1D.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_1D.h deleted file mode 100644 index ffc700556..000000000 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_1D.h +++ /dev/null @@ -1,206 +0,0 @@ -/*************************************************************************** - DistributedGridSynchronizer_1D.h - description - ------------------- - begin : Aug 15, 2018 - copyright : (C) 2018 by Tomas Oberhuber - email : tomas.oberhuber@fjfi.cvut.cz - ***************************************************************************/ - -/* See Copyright Notice in tnl/Copyright */ - -#pragma once - -#include -#include -#include - - -namespace TNL { -namespace Functions{ -template< typename Mesh, - int MeshEntityDimension, - typename Real > -class MeshFunction; -}//Functions -}//TNL - -namespace TNL { -namespace Meshes { -namespace DistributedMeshes { - -template -class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 1, GridReal, Device, Index >, EntityDimension, Real > > -{ - - public: - using RealType = Real; - typedef typename Grid< 1, GridReal, Device, Index >::Cell Cell; - // FIXME: clang does not like this (incomplete type error) -// typedef typename Functions::MeshFunction< Grid< 1, GridReal, Device, Index >,EntityDimension, RealType> MeshFunctionType; - typedef typename Grid< 1, GridReal, Device, Index >::DistributedMeshType DistributedGridType; - typedef typename DistributedGridType::CoordinatesType CoordinatesType; - using SubdomainOverlapsType = typename DistributedGridType::SubdomainOverlapsType; - - DistributedMeshSynchronizer() - { - isSet = false; - }; - - DistributedMeshSynchronizer( DistributedGridType *distributedGrid ) - { - isSet = false; - setDistributedGrid( distributedGrid ); - }; - - void setDistributedGrid( DistributedGridType *distributedGrid) - { - isSet=true; - - this->distributedGrid=distributedGrid; - - const SubdomainOverlapsType& lowerOverlap = distributedGrid->getLowerOverlap(); - const SubdomainOverlapsType& upperOverlap = distributedGrid->getUpperOverlap(); - const CoordinatesType& localSize = distributedGrid->getLocalSize(); - const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); - - - sendBuffers[ ZzYzXm ].setSize( lowerOverlap.x() ); - sendBuffers[ ZzYzXp ].setSize( upperOverlap.x() ); - receiveBuffers[ ZzYzXm ].setSize( lowerOverlap.x() ); - receiveBuffers[ ZzYzXp ].setSize( upperOverlap.x() ); - - }; - - template< typename CommunicatorType, - typename MeshFunctionType, - typename PeriodicBoundariesMaskPointer = Pointers::SharedPointer< MeshFunctionType > > - void synchronize( MeshFunctionType &meshFunction, - bool periodicBoundaries = false, - const PeriodicBoundariesMaskPointer& mask = PeriodicBoundariesMaskPointer( nullptr ) ) - { - TNL_ASSERT_TRUE( isSet, "Synchronizer is not set, but used to synchronize" ); - - if( !distributedGrid->isDistributed() ) - return; - - int totalSize = meshFunction.getMesh().getDimensions().x(); - const SubdomainOverlapsType& lowerOverlap = distributedGrid->getLowerOverlap(); - const SubdomainOverlapsType& upperOverlap = distributedGrid->getUpperOverlap(); - const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); - - leftSource = lowerOverlap.x(); - rightSource = localGridSize.x() - 2 * upperOverlap.x(); - leftDestination = 0; - rightDestination = localGridSize.x() - upperOverlap.x(); - - const int *neighbors = distributedGrid->getNeighbors(); - const int *periodicNeighbors = distributedGrid->getPeriodicNeighbors(); - - if( periodicBoundaries ) - { - if( neighbors[ ZzYzXm ] == -1 ) - swap( leftSource, leftDestination ); - if( neighbors[ ZzYzXp ] == -1 ) - swap( rightSource, rightDestination ); - } - - copyBuffers( meshFunction, sendBuffers, true, - leftSource, rightSource, - lowerOverlap, upperOverlap, - neighbors, - periodicBoundaries, - PeriodicBoundariesMaskPointer( nullptr ) ); // the mask is used only when receiving data - - //async send - typename CommunicatorType::Request requests[ 4 ]; - typename CommunicatorType::CommunicationGroup group; - group=*((typename CommunicatorType::CommunicationGroup *)(distributedGrid->getCommunicationGroup())); - int requestsCount( 0 ); - - //send everything, recieve everything - if( neighbors[ ZzYzXm ] != -1 ) - { - TNL_ASSERT_GE( sendBuffers[ ZzYzXm ].getSize(), lowerOverlap.x(), "" ); - TNL_ASSERT_GE( receiveBuffers[ ZzYzXm ].getSize(), lowerOverlap.x(), "" ); - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ ZzYzXm ].getData(), lowerOverlap.x(), neighbors[ ZzYzXm ],0, group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ ZzYzXm ].getData(), lowerOverlap.x(), neighbors[ ZzYzXm ],0, group ); - } - else if( periodicBoundaries ) - { - TNL_ASSERT_GE( sendBuffers[ ZzYzXm ].getSize(), lowerOverlap.x(), "" ); - TNL_ASSERT_GE( receiveBuffers[ ZzYzXm ].getSize(), lowerOverlap.x(), "" ); - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ ZzYzXm ].getData(), lowerOverlap.x(), periodicNeighbors[ ZzYzXm ],1, group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ ZzYzXm ].getData(), lowerOverlap.x(), periodicNeighbors[ ZzYzXm ],1, group ); - } - - if( neighbors[ ZzYzXp ] != -1 ) - { - TNL_ASSERT_GE( sendBuffers[ ZzYzXp ].getSize(), upperOverlap.x(), "" ); - TNL_ASSERT_GE( receiveBuffers[ ZzYzXp ].getSize(), upperOverlap.x(), "" ); - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ ZzYzXp ].getData(), upperOverlap.x(), neighbors[ ZzYzXp ], 0, group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ ZzYzXp ].getData(), upperOverlap.x(), neighbors[ ZzYzXp ], 0, group ); - } - else if( periodicBoundaries ) - { - TNL_ASSERT_GE( sendBuffers[ ZzYzXp ].getSize(), upperOverlap.x(), "" ); - TNL_ASSERT_GE( receiveBuffers[ ZzYzXp ].getSize(), upperOverlap.x(), "" ); - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ ZzYzXp ].getData(), upperOverlap.x(), periodicNeighbors[ ZzYzXp ], 1, group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ ZzYzXp ].getData(), upperOverlap.x(), periodicNeighbors[ ZzYzXp ], 1, group ); - } - - //wait until send and receive is done - CommunicatorType::WaitAll( requests, requestsCount ); - - copyBuffers( meshFunction, receiveBuffers, false, - leftDestination, rightDestination, - lowerOverlap, - upperOverlap, - neighbors, - periodicBoundaries, - mask ); - } - - private: - template< typename Real_, - typename MeshFunctionType, - typename PeriodicBoundariesMaskPointer > - void copyBuffers( - MeshFunctionType& meshFunction, - TNL::Containers::Array* buffers, - bool toBuffer, - int left, int right, - const SubdomainOverlapsType& lowerOverlap, - const SubdomainOverlapsType& upperOverlap, - const int* neighbors, - bool periodicBoundaries, - const PeriodicBoundariesMaskPointer& mask ) - - { - typedef BufferEntitiesHelper< MeshFunctionType, PeriodicBoundariesMaskPointer, 1, Real_, Device > Helper; - bool leftIsBoundary = ( neighbors[ ZzYzXm ] == -1 ); - bool rightIsBoundary = ( neighbors[ ZzYzXp ] == -1 ); - if( ! leftIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYzXm ].getData(), leftIsBoundary, left, lowerOverlap.x(), toBuffer ); - if( ! rightIsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYzXp ].getData(), rightIsBoundary, right, upperOverlap.x(), toBuffer ); - } - - Containers::Array sendBuffers[ 2 ], receiveBuffers[ 2 ]; - - DistributedGridType *distributedGrid; - - int leftSource; - int rightSource; - int leftDestination; - int rightDestination; - - bool isSet; -}; - -} // namespace DistributedMeshes -} // namespace Meshes -} // namespace TNL diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_2D.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_2D.h deleted file mode 100644 index 047e81748..000000000 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_2D.h +++ /dev/null @@ -1,246 +0,0 @@ -/*************************************************************************** - DistributedGridSynchronizer_2D.h - description - ------------------- - begin : Aug 15, 2018 - copyright : (C) 2018 by Tomas Oberhuber - email : tomas.oberhuber@fjfi.cvut.cz - ***************************************************************************/ - -/* See Copyright Notice in tnl/Copyright */ - -#pragma once - -#include -#include -#include - - -namespace TNL { -namespace Functions{ -template< typename Mesh, - int MeshEntityDimension, - typename Real > -class MeshFunction; -}//Functions -}//TNL - -namespace TNL { -namespace Meshes { -namespace DistributedMeshes { - -template -class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< 2, GridReal, Device, Index >,EntityDimension, RealType>> -{ - - public: - - typedef typename Grid< 2, GridReal, Device, Index >::Cell Cell; - // FIXME: clang does not like this (incomplete type error) -// typedef typename Functions::MeshFunction< Grid< 2, GridReal, Device, Index >,EntityDimension, RealType> MeshFunctionType; - typedef typename Grid< 2, GridReal, Device, Index >::DistributedMeshType DistributedGridType; - typedef typename DistributedGridType::CoordinatesType CoordinatesType; - using SubdomainOverlapsType = typename DistributedGridType::SubdomainOverlapsType; - - - DistributedMeshSynchronizer() - { - isSet=false; - }; - - DistributedMeshSynchronizer(DistributedGridType *distributedGrid) - { - isSet=false; - setDistributedGrid( distributedGrid ); - }; - - void setDistributedGrid( DistributedGridType *distributedGrid ) - { - isSet=true; - - this->distributedGrid = distributedGrid; - - const SubdomainOverlapsType& lowerOverlap = distributedGrid->getLowerOverlap(); - const SubdomainOverlapsType& upperOverlap = distributedGrid->getUpperOverlap(); - const CoordinatesType& localSize = distributedGrid->getLocalSize(); - const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); - - sizes[ ZzYzXm ] = localSize.y() * lowerOverlap.x(); - sizes[ ZzYzXp ] = localSize.y() * upperOverlap.x(); - sizes[ ZzYmXz ] = localSize.x() * lowerOverlap.y(); - sizes[ ZzYpXz ] = localSize.x() * upperOverlap.y(); - sizes[ ZzYmXm ] = lowerOverlap.x() * lowerOverlap.y(); - sizes[ ZzYpXm ] = lowerOverlap.x() * upperOverlap.y(); - sizes[ ZzYmXp ] = upperOverlap.x() * lowerOverlap.y(); - sizes[ ZzYpXp ] = upperOverlap.x() * upperOverlap.y(); - - for(int i=0;i<8;i++) - { - sendBuffers[ i ].setSize( sizes[ i ] ); - receiveBuffers[ i ].setSize( sizes[ i ] ); - } - - } - - template< typename CommunicatorType, - typename MeshFunctionType, - typename PeriodicBoundariesMaskPointer = Pointers::SharedPointer< MeshFunctionType > > - void synchronize( MeshFunctionType &meshFunction, - bool periodicBoundaries = false, - const PeriodicBoundariesMaskPointer& mask = PeriodicBoundariesMaskPointer( nullptr ) ) - { - - TNL_ASSERT_TRUE( isSet, "Synchronizer is not set, but used to synchronize" ); - - if( !distributedGrid->isDistributed() ) - return; - - const SubdomainOverlapsType& lowerOverlap = distributedGrid->getLowerOverlap(); - const SubdomainOverlapsType& upperOverlap = distributedGrid->getUpperOverlap(); - const CoordinatesType& localSize = distributedGrid->getLocalSize(); - const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); - - ZzYzXm_Source = lowerOverlap.x(); - ZzYzXp_Source = localGridSize.x() - 2 * upperOverlap.x(); - ZzYmXz_Source = lowerOverlap.y(); - ZzYpXz_Source = localGridSize.y() - 2 * upperOverlap.y(); - - xCenter = lowerOverlap.x(); - yCenter = lowerOverlap.y(); - - ZzYzXm_Destination = 0; - ZzYzXp_Destination = localGridSize.x() - upperOverlap.x(); - ZzYmXz_Destination = 0; - ZzYpXz_Destination = localGridSize.y() - upperOverlap.y(); - - const int *neighbors = distributedGrid->getNeighbors(); - const int *periodicNeighbors = distributedGrid->getPeriodicNeighbors(); - - if( periodicBoundaries ) - { - if( neighbors[ ZzYzXm ] == -1 ) - swap( ZzYzXm_Source, ZzYzXm_Destination ); - if( neighbors[ ZzYzXp ] == -1 ) - swap( ZzYzXp_Source, ZzYzXp_Destination ); - if( neighbors[ ZzYmXz ] == -1 ) - swap( ZzYmXz_Source, ZzYmXz_Destination ); - if( neighbors[ ZzYpXz ] == -1 ) - swap( ZzYpXz_Source, ZzYpXz_Destination ); - } - - copyBuffers(meshFunction, sendBuffers, true, - ZzYzXm_Source, ZzYzXp_Source, ZzYmXz_Source, ZzYpXz_Source, - xCenter, yCenter, - lowerOverlap, upperOverlap, localSize, - neighbors, - periodicBoundaries, - PeriodicBoundariesMaskPointer( nullptr ) ); // the mask is used only when receiving data - - //async send and receive - typename CommunicatorType::Request requests[ 16 ]; - typename CommunicatorType::CommunicationGroup group; - group=*((typename CommunicatorType::CommunicationGroup *)(distributedGrid->getCommunicationGroup())); - int requestsCount( 0 ); - - //send everything, receive everything - for( int i = 0; i < 8; i++ ) - { - if( neighbors[ i ] != -1 ) - { - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sizes[ i ], neighbors[ i ], 0, group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ i ].getData(), sizes[ i ], neighbors[ i ], 0, group ); - } - else if( periodicBoundaries ) - { - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sizes[ i ], periodicNeighbors[ i ], 1, group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( receiveBuffers[ i ].getData(), sizes[ i ], periodicNeighbors[ i ], 1, group ); - } - } - - //wait until send is done - CommunicatorType::WaitAll( requests, requestsCount ); - - //copy data from receive buffers - copyBuffers(meshFunction, receiveBuffers, false, - ZzYzXm_Destination, ZzYzXp_Destination, ZzYmXz_Destination, ZzYpXz_Destination, - xCenter, yCenter, - lowerOverlap, upperOverlap, localSize, - neighbors, - periodicBoundaries, - mask ); - } - - private: - - template< typename Real_, - typename MeshFunctionType, - typename PeriodicBoundariesMaskPointer > - void copyBuffers( - MeshFunctionType& meshFunction, - Containers::Array* buffers, - bool toBuffer, - int ZzYzXm_Position, int ZzYzXp_Position, int ZzYmXz_Position, int ZzYpXz_Position, - int xcenter, int ycenter, - const CoordinatesType& lowerOverlap, - const CoordinatesType& upperOverlap, - const CoordinatesType& localSize, - const int *neighbors, - bool periodicBoundaries, - const PeriodicBoundariesMaskPointer& mask ) - { - bool ZzYzXm_IsBoundary = ( neighbors[ ZzYzXm ] == -1 ); - bool ZzYzXp_IsBoundary = ( neighbors[ ZzYzXp ] == -1 ); - bool ZzYmXz_IsBoundary = ( neighbors[ ZzYmXz ] == -1 ); - bool ZzYpXz_IsBoundary = ( neighbors[ ZzYpXz ] == -1 ); - bool ZzYmXm_IsBoundary = ( neighbors[ ZzYmXm ] == -1 ); - bool ZzYmXp_IsBoundary = ( neighbors[ ZzYmXp ] == -1 ); - bool ZzYpXm_IsBoundary = ( neighbors[ ZzYpXm ] == -1 ); - bool ZzYpXp_IsBoundary = ( neighbors[ ZzYpXp ] == -1 ); - - using Helper = BufferEntitiesHelper< MeshFunctionType, PeriodicBoundariesMaskPointer, 2, Real_, Device >; - if( ! ZzYzXm_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYzXm ].getData(), ZzYzXm_IsBoundary, ZzYzXm_Position, ycenter, lowerOverlap.x(), localSize.y(), toBuffer ); - if( ! ZzYzXp_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYzXp ].getData(), ZzYzXp_IsBoundary, ZzYzXp_Position, ycenter, upperOverlap.x(), localSize.y(), toBuffer ); - if( ! ZzYmXz_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYmXz ].getData(), ZzYmXz_IsBoundary, xcenter, ZzYmXz_Position, localSize.x(), lowerOverlap.y(), toBuffer ); - if( ! ZzYpXz_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYpXz ].getData(), ZzYpXz_IsBoundary, xcenter, ZzYpXz_Position, localSize.x(), upperOverlap.y(), toBuffer ); - if( ! ZzYmXm_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYmXm ].getData(), ZzYmXm_IsBoundary, ZzYzXm_Position, ZzYmXz_Position, lowerOverlap.x(), lowerOverlap.y(), toBuffer ); - if( ! ZzYmXp_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYmXp ].getData(), ZzYmXp_IsBoundary, ZzYzXp_Position, ZzYmXz_Position, upperOverlap.x(), lowerOverlap.y(), toBuffer ); - if( ! ZzYpXm_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYpXm ].getData(), ZzYpXm_IsBoundary, ZzYzXm_Position, ZzYpXz_Position, lowerOverlap.x(), upperOverlap.y(), toBuffer ); - if( ! ZzYpXp_IsBoundary || periodicBoundaries ) - Helper::BufferEntities( meshFunction, mask, buffers[ ZzYpXp ].getData(), ZzYpXp_IsBoundary, ZzYzXp_Position, ZzYpXz_Position, upperOverlap.x(), upperOverlap.y(), toBuffer ); - } - - DistributedGridType *distributedGrid; - - Containers::Array sendBuffers[8]; - Containers::Array receiveBuffers[8]; - Containers::StaticArray< 8, int > sizes; - - int ZzYzXm_Source; - int ZzYzXp_Source; - int ZzYmXz_Source; - int ZzYpXz_Source; - int xCenter; - int yCenter; - int ZzYzXm_Destination; - int ZzYzXp_Destination; - int ZzYmXz_Destination; - int ZzYpXz_Destination; - - bool isSet; -}; - -} // namespace DistributedMeshes -} // namespace Meshes -} // namespace TNL - - diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h deleted file mode 100644 index 80f35b5a7..000000000 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer_3D.h +++ /dev/null @@ -1,227 +0,0 @@ -/*************************************************************************** - DistributedGridSynchronizer_3D.h - description - ------------------- - begin : Aug 15, 2018 - copyright : (C) 2018 by Tomas Oberhuber - email : tomas.oberhuber@fjfi.cvut.cz - ***************************************************************************/ - -/* See Copyright Notice in tnl/Copyright */ - -#pragma once - -#include -#include -#include -#include - -namespace TNL { -namespace Functions{ -template< typename Mesh, - int MeshEntityDimension, - typename Real > -class MeshFunction; -}//Functions -}//TNL - -namespace TNL { -namespace Meshes { -namespace DistributedMeshes { - -template -class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension, GridReal, Device, Index >,EntityDimension, RealType>> -{ - - public: - static constexpr int getMeshDimension() { return MeshDimension; }; - static constexpr int getNeighborCount() {return DirectionCount::get();}; - - typedef typename Grid< MeshDimension, GridReal, Device, Index >::Cell Cell; - // FIXME: clang does not like this (incomplete type error) -// typedef typename Functions::MeshFunction< Grid< 3, GridReal, Device, Index >,EntityDimension, RealType> MeshFunctionType; - typedef typename Grid< MeshDimension, GridReal, Device, Index >::DistributedMeshType DistributedGridType; - typedef typename DistributedGridType::CoordinatesType CoordinatesType; - using SubdomainOverlapsType = typename DistributedGridType::SubdomainOverlapsType; - - DistributedMeshSynchronizer() - { - isSet = false; - }; - - DistributedMeshSynchronizer( DistributedGridType *distributedGrid ) - { - isSet = false; - setDistributedGrid( distributedGrid ); - }; - - void setDistributedGrid( DistributedGridType *distributedGrid ) - { - isSet = true; - - this->distributedGrid = distributedGrid; - - const SubdomainOverlapsType& globalLowerOverlap = this->distributedGrid->getGlobalLowerOverlap(); - const SubdomainOverlapsType& globalUpperOverlap = this->distributedGrid->getGlobalUpperOverlap(); - // const SubdomainOverlapsType& globalPeriodicBCOverlap = this->distributedGrid->getGlobalPeriodicBCOverlap(); - const CoordinatesType& localBegin = this->distributedGrid->getLocalBegin(); - const CoordinatesType& localSize = this->distributedGrid->getLocalSize(); - const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); - - for( int i=0; igetNeighborCount(); i++ ) - { - Index sendSize=1; - Index rcvSize=1; - - //bool isBoundary=( neighbor[ i ] == -1 ); - auto directions=Directions::template getXYZgetMeshDimension()>(i); - - sendDimensions[i]=localSize; - recieveDimensions[i]=localSize; - sendBegin[i]=localBegin; - recieveBegin[i]=localBegin; - - for(int j=0;jgetMeshDimension();j++) - { - if(directions[j]==-1) - { - //TODO:periodicBC - sendDimensions[i][j]=globalUpperOverlap[j]; - recieveDimensions[i][j]=globalLowerOverlap[j]; - recieveBegin[i][j]=0; - } - - if(directions[j]==1) - { - //TODO:periodicBC - sendDimensions[i][j]=globalLowerOverlap[j]; - recieveDimensions[i][j]=globalUpperOverlap[j]; - sendBegin[i][j]=localBegin[j]+localSize[j]-globalLowerOverlap[j]; - recieveBegin[i][j]=localBegin[j]+localSize[j]; - } - - sendSize*=sendDimensions[i][j]; - rcvSize*=recieveDimensions[i][j]; - } - - sendSizes[ i ] = sendSize; - recieveSizes[ i ] = rcvSize; - sendBuffers[ i ].setSize( sendSize ); - recieveBuffers[ i ].setSize( rcvSize); - } - - } - - template< typename CommunicatorType, - typename MeshFunctionType, - typename PeriodicBoundariesMaskPointer = Pointers::SharedPointer< MeshFunctionType > > - void synchronize( MeshFunctionType &meshFunction, - bool periodicBoundaries = false, - const PeriodicBoundariesMaskPointer& mask = PeriodicBoundariesMaskPointer( nullptr ) ) - { - - TNL_ASSERT_TRUE( isSet, "Synchronizer is not set, but used to synchronize" ); - - if( !distributedGrid->isDistributed() ) return; - - const int *neighbors = distributedGrid->getNeighbors(); - const int *periodicNeighbors = distributedGrid->getPeriodicNeighbors(); - - if( periodicBoundaries ) - { - std::cerr<<"TOTALY DEMAGED by refactorization" << std::endl; - } - - //fill send buffers - copyBuffers( meshFunction, - sendBuffers, sendBegin,sendDimensions, - true, - neighbors, - periodicBoundaries, - PeriodicBoundariesMaskPointer( nullptr ) ); // the mask is used only when receiving data ); - - //async send and receive - typename CommunicatorType::Request requests[2*this->getNeighborCount()]; - typename CommunicatorType::CommunicationGroup group; - group=*((typename CommunicatorType::CommunicationGroup *)(distributedGrid->getCommunicationGroup())); - int requestsCount( 0 ); - - //send everything, recieve everything - for( int i=0; igetNeighborCount(); i++ ) - if( neighbors[ i ] != -1 ) - { - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sendSizes[ i ], neighbors[ i ],0, group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( recieveBuffers[ i ].getData(), recieveSizes[ i ], neighbors[ i ], 0, group ); - } - else if( periodicBoundaries ) - { - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sendSizes[ i ], periodicNeighbors[ i ], 1, group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( recieveBuffers[ i ].getData(), recieveSizes[ i ], periodicNeighbors[ i ],1, group ); - } - - //wait until send is done - CommunicatorType::WaitAll( requests, requestsCount ); - - //copy data from receive buffers - copyBuffers(meshFunction, - recieveBuffers,recieveBegin,recieveDimensions , - false, - neighbors, - periodicBoundaries, - mask ); - } - - private: - template< typename Real_, - typename MeshFunctionType, - typename PeriodicBoundariesMaskPointer > - void copyBuffers( - MeshFunctionType& meshFunction, - Containers::Array* buffers, - CoordinatesType* begins, - CoordinatesType* sizes, - bool toBuffer, - const int* neighbor, - bool periodicBoundaries, - const PeriodicBoundariesMaskPointer& mask ) - { - using Helper = BufferEntitiesHelper< MeshFunctionType, PeriodicBoundariesMaskPointer, this->getMeshDimension(), Real_, Device >; - - for(int i=0;igetNeighborCount();i++) - { - bool isBoundary=( neighbor[ i ] == -1 ); - if( ! isBoundary || periodicBoundaries ) - { - Helper::BufferEntities( meshFunction, mask, buffers[ i ].getData(), isBoundary, begins[i], sizes[i], toBuffer ); - } - } - } - - private: - - Containers::Array sendBuffers[getNeighborCount()]; - Containers::Array recieveBuffers[getNeighborCount()]; - Containers::StaticArray< getNeighborCount(), int > sendSizes; - Containers::StaticArray< getNeighborCount(), int > recieveSizes; - - - CoordinatesType sendDimensions[getNeighborCount()]; - CoordinatesType recieveDimensions[getNeighborCount()]; - CoordinatesType sendBegin[getNeighborCount()]; - CoordinatesType recieveBegin[getNeighborCount()]; - - DistributedGridType *distributedGrid; - - bool isSet; - -}; - - -} // namespace DistributedMeshes -} // namespace Meshes -} // namespace TNL - diff --git a/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.h b/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.h index d5e7edd2a..774f5c24e 100644 --- a/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.h +++ b/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.h @@ -73,8 +73,6 @@ class SubdomainOverlapsGetter< Grid< Dimension, Real, Device, Index >, Communica static void getOverlaps( const DistributedMeshType* distributedMesh, SubdomainOverlapsType& lower, SubdomainOverlapsType& upper, - SubdomainOverlapsType& globalLower, - SubdomainOverlapsType& globalUpper, IndexType subdomainOverlapSize, const SubdomainOverlapsType& periodicBoundariesOverlapSize = 0 ); diff --git a/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.hpp b/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.hpp index 5ff558d40..7d84382b9 100644 --- a/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.hpp +++ b/src/TNL/Meshes/DistributedMeshes/SubdomainOverlapsGetter.hpp @@ -27,8 +27,6 @@ SubdomainOverlapsGetter< Grid< Dimension, Real, Device, Index >, Communicator >: getOverlaps( const DistributedMeshType* distributedMesh, SubdomainOverlapsType& lower, SubdomainOverlapsType& upper, - SubdomainOverlapsType& globalLower, - SubdomainOverlapsType& globalUpper, IndexType subdomainOverlapSize, const SubdomainOverlapsType& periodicBoundariesOverlapSize ) { @@ -40,8 +38,6 @@ getOverlaps( const DistributedMeshType* distributedMesh, for( int i = 0; i < Dimension; i++ ) { - globalLower[i]=subdomainOverlapSize; - globalUpper[i]=subdomainOverlapSize; if( subdomainCoordinates[ i ] > 0 ) lower[ i ] = subdomainOverlapSize; diff --git a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp index 2e5cdb4b2..f3a8b8cd8 100644 --- a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp +++ b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp @@ -140,12 +140,12 @@ class DistributedGridTest_1D : public ::testing::Test overlap.setValue(1); distributedGrid=new DistributedGridType(); - typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap; + typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap; distributedGrid->template setGlobalGrid( globalGrid ); //distributedGrid->setupGrid(*gridptr); SubdomainOverlapsGetter< GridType, CommunicatorType >:: - getOverlaps( distributedGrid, lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap, 1 ); - distributedGrid->setOverlaps( lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap ); + getOverlaps( distributedGrid, lowerOverlap, upperOverlap, 1 ); + distributedGrid->setOverlaps( lowerOverlap, upperOverlap ); distributedGrid->setupGrid(*gridptr); dof.setSize( gridptr->template getEntitiesCount< Cell >() ); @@ -157,10 +157,10 @@ class DistributedGridTest_1D : public ::testing::Test void SetUpPeriodicBoundaries() { - typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap; + typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap; SubdomainOverlapsGetter< GridType, CommunicatorType >:: - getOverlaps( distributedGrid, lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap, 1 ); - distributedGrid->setOverlaps( lowerOverlap, upperOverlap, globalLowerOverlap, globalUpperOverlap ); + getOverlaps( distributedGrid, lowerOverlap, upperOverlap, 1 ); + distributedGrid->setOverlaps( lowerOverlap, upperOverlap ); distributedGrid->setupGrid(*gridptr); } @@ -238,8 +238,8 @@ TEST_F(DistributedGridTest_1D, EvaluateLinearFunction ) EXPECT_EQ(meshFunctionPtr->getValue(entity), (*linearFunctionPtr)(entity)) << "Linear function Overlap error on right Edge."; } -/* -TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithoutMask ) + +/*TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithoutMask ) { // Setup periodic boundaries // TODO: I do not know how to do it better with GTEST @@ -257,12 +257,16 @@ TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithoutMask ) maskDofs.setValue( true ); constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr, constFunctionPtr ); meshFunctionPtr->template synchronize( true ); + + std::cout << rank << ": " << dof << std::endl; + if( rank == 0 ) EXPECT_EQ( dof[ 1 ], -nproc ) << "Left Overlap was filled by wrong process."; if( rank == nproc-1 ) EXPECT_EQ( dof[ dof.getSize() - 2 ], -1 )<< "Right Overlap was filled by wrong process."; -} +}*/ +/* TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithActiveMask ) { // Setup periodic boundaries diff --git a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp index 3f6c67c2b..a066844f9 100644 --- a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp +++ b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp @@ -369,10 +369,10 @@ class DistributedGridTest_2D : public ::testing::Test distributedGrid=new DistributedGridType(); distributedGrid->setDomainDecomposition( typename DistributedGridType::CoordinatesType( 3, 3 ) ); distributedGrid->template setGlobalGrid( globalGrid ); - typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap; + typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap; SubdomainOverlapsGetter< GridType, CommunicatorType >:: - getOverlaps( distributedGrid, lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap, 1 ); - distributedGrid->setOverlaps( lowerOverlap, upperOverlap, globalLowerOverlap, globalUpperOverlap ); + getOverlaps( distributedGrid, lowerOverlap, upperOverlap, 1 ); + distributedGrid->setOverlaps( lowerOverlap, upperOverlap ); distributedGrid->setupGrid(*gridPtr); dof=new DofType(gridPtr->template getEntitiesCount< Cell >()); diff --git a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_3D.cpp b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_3D.cpp index dab290247..f92ef08d7 100644 --- a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_3D.cpp +++ b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_3D.cpp @@ -646,10 +646,10 @@ class DistributedGirdTest_3D : public ::testing::Test distributedGrid->setDomainDecomposition( typename DistributedGridType::CoordinatesType( 3, 3, 3 ) ); distributedGrid->template setGlobalGrid( globalGrid ); distributedGrid->setupGrid(*gridptr); - typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap; + typename DistributedGridType::SubdomainOverlapsType lowerOverlap, upperOverlap; SubdomainOverlapsGetter< GridType, CommunicatorType >:: - getOverlaps( distributedGrid, lowerOverlap, upperOverlap,globalLowerOverlap, globalUpperOverlap, 1 ); - distributedGrid->setOverlaps( lowerOverlap, upperOverlap, globalLowerOverlap, globalUpperOverlap ); + getOverlaps( distributedGrid, lowerOverlap, upperOverlap, 1 ); + distributedGrid->setOverlaps( lowerOverlap, upperOverlap ); distributedGrid->setupGrid(*gridptr); dof=new DofType(gridptr->template getEntitiesCount< Cell >()); -- GitLab From 094ffa0ea6325fc923067de02f1868f962f2893c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Hanousek=20-=20vz?= Date: Thu, 22 Nov 2018 11:34:53 +0100 Subject: [PATCH 5/7] FIX after rebase Issue #15 working basic synchronizer -Periodic BC doesnt work --- .../DistributedMeshes/DistributedGridSynchronizer.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h index e07eec29b..17e3e41bb 100644 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h +++ b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h @@ -146,13 +146,13 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension, for( int i=0; igetNeighborCount(); i++ ) if( neighbors[ i ] != -1 ) { - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sendSizes[ i ], neighbors[ i ], group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( recieveBuffers[ i ].getData(), sendSizes[ i ], neighbors[ i ], group ); + requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sendSizes[ i ], neighbors[ i ], 0, group ); + requests[ requestsCount++ ] = CommunicatorType::IRecv( recieveBuffers[ i ].getData(), sendSizes[ i ], neighbors[ i ], 0, group ); } else if( periodicBoundaries && sendSizes[ i ] !=0 ) { - requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sendSizes[ i ], periodicNeighbors[ i ], group ); - requests[ requestsCount++ ] = CommunicatorType::IRecv( recieveBuffers[ i ].getData(), sendSizes[ i ], periodicNeighbors[ i ], group ); + requests[ requestsCount++ ] = CommunicatorType::ISend( sendBuffers[ i ].getData(), sendSizes[ i ], periodicNeighbors[ i ], 1, group ); + requests[ requestsCount++ ] = CommunicatorType::IRecv( recieveBuffers[ i ].getData(), sendSizes[ i ], periodicNeighbors[ i ], 1, group ); } //wait until send is done -- GitLab From dfcbf832d5ebb847925a544c25c437fcebc3a7a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Hanousek=20-=20vz?= Date: Thu, 22 Nov 2018 12:44:12 +0100 Subject: [PATCH 6/7] Issue #15 - new DistributedGridSynchronizer pass all tests - added compatibility code for periodic-BC - this part of code needs discusion if work corectly --- .../DistributedGridSynchronizer.h | 36 +++++++++++++------ .../DistributedGridTest_1D.cpp | 10 +++--- .../DistributedGridTest_2D.cpp | 4 +-- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h index 17e3e41bb..4106a0a54 100644 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h +++ b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h @@ -70,16 +70,18 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension, const CoordinatesType& localBegin = this->distributedGrid->getLocalBegin(); const CoordinatesType& localSize = this->distributedGrid->getLocalSize(); - const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); + const CoordinatesType& localGridSize = this->distributedGrid->getLocalGridSize(); + + const int *neighbors = distributedGrid->getNeighbors(); for( int i=0; igetNeighborCount(); i++ ) { Index sendSize=1;//sended and recieve areas has same size - //bool isBoundary=( neighbor[ i ] == -1 ); - auto directions=Directions::template getXYZgetMeshDimension()>(i); + // bool isBoundary=( neighbor[ i ] == -1 ); + auto directions=Directions::template getXYZ(i); - sendDimensions[i]=localSize;//sended and recieve areas has same dimensions + sendDimensions[i]=localSize;//send and recieve areas has same dimensions sendBegin[i]=localBegin; recieveBegin[i]=localBegin; @@ -104,8 +106,25 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension, sendSizes[ i ] = sendSize; sendBuffers[ i ].setSize( sendSize ); recieveBuffers[ i ].setSize( sendSize); + + //Periodic-BC copy from overlap into domain + //if Im on boundary, and i is direction of the boundary i will swap source and destination + //i do this only for basic 6 directions, + //because this swap at conners and edges produces writing multiple values at sam place in localsubdomain + { + if( ( i==ZzYzXm || i==ZzYzXp + ||i==ZzYmXz || i==ZzYpXz + ||i==ZmYzXz || i==ZpYzXz ) + && neighbors[ i ] == -1) + { + //swap begins + CoordinatesType tmp = sendBegin[i]; + sendBegin[i]=recieveBegin[i]; + recieveBegin[i]=tmp; + } + } + } - } template< typename CommunicatorType, @@ -122,11 +141,6 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension, const int *neighbors = distributedGrid->getNeighbors(); const int *periodicNeighbors = distributedGrid->getPeriodicNeighbors(); - - if( periodicBoundaries ) - { - std::cerr<<"TOTALY DEMAGED by refactorization" << std::endl; - } //fill send buffers copyBuffers( meshFunction, @@ -181,7 +195,7 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension, bool periodicBoundaries, const PeriodicBoundariesMaskPointer& mask ) { - using Helper = BufferEntitiesHelper< MeshFunctionType, PeriodicBoundariesMaskPointer, this->getMeshDimension(), Real_, Device >; + using Helper = BufferEntitiesHelper< MeshFunctionType, PeriodicBoundariesMaskPointer, getMeshDimension(), Real_, Device >; for(int i=0;igetNeighborCount();i++) { diff --git a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp index f3a8b8cd8..fca8bb8cf 100644 --- a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp +++ b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_1D.cpp @@ -239,7 +239,7 @@ TEST_F(DistributedGridTest_1D, EvaluateLinearFunction ) } -/*TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithoutMask ) +TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithoutMask ) { // Setup periodic boundaries // TODO: I do not know how to do it better with GTEST @@ -258,15 +258,13 @@ TEST_F(DistributedGridTest_1D, EvaluateLinearFunction ) constFunctionEvaluator.evaluateAllEntities( meshFunctionPtr, constFunctionPtr ); meshFunctionPtr->template synchronize( true ); - std::cout << rank << ": " << dof << std::endl; - if( rank == 0 ) EXPECT_EQ( dof[ 1 ], -nproc ) << "Left Overlap was filled by wrong process."; if( rank == nproc-1 ) EXPECT_EQ( dof[ dof.getSize() - 2 ], -1 )<< "Right Overlap was filled by wrong process."; -}*/ +} + -/* TEST_F(DistributedGridTest_1D, SynchronizePeriodicNeighborsWithActiveMask ) { // Setup periodic boundaries @@ -377,7 +375,7 @@ TEST_F(DistributedGridTest_1D, SynchronizePeriodicBoundariesLinearTest ) if( rank == nproc - 1 ) EXPECT_EQ( meshFunctionPtr->getValue(entity2), -1 ) << "Linear function Overlap error on right Edge."; } -*/ + #else diff --git a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp index a066844f9..26bfbb457 100644 --- a/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp +++ b/src/UnitTests/Meshes/DistributedMeshes/DistributedGridTest_2D.cpp @@ -525,7 +525,7 @@ TEST_F(DistributedGridTest_2D, SynchronizerNeighborTest ) } } -/*TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithoutMask ) +TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithoutMask ) { // Setup periodic boundaries // TODO: I do not know how to do it better with GTEST - additional setup @@ -1004,7 +1004,7 @@ TEST_F(DistributedGridTest_2D, SynchronizerNeighborPeriodicBoundariesWithInActiv checkDownBoundary( *gridPtr, *dof, true, false, 8 ); checkRightBoundary( *gridPtr, *dof, true, false, -7 ); } -}*/ +} #else TEST(NoMPI, NoTest) -- GitLab From e1dec97775349fa89e3b99ead99d58b3eddc7680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Hanousek=20-=20vz?= Date: Thu, 22 Nov 2018 13:00:47 +0100 Subject: [PATCH 7/7] FIX remove unused field --- src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h index 4106a0a54..b68136e07 100644 --- a/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h +++ b/src/TNL/Meshes/DistributedMeshes/DistributedGridSynchronizer.h @@ -212,8 +212,6 @@ class DistributedMeshSynchronizer< Functions::MeshFunction< Grid< MeshDimension, Containers::Array sendBuffers[getNeighborCount()]; Containers::Array recieveBuffers[getNeighborCount()]; Containers::StaticArray< getNeighborCount(), int > sendSizes; - Containers::StaticArray< getNeighborCount(), int > recieveSizes; - CoordinatesType sendDimensions[getNeighborCount()]; CoordinatesType recieveDimensions[getNeighborCount()]; -- GitLab