Simple Test of 1D distribued grid,build and run by cmake. (6acb41f3) · Commits · TNL / tnl-dev

CMakeLists.txt

+18 −10

Original line number	Diff line number	Diff line
		@@ -53,7 +53,7 @@ if( CXX_COMPILER_NAME MATCHES "icpc" )
		message( "Intel compiler detected..." )
		set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_ICPC -wd2568 -wd2571 -wd2570")
		#####
		# Ckeck for MIC
		# Check for MIC
		#
		if( WITH_MIC STREQUAL "yes" )
		message( "Compile MIC support..." )
		@@ -66,6 +66,23 @@ if( CXX_COMPILER_NAME MATCHES "icpc" )
		endif( )
		endif()

		#####
		# Check for MPI -- poznej podle vraperu compileru -- da se testovat preklad bez MPI
		#
		if( ${CXX_COMPILER_NAME} STREQUAL "mpic++" )
		message( "MPI compiler detected." )
		set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_MPI" )
		endif()

		####
		# Check for MPI
		#
		#find_package( MPI )
		#if( MPI_CXX_FOUND )
		# set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_MPI" )
		# message( "MPI headers found -- ${MPI_CXX_INCLUDE_PATH}")
		#endif()

		#####
		# Check for CUDA
		#
		@@ -195,15 +212,6 @@ if( OPENMP_FOUND )
		endif()
		#set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_OPENMP -fopenmp=libgomp" )

		####
		# Check for MPI
		#
		#find_package( MPI )
		#if( MPI_CXX_FOUND )
		# set( BUILD_MPI TRUE )
		# message( "MPI headers found -- ${MPI_CXX_INCLUDE_PATH}")
		#endif()

		find_package( DCMTK )
		if( DCMTK_FOUND )
		set( HAVE_DCMTK_H "#define HAVE_DCMTK_H 1" )

src/TNL/Config/ParameterContainer.cpp

+1 −1

Original line number	Diff line number	Diff line
		@@ -95,7 +95,7 @@ ParameterContainer::

		/*void ParameterContainer::MPIBcast( int root, MPI_Comm mpi_comm )
		{
		#ifdef HAVE_MPI
		#ifdef USE_MPI
		int i;
		int size = parameters. getSize();
		:: MPIBcast( size, 1, root, mpi_comm );

src/TNL/Functions/MeshFunction_impl.h

+2 −2

Original line number	Diff line number	Diff line
		@@ -312,7 +312,7 @@ MeshFunction< Mesh, MeshEntityDimension, Real >::
		getValue( const EntityType& meshEntity ) const
		{
		static_assert( EntityType::getEntityDimension() == MeshEntityDimension, "Calling with wrong EntityType -- entity dimensions do not match." );
		return this->data.getValue( meshEntity.getIndex() );
		return this->data.getElement( meshEntity.getIndex() );
		}

		template< typename Mesh,
		@@ -325,7 +325,7 @@ setValue( const EntityType& meshEntity,
		const RealType& value )
		{
		static_assert( EntityType::getEntityDimension() == MeshEntityDimension, "Calling with wrong EntityType -- entity dimensions do not match." );
		this->data.setValue( meshEntity.getIndex(), value );
		this->data.setElement( meshEntity.getIndex(), value );
		}

		template< typename Mesh,

src/TNL/Meshes/DistributedGrid.h

+15 −10

Original line number	Diff line number	Diff line
		@@ -10,28 +10,28 @@

		#pragma once

		#include <TNL/Meshes/Grid.h>
		#include <TNL/mpi-supp.h>
		#include <iostream>


		namespace TNL {
		namespace Meshes {

		/*template< typename OutMeshFunction,
		typename InFunction,
		typename Real >*/

		template<typename GridType,
		int meshDimensions= GridType::getMeshDimension()>
		class DistributedGrid
		{

		};
		}
		}

		#include <TNL/Meshes/Grid.h>
		#include <TNL/mpi-supp.h>

		namespace TNL {
		namespace Meshes {

		//#define HAVE_MPI
		#ifndef USE_MPI

		#ifndef HAVE_MPI
		template<typename GridType>
		class DistributedGrid <GridType,1>
		{
		@@ -106,7 +106,7 @@ class DistributedGrid <GridType,1>
		left=-1;
		right=-1;

		Dimensions= GridType::meshDimensions;
		Dimensions= GridType::getMeshDimension();
		GlobalGrid=globalGrid;
		//Detect MPI and number of process
		mpiInUse=false;
		@@ -707,6 +707,11 @@ class DistributedGrid <GridType,3>
		return this->localsize;
		}

		CoordinatesType getLocalGridSize()
		{
		return this->localgridsize;
		}

		CoordinatesType getLocalBegin()
		{
		return this->localbegin;

src/TNL/Meshes/DistributedGridSynchronizer.h

+165 −123

Original line number	Diff line number	Diff line
		@@ -32,13 +32,15 @@ template <typename DistributedGridType,
		typename MeshFunctionType>
		class DistributedGridSynchronizer<DistributedGridType,MeshFunctionType,1>
		{
		typedef typename MeshFunctionType::RealType Real;

		public:
		static void Synchronize(DistributedGridType distributedgrid, MeshFunctionType meshfunction)
		{
		if(!distributedgrid.isMPIUsed())
		return;
		#ifdef HAVE_MPI
		typedef typename MeshFunctionType::RealType Real;
		#ifdef USE_MPI


		Real * leftsendbuf;
		Real * rightsendbuf;
		@@ -144,7 +146,7 @@ class DistributedGridSynchronizer<DistributedGridType,MeshFunctionType,2>
		{
		if(!distributedgrid.isMPIUsed())
		return;
		#ifdef HAVE_MPI
		#ifdef USE_MPI
		typedef typename MeshFunctionType::RealType Real;
		typedef typename DistributedGridType::CoordinatesType CoordinatesType;

		@@ -271,25 +273,45 @@ template <typename DistributedGridType,
		typename MeshFunctionType>
		class DistributedGridSynchronizer<DistributedGridType,MeshFunctionType,3>
		{
		public:
		static void Synchronize(DistributedGridType distributedgrid, MeshFunctionType meshfunction)
		{
		if(!distributedgrid.isMPIUsed())
		return;
		#ifdef HAVE_MPI
		typedef typename MeshFunctionType::RealType Real;
		typedef typename DistributedGridType::CoordinatesType CoordinatesType;


		private:
		Real ** sendbuffs=new Real*[26];
		Real ** rcvbuffs=new Real*[26];
		int sizes[26];
		DistributedGridType *distributedgrid;

		int westSrc;
		int eastSrc;
		int nordSrc;
		int southSrc;
		int bottomSrc;
		int topSrc;
		int xcenter;
		int ycenter;
		int zcenter;
		int westDst;
		int eastDst;
		int nordDst;
		int southDst;
		int bottomDst;
		int topDst;

		CoordinatesType overlap;
		CoordinatesType localsize;

		CoordinatesType overlap = distributedgrid.getOverlap();
		CoordinatesType localgridsize = meshfunction.getMesh().getDimensions();
		CoordinatesType localsize=distributedgrid.getLocalSize();
		CoordinatesType localbegin=distributedgrid.getLocalBegin();
		public:

		int sizes[26];
		DistributedGridSynchronizer(DistributedGridType *distgrid)
		{
		#ifdef USE_MPI
		this->distributedgrid=distgrid;
		overlap = this->distributedgrid->getOverlap();
		localsize=this->distributedgrid->getLocalSize();

		CoordinatesType localbegin=this->distributedgrid->getLocalBegin();
		CoordinatesType localgridsize = this->distributedgrid->getLocalGridSize();

		sizes[West]=sizes[East]=localsize.y()localsize.z()overlap.x();
		sizes[Nord]=sizes[South]=localsize.x()localsize.z()overlap.y();
		@@ -309,46 +331,59 @@ class DistributedGridSynchronizer<DistributedGridType,MeshFunctionType,3>
		rcvbuffs[i]=new Real[sizes[i]];
		}

		westSrc=localbegin.x();
		eastSrc=localgridsize.x()-2*overlap.x();
		nordSrc=localbegin.y();
		southSrc=localgridsize.y()-2*overlap.y();
		bottomSrc=localbegin.z();
		topSrc=localgridsize.z()-2*overlap.z();

		xcenter=localbegin.x();
		ycenter=localbegin.y();
		zcenter=localbegin.z();

		westDst=0;
		eastDst=localgridsize.x()-overlap.x();
		nordDst=0;
		southDst=localgridsize.y()-overlap.y();
		bottomDst=0;
		topDst=localgridsize.z()-overlap.z();

		#endif
		}

		~DistributedGridSynchronizer()
		{
		#ifdef USE_MPI
		//free buffers
		for(int i=0;i<26;i++)
		{
		delete [] sendbuffs[i];
		delete [] rcvbuffs[i];
		}
		#endif

		}

		void Synchronize(MeshFunctionType meshfunction)
		{
		if(!distributedgrid->isMPIUsed())
		return;
		#ifdef USE_MPI

		int *neighbor=distributedgrid->getNeighbors();

		//fill send buffers
		//X-Y-Z
		BufferEntities(meshfunction,sendbuffs[West],localbegin.x(),localbegin.y(),localbegin.z(),overlap.x(),localsize.y(),localsize.z(),true);
		BufferEntities(meshfunction,sendbuffs[East],localgridsize.x()-2*overlap.x(),localbegin.y(),localbegin.z(),overlap.x(),localsize.y(),localsize.z(),true);
		BufferEntities(meshfunction,sendbuffs[Nord],localbegin.x(),localbegin.y(),localbegin.z(),localsize.x(),overlap.y(),localsize.z(),true);
		BufferEntities(meshfunction,sendbuffs[South],localbegin.x(),localgridsize.y()-2*overlap.y(),localbegin.z(),localsize.x(),overlap.y(),localsize.z(),true);
		BufferEntities(meshfunction,sendbuffs[Bottom],localbegin.x(),localbegin.y(),localbegin.z(),localsize.x(),localsize.y(),overlap.z(),true);
		BufferEntities(meshfunction,sendbuffs[Top],localbegin.x(),localbegin.y(),localgridsize.z()-2*overlap.z(),localsize.x(),localsize.y(),overlap.z(),true);
		//XY
		BufferEntities(meshfunction,sendbuffs[NordWest],localbegin.x(),localbegin.y(),localbegin.z(),overlap.x(),overlap.y(),localsize.z(),true);
		BufferEntities(meshfunction,sendbuffs[NordEast],localgridsize.x()-2*overlap.x(),localbegin.y(),localbegin.z(),overlap.x(),overlap.y(),localsize.z(),true);
		BufferEntities(meshfunction,sendbuffs[SouthWest],localbegin.x(),localgridsize.y()-2*overlap.y(),localbegin.z(),overlap.x(),overlap.y(),localsize.z(),true);
		BufferEntities(meshfunction,sendbuffs[SouthEast],localgridsize.x()-2overlap.x(),localgridsize.y()-2overlap.y(),localbegin.z(),overlap.x(),overlap.y(),localsize.z(),true);
		//XZ
		BufferEntities(meshfunction,sendbuffs[BottomWest],localbegin.x(),localbegin.y(),localbegin.z(),overlap.x(),localsize.y(),overlap.z(),true);
		BufferEntities(meshfunction,sendbuffs[BottomEast],localgridsize.x()-2*overlap.x(),localbegin.y(),localbegin.z(),overlap.x(),localsize.y(),overlap.z(),true);
		BufferEntities(meshfunction,sendbuffs[TopWest],localbegin.x(),localbegin.y(),localgridsize.z()-2*overlap.z(),overlap.x(),localsize.y(),overlap.z(),true);
		BufferEntities(meshfunction,sendbuffs[TopEast],localgridsize.x()-2overlap.x(),localbegin.y(),localgridsize.z()-2overlap.z(),overlap.x(),localsize.y(),overlap.z(),true);
		//YZ
		BufferEntities(meshfunction,sendbuffs[BottomNord],localbegin.x(),localbegin.y(),localbegin.z(),localsize.x(),overlap.y(),overlap.z(),true);
		BufferEntities(meshfunction,sendbuffs[BottomSouth],localbegin.x(),localgridsize.y()-2*overlap.y(),localbegin.z(),localsize.x(),overlap.y(),overlap.z(),true);
		BufferEntities(meshfunction,sendbuffs[TopNord],localbegin.x(),localbegin.y(),localgridsize.z()-2*overlap.z(),localsize.x(),overlap.y(),overlap.z(),true);
		BufferEntities(meshfunction,sendbuffs[TopSouth],localbegin.x(),localgridsize.y()-2overlap.y(),localgridsize.z()-2overlap.z(),localsize.x(),overlap.y(),overlap.z(),true);
		//XYZ
		BufferEntities(meshfunction,sendbuffs[BottomNordWest],localbegin.x(),localbegin.y(),localbegin.z(),overlap.x(),overlap.y(),overlap.z(),true);
		BufferEntities(meshfunction,sendbuffs[BottomNordEast],localgridsize.x()-2*overlap.x(),localbegin.y(),localbegin.z(),overlap.x(),overlap.y(),overlap.z(),true);
		BufferEntities(meshfunction,sendbuffs[BottomSouthWest],localbegin.x(),localgridsize.y()-2*overlap.y(),localbegin.z(),overlap.x(),overlap.y(),overlap.z(),true);
		BufferEntities(meshfunction,sendbuffs[BottomSouthEast],localgridsize.x()-2overlap.x(),localgridsize.y()-2overlap.y(),localbegin.z(),overlap.x(),overlap.y(),overlap.z(),true);
		BufferEntities(meshfunction,sendbuffs[TopNordWest],localbegin.x(),localbegin.y(),localgridsize.z()-2*overlap.z(),overlap.x(),overlap.y(),overlap.z(),true);
		BufferEntities(meshfunction,sendbuffs[TopNordEast],localgridsize.x()-2overlap.x(),localbegin.y(),localgridsize.z()-2overlap.z(),overlap.x(),overlap.y(),overlap.z(),true);
		BufferEntities(meshfunction,sendbuffs[TopSouthWest],localbegin.x(),localgridsize.y()-2overlap.y(),localgridsize.z()-2overlap.z(),overlap.x(),overlap.y(),overlap.z(),true);
		BufferEntities(meshfunction,sendbuffs[TopSouthEast],localgridsize.x()-2overlap.x(),localgridsize.y()-2overlap.y(),localgridsize.z()-2*overlap.z(),overlap.x(),overlap.y(),overlap.z(),true);
		CopyBuffers(meshfunction, sendbuffs, true,
		westSrc, eastSrc, nordSrc, southSrc, bottomSrc, topSrc,
		xcenter, ycenter, zcenter,
		overlap, localsize,
		neighbor);

		//async send
		MPI::Request sendreq[26];
		MPI::Request rcvreq[26];

		int *neighbor=distributedgrid.getNeighbors();

		for(int i=0;i<26;i++)
		if(neighbor[i]!=-1)
		{
		@@ -367,100 +402,107 @@ class DistributedGridSynchronizer<DistributedGridType,MeshFunctionType,3>
		}

		//copy data form rcv buffers
		//fill send buffers
		CopyBuffers(meshfunction, rcvbuffs, false,
		westDst, eastDst, nordDst, southDst, bottomDst, topDst,
		xcenter, ycenter, zcenter,
		overlap, localsize,
		neighbor);

		};

		private:
		template <typename Real>
		void BufferEntities(MeshFunctionType meshfunction, Real * buffer, int beginx, int beginy, int beginz, int sizex, int sizey, int sizez, bool tobuffer)
		{

		typename MeshFunctionType::MeshType::Cell entity(meshfunction.getMesh());
		for(int k=0;k<sizez;k++)
		{
		for(int i=0;i<sizey;i++)
		{
		for(int j=0;j<sizex;j++)
		{
		entity.getCoordinates().x()=beginx+j;
		entity.getCoordinates().y()=beginy+i;
		entity.getCoordinates().z()=beginz+k;
		entity.refresh();
		if(tobuffer)
		buffer[ksizexsizey+i*sizex+j]=meshfunction.getData()[entity.getIndex()];
		else
		meshfunction.getData()[entity.getIndex()]=buffer[ksizexsizey+i*sizex+j];
		}
		}
		}

		};

		template <typename Real>
		void CopyBuffers(MeshFunctionType meshfunction, Real ** buffers, bool toBuffer,
		int west, int east, int nord, int south, int bottom, int top,
		int xcenter, int ycenter, int zcenter,
		CoordinatesType shortDim, CoordinatesType longDim,
		int *neighbor)
		{
		//X-Y-Z
		if(neighbor[West]!=-1)
		BufferEntities(meshfunction,rcvbuffs[West],0,localbegin.y(),localbegin.z(),overlap.x(),localsize.y(),localsize.z(),false);
		BufferEntities(meshfunction,buffers[West],west,ycenter,zcenter,shortDim.x(),longDim.y(),longDim.z(),toBuffer);
		if(neighbor[East]!=-1)
		BufferEntities(meshfunction,rcvbuffs[East],localgridsize.x()-overlap.x(),localbegin.y(),localbegin.z(),overlap.x(),localsize.y(),localsize.z(),false);
		BufferEntities(meshfunction,buffers[East],east,ycenter,zcenter,shortDim.x(),longDim.y(),longDim.z(),toBuffer);
		if(neighbor[Nord]!=-1)
		BufferEntities(meshfunction,rcvbuffs[Nord],localbegin.x(),0,localbegin.z(),localsize.x(),overlap.y(),localsize.z(),false);
		BufferEntities(meshfunction,buffers[Nord],xcenter,nord,zcenter,longDim.x(),shortDim.y(),longDim.z(),toBuffer);
		if(neighbor[South]!=-1)
		BufferEntities(meshfunction,rcvbuffs[South],localbegin.x(),localgridsize.y()-overlap.y(),localbegin.z(),localsize.x(),overlap.y(),localsize.z(),false);
		BufferEntities(meshfunction,buffers[South],xcenter,south,zcenter,longDim.x(),shortDim.y(),longDim.z(),toBuffer);
		if(neighbor[Bottom]!=-1)
		BufferEntities(meshfunction,rcvbuffs[Bottom],localbegin.x(),localbegin.y(),0,localsize.x(),localsize.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[Bottom],xcenter,ycenter,bottom,longDim.x(),longDim.y(),shortDim.z(),toBuffer);
		if(neighbor[Top]!=-1)
		BufferEntities(meshfunction,rcvbuffs[Top],localbegin.x(),localbegin.y(),localgridsize.z()-overlap.z(),localsize.x(),localsize.y(),overlap.z(),false);

		BufferEntities(meshfunction,buffers[Top],xcenter,ycenter,top,longDim.x(),longDim.y(),shortDim.z(),toBuffer);
		//XY
		if(neighbor[NordWest]!=-1)
		BufferEntities(meshfunction,rcvbuffs[NordWest],0,0,localbegin.z(),overlap.x(),overlap.y(),localsize.z(),false);
		BufferEntities(meshfunction,buffers[NordWest],west,nord,zcenter,shortDim.x(),shortDim.y(),longDim.z(),toBuffer);
		if(neighbor[NordEast]!=-1)
		BufferEntities(meshfunction,rcvbuffs[NordEast],localgridsize.x()-overlap.x(),0,localbegin.z(),overlap.x(),overlap.y(),localsize.z(),false);
		BufferEntities(meshfunction,buffers[NordEast],east,nord,zcenter,shortDim.x(),shortDim.y(),longDim.z(),toBuffer);
		if(neighbor[SouthWest]!=-1)
		BufferEntities(meshfunction,rcvbuffs[SouthWest],0,localgridsize.y()-overlap.y(),localbegin.z(),overlap.x(),overlap.y(),localsize.z(),false);
		BufferEntities(meshfunction,buffers[SouthWest],west,south,zcenter,shortDim.x(),shortDim.y(),longDim.z(),toBuffer);
		if(neighbor[SouthEast]!=-1)
		BufferEntities(meshfunction,rcvbuffs[SouthEast],localgridsize.x()-overlap.x(),localgridsize.y()-overlap.y(),localbegin.z(),overlap.x(),overlap.y(),localsize.z(),false);

		BufferEntities(meshfunction,buffers[SouthEast],east,south,zcenter,shortDim.x(),shortDim.y(),longDim.z(),toBuffer);
		//XZ
		if(neighbor[BottomWest]!=-1)
		BufferEntities(meshfunction,rcvbuffs[BottomWest],0,localbegin.y(),0,overlap.x(),localsize.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[BottomWest],west,ycenter,bottom,shortDim.x(),longDim.y(),shortDim.z(),toBuffer);
		if(neighbor[BottomEast]!=-1)
		BufferEntities(meshfunction,rcvbuffs[BottomEast],localgridsize.x()-overlap.x(),localbegin.y(),0,overlap.x(),localsize.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[BottomEast],east,ycenter,bottom,shortDim.x(),longDim.y(),shortDim.z(),toBuffer);
		if(neighbor[TopWest]!=-1)
		BufferEntities(meshfunction,rcvbuffs[TopWest],0,localbegin.y(),localgridsize.z()-overlap.z(),overlap.x(),localsize.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[TopWest],west,ycenter,top,shortDim.x(),longDim.y(),shortDim.z(),toBuffer);
		if(neighbor[TopEast]!=-1)
		BufferEntities(meshfunction,rcvbuffs[TopEast],localgridsize.x()-overlap.x(),localbegin.y(),localgridsize.z()-overlap.z(),overlap.x(),localsize.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[TopEast],east,ycenter,top,shortDim.x(),longDim.y(),shortDim.z(),toBuffer);

		//YZ
		if(neighbor[BottomNord]!=-1)
		BufferEntities(meshfunction,rcvbuffs[BottomNord],localbegin.x(),0,0,localsize.x(),overlap.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[BottomNord],xcenter,nord,bottom,longDim.x(),shortDim.y(),shortDim.z(),toBuffer);
		if(neighbor[BottomSouth]!=-1)
		BufferEntities(meshfunction,rcvbuffs[BottomSouth],localbegin.x(),localgridsize.y()-overlap.y(),0,localsize.x(),overlap.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[BottomSouth],xcenter,south,bottom,longDim.x(),shortDim.y(),shortDim.z(),toBuffer);
		if(neighbor[TopNord]!=-1)
		BufferEntities(meshfunction,rcvbuffs[TopNord],localbegin.x(),0,localgridsize.z()-overlap.z(),localsize.x(),overlap.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[TopNord],xcenter,nord,top,longDim.x(),shortDim.y(),shortDim.z(),toBuffer);
		if(neighbor[TopSouth]!=-1)
		BufferEntities(meshfunction,rcvbuffs[TopSouth],localbegin.x(),localgridsize.y()-overlap.y(),localgridsize.z()-overlap.z(),localsize.x(),overlap.y(),overlap.z(),false);

		BufferEntities(meshfunction,buffers[TopSouth],xcenter,south,top,longDim.x(),shortDim.y(),shortDim.z(),toBuffer);
		//XYZ
		if(neighbor[BottomNordWest]!=-1)
		BufferEntities(meshfunction,rcvbuffs[BottomNordWest],0,0,0,overlap.x(),overlap.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[BottomNordWest],west,nord,bottom,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
		if(neighbor[BottomNordEast]!=-1)
		BufferEntities(meshfunction,rcvbuffs[BottomNordEast],localgridsize.x()-overlap.x(),0,0,overlap.x(),overlap.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[BottomNordEast],east,nord,bottom,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
		if(neighbor[BottomSouthWest]!=-1)
		BufferEntities(meshfunction,rcvbuffs[BottomSouthWest],0,localgridsize.y()-overlap.y(),0,overlap.x(),overlap.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[BottomSouthWest],west,south,bottom,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
		if(neighbor[BottomSouthEast]!=-1)
		BufferEntities(meshfunction,rcvbuffs[BottomSouthEast],localgridsize.x()-overlap.x(),localgridsize.y()-overlap.y(),0,overlap.x(),overlap.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[BottomSouthEast],east,south,bottom,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
		if(neighbor[TopNordWest]!=-1)
		BufferEntities(meshfunction,rcvbuffs[TopNordWest],0,0,localgridsize.z()-overlap.z(),overlap.x(),overlap.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[TopNordWest],west,nord,top,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
		if(neighbor[TopNordEast]!=-1)
		BufferEntities(meshfunction,rcvbuffs[TopNordEast],localgridsize.x()-overlap.x(),0,localgridsize.z()-overlap.z(),overlap.x(),overlap.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[TopNordEast],east,nord,top,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
		if(neighbor[TopSouthWest]!=-1)
		BufferEntities(meshfunction,rcvbuffs[TopSouthWest],0,localgridsize.y()-overlap.y(),localgridsize.z()-overlap.z(),overlap.x(),overlap.y(),overlap.z(),false);
		BufferEntities(meshfunction,buffers[TopSouthWest],west,south,top,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);
		if(neighbor[TopSouthEast]!=-1)
		BufferEntities(meshfunction,rcvbuffs[TopSouthEast],localgridsize.x()-overlap.x(),localgridsize.y()-overlap.y(),localgridsize.z()-overlap.z(),overlap.x(),overlap.y(),overlap.z(),false);

		//free buffers
		for(int i=0;i<26;i++)
		{
		delete [] sendbuffs[i];
		delete [] rcvbuffs[i];
		}

		};

		private:
		template <typename Real>
		static void BufferEntities(MeshFunctionType meshfunction, Real * buffer, int beginx, int beginy, int beginz, int sizex, int sizey, int sizez, bool tobuffer)
		{
		BufferEntities(meshfunction,buffers[TopSouthEast],east,south,top,shortDim.x(),shortDim.y(),shortDim.z(),toBuffer);

		typename MeshFunctionType::MeshType::Cell entity(meshfunction.getMesh());
		for(int k=0;k<sizez;k++)
		{
		for(int i=0;i<sizey;i++)
		{
		for(int j=0;j<sizex;j++)
		{
		entity.getCoordinates().x()=beginx+j;
		entity.getCoordinates().y()=beginy+i;
		entity.getCoordinates().z()=beginz+k;
		entity.refresh();
		if(tobuffer)
		buffer[ksizexsizey+i*sizex+j]=meshfunction.getData()[entity.getIndex()];
		else
		meshfunction.getData()[entity.getIndex()]=buffer[ksizexsizey+i*sizex+j];
		}
		}
		}
		#endif
		};
		};