Commit 5b295f52 authored by Vít Hanousek's avatar Vít Hanousek
Browse files

Merge branch 'mpi-new' into MPI-explosive 

Conflicts:
	src/TNL/Meshes/DistributedMeshes/DistributedGridIO.h
	src/Tools/tnl-init.h
	src/UnitTests/Mpi/DistributedGridIOTest.h
	tests/mpi/CMakeLists.txt
parents 12fbb34d b6b7fd74

CMakeLists.txt

+2 −2
Original line number Diff line number Diff line
@@ -188,7 +188,7 @@ if( WITH_CUDA STREQUAL "yes" ) 
        if( NOT WITH_CUBLAS STREQUAL "no" )

            find_path( CUBLAS_INCLUDE_DIR cublas_v2.h

                       /usr/local/cuda/include

                       ${CUDA_INCLUDE_DIR}

                       ${CUDA_INCLUDE_DIRS}

                       DOC "CUBLAS headers." )

            if( ${CUBLAS_INCLUDE_DIR} STREQUAL "CUBLAS_INCLUDE_DIR-NOTFOUND" )

                message( "CUBLAS not found." )
@@ -219,7 +219,7 @@ if( WITH_CUDA STREQUAL "yes" ) 
        if( NOT WITH_CUSPARSE STREQUAL "no" )

           find_path( CUSPARSE_INCLUDE_DIR cusparse.h

                      /usr/local/cuda/include                   

                      ${CUDA_INCLUDE_DIR}  

                      ${CUDA_INCLUDE_DIRS}  

                      DOC "CUSPARSE headers." )

           if( ${CUSPARSE_INCLUDE_DIR} STREQUAL "CUSPARSE_INCLUDE_DIR-NOTFOUND" )

               message( "CUSPARSE not found." )

src/TNL/Meshes/DistributedMeshes/DistributedGridIO.h

+234 −23
Original line number Diff line number Diff line
@@ -18,11 +18,15 @@ 


#include <iostream>



#ifdef MPIIO

#include <TNL/Communicators/MpiCommunicator.h>

#endif



namespace TNL {

namespace Meshes {   

namespace DistributedMeshes {



enum DistrGridIOTypes { Dummy = 0 , LocalCopy = 1, MPIIO=2 };

enum DistrGridIOTypes { Dummy = 0 , LocalCopy = 1, MpiIO=2 };

    

template<typename MeshFunctionType,

         DistrGridIOTypes type = LocalCopy>
@@ -33,12 +37,12 @@ class DistributedGridIO 
template<typename MeshFunctionType> 

class DistributedGridIO<MeshFunctionType,Dummy>

{

    bool save(File &file,MeshFunctionType meshFunction)

    bool save(const String& fileName, MeshFunctionType &meshFunction)

    {

        return true;

    };

            

    bool load(File &file,MeshFunctionType &meshFunction) 

    bool load(const String& fileName, MeshFunctionType &meshFunction) 

    {

        return true;

    };
@@ -61,13 +65,13 @@ class DistributedGridIO<MeshFunctionType,LocalCopy> 
    typedef typename MeshFunctionType::VectorType VectorType;

    //typedef DistributedGrid< MeshType,MeshFunctionType::getMeshDimension()> DistributedGridType;

    

    static bool save(File &file,File &meshOutputFile, MeshFunctionType &meshFunction)

    static bool save(const String& fileName, MeshFunctionType &meshFunction)

    {

        auto *distrGrid=meshFunction.getMesh().GetDistMesh();

        

        if(distrGrid==NULL) //not distributed

        {

            return meshFunction.save(file);

            return meshFunction.save(fileName);

        }



        MeshType mesh=meshFunction.getMesh();
@@ -83,7 +87,10 @@ class DistributedGridIO<MeshFunctionType,LocalCopy> 
        newMesh->setSpaceSteps(spaceSteps);

        newMesh->setOrigin(origin+TNL::Containers::tnlDotProduct(spaceSteps,localBegin));

        

        newMesh->save( meshOutputFile );

        File meshFile;

        meshFile.open( fileName+String("-mesh-")+distrGrid->printProcessCoords(),IOMode::write);

        newMesh->save( meshFile );

        meshFile.close();



        VectorType newDof(newMesh-> template getEntitiesCount< typename MeshType::Cell >());
@@ -94,16 +101,22 @@ class DistributedGridIO<MeshFunctionType,LocalCopy> 
        zeroCoord.setValue(0);



        CopyEntitiesHelper<MeshFunctionType>::Copy(meshFunction,newMeshFunction,localBegin,zeroCoord,localSize);

        return newMeshFunction.save(file);



        File file;

        file.open( fileName+String("-")+distrGrid->printProcessCoords(), IOMode::write );

        bool ret=newMeshFunction.save(file);

        file.close();



        return ret;

        

    };

            

    static bool load(File &file,MeshFunctionType &meshFunction) 

    static bool load(const String& fileName,MeshFunctionType &meshFunction) 

    {

        auto *distrGrid=meshFunction.getMesh().GetDistMesh();

        if(distrGrid==NULL) //not distributed

        {

            return meshFunction.boundLoad(file);

            return meshFunction.boundLoad(fileName);

        }



        MeshType mesh=meshFunction.getMesh();
@@ -126,7 +139,10 @@ class DistributedGridIO<MeshFunctionType,LocalCopy> 
        CoordinatesType zeroCoord;

        zeroCoord.setValue(0);        



        File file;

        file.open( fileName+String("-")+distrGrid->printProcessCoords(), IOMode::read );

        bool result=newMeshFunction.boundLoad(file);

        file.close();

        CopyEntitiesHelper<MeshFunctionType>::Copy(newMeshFunction,meshFunction,zeroCoord,localBegin,localSize);

        

        return result;
@@ -136,10 +152,13 @@ class DistributedGridIO<MeshFunctionType,LocalCopy> 


/*

 * Save distributed data into single file without overlaps using MPIIO and MPI datatypes, 

 * EXPLOSIVE: works with only Grids

 * EXPLOSIVE: works with only Grids and MPI

 * BAD IMPLEMENTTION creating MPI-Types at every save! -- I dont want contamine more places by MPI..

 */

/*template<typename MeshFunctionType> 

class DistributedGridIO<MeshFunctionType,MPIIO>



#ifdef MPIIO

template<typename MeshFunctionType> 

class DistributedGridIO<MeshFunctionType,MpiIO>

{



    public:
@@ -150,39 +169,231 @@ class DistributedGridIO<MeshFunctionType,MPIIO> 
    typedef typename MeshFunctionType::VectorType VectorType;

    //typedef DistributedGrid< MeshType,MeshFunctionType::getMeshDimension()> DistributedGridType;

    

    static bool save(File &file,File &meshOutputFile, MeshFunctionType &meshFunction)

    static bool save(const String& fileName, MeshFunctionType &meshFunction)

    {

        auto *distrGrid=meshFunction.getMesh().GetDistMesh();

        

        if(distrGrid==NULL) //not distributed

        {

            return meshFunction.save(file);

            return meshFunction.save(fileName);

        }



        int dim=distrGrid.getMeshDimension();

       MPI_Datatype ftype;

       MPI_Datatype atype;

       int dataCount=CreateDataTypes(distrGrid,&ftype,&atype);



        MeshType mesh=meshFunction.getMesh();

       double * data=meshFunction.getData().getData();//TYP



        fgsize[2],flsize[2],fstarts[2];

       //write 

       MPI_File file;

       MPI_File_open(MPI_COMM_WORLD,fileName.getString(),

        	   MPI_MODE_CREATE|MPI_MODE_WRONLY,

               MPI_INFO_NULL, &file);



        return newMeshFunction.save(file);

       int headerSize=0;



       if(Communicators::MpiCommunicator::GetRank()==0)

       {

            headerSize=writeMeshFunctionHeader(file,meshFunction,dataCount);

       }

       MPI_Bcast(&headerSize, 1, MPI_INT,0, MPI_COMM_WORLD);



       MPI_File_set_view(file,headerSize,MPI_DOUBLE,ftype,"native",MPI_INFO_NULL);//TYP

       MPI_Status wstatus;



       MPI_File_write(file,data,1,atype,&wstatus);



       MPI_File_close(&file);



       MPI_Type_free(&atype);

       MPI_Type_free(&ftype);



       return true;

    };



    static bool load(File &file,MeshFunctionType &meshFunction) 

    template<typename DitsributedGridType>

    static int CreateDataTypes(DitsributedGridType *distrGrid,MPI_Datatype *ftype,MPI_Datatype *atype)

    {

        int dim=distrGrid->getMeshDimension();



        int fstarts[dim];

        int flsize[dim];

        int fgsize[dim];

        

        hackArray(dim,fstarts,distrGrid->getGlobalBegin().getData());

        hackArray(dim,flsize,distrGrid->getLocalSize().getData());

        hackArray(dim,fgsize,distrGrid->getGlobalSize().getData());



        MPI_Type_create_subarray(dim,

            fgsize,flsize,fstarts,

            MPI_ORDER_C,MPI_DOUBLE,ftype); //TYP

        MPI_Type_commit(ftype);



       int agsize[dim];

       int alsize[dim];

       int astarts[dim]; 



       hackArray(dim,astarts,distrGrid->getLocalBegin().getData());

       hackArray(dim,alsize,distrGrid->getLocalSize().getData());

       hackArray(dim,agsize,distrGrid->getLocalGridSize().getData());



       MPI_Type_create_subarray(dim,

            agsize,alsize,astarts,

            MPI_ORDER_C,MPI_DOUBLE,atype); //TYP

       MPI_Type_commit(atype);



        int dataCount=1;

        for(int i=0;i<dim;i++)

            dataCount*=fgsize[i];



        return dataCount;



    }



    template<typename Index>

    static void hackArray(int dim, int* out, Index* in)

    {

        if(dim==1)

        {

            out[0]=in[0];

        }



        if(dim==2)

        {

           out[1]=in[0];

           out[0]=in[1];

        }



        if(dim==3)

        {

           out[0]=in[2];

           out[1]=in[1];

           out[2]=in[0];

        }

    }



    static unsigned int writeMeshFunctionHeader(MPI_File file,MeshFunctionType &meshFunction, int length )

    {



        unsigned int size=0;

        int count;

        MPI_Status wstatus;



        //Magic

        MPI_File_write(file,"TNLMN",5,MPI_CHAR,&wstatus);

        MPI_Get_count(&wstatus,MPI_CHAR,&count);

        size+=count*sizeof(char);



        //Meshfunction type

        String meshFunctionSerializationType=meshFunction.getSerializationTypeVirtual();

        int meshFunctionSerializationTypeLength=meshFunctionSerializationType.getLength();

        MPI_File_write(file,&meshFunctionSerializationTypeLength,1,MPI_INT,&wstatus);

        MPI_Get_count(&wstatus,MPI_INT,&count);

        size+=count*sizeof(int);

        MPI_File_write(file,meshFunctionSerializationType.getString(),meshFunctionSerializationType.getLength(),MPI_CHAR,&wstatus);

        MPI_Get_count(&wstatus,MPI_CHAR,&count);

        size+=count*sizeof(char);



        //Magic

        MPI_File_write(file,"TNLMN",5,MPI_CHAR,&wstatus);

        MPI_Get_count(&wstatus,MPI_CHAR,&count);

        size+=count*sizeof(char);

        //Vector Type

        String dataSerializationType=meshFunction.getData().getSerializationTypeVirtual();

        int dataSerializationTypeLength=dataSerializationType.getLength();

        MPI_File_write(file,&dataSerializationTypeLength,1,MPI_INT,&wstatus);

        MPI_Get_count(&wstatus,MPI_INT,&count);

        size+=count*sizeof(int);

        MPI_File_write(file,dataSerializationType.getString(),dataSerializationType.getLength(),MPI_CHAR,&wstatus);

        MPI_Get_count(&wstatus,MPI_CHAR,&count);

        size+=count*sizeof(char);

        //Data count

        MPI_File_write(file,&(length),1,MPI_INT,&wstatus);

        MPI_Get_count(&wstatus,MPI_INT,&count);

        size+=count*sizeof(int);



        return size;

    };

            

    /* Funky bomb - no checks - only dirty load */

    static bool load(const String& fileName,MeshFunctionType &meshFunction) 

    {

        auto *distrGrid=meshFunction.getMesh().GetDistMesh();

        if(distrGrid==NULL) //not distributed

        {

            return meshFunction.boundLoad(file);

            return meshFunction.boundLoad(fileName);

        }



        MeshType mesh=meshFunction.getMesh();

       MPI_Datatype ftype;

       MPI_Datatype atype;

       int dataCount=CreateDataTypes(distrGrid,&ftype,&atype);



       double * data=meshFunction.getData().getData();//TYP



       //write 

       MPI_File file;

       MPI_File_open(MPI_COMM_WORLD,fileName.getString(),

        	   MPI_MODE_RDONLY,

               MPI_INFO_NULL, &file);

       

       int headerSize=0;



       if(Communicators::MpiCommunicator::GetRank()==0)

       {

            headerSize=readMeshFunctionHeader(file,meshFunction,dataCount);

       }

       MPI_Bcast(&headerSize, 1, MPI_INT,0, MPI_COMM_WORLD);

       

       if(headerSize<0)

            return false;



       MPI_File_set_view(file,headerSize,MPI_DOUBLE,ftype,"native",MPI_INFO_NULL);//TYP

       MPI_Status wstatus;

       MPI_File_read(file,(void*)data,1,atype,&wstatus);

       MPI_File_close(&file);

        

       MPI_Type_free(&atype);

       MPI_Type_free(&ftype);



       return true;

    };



    //tak mohlo by to něco kontrolovat...ale nic to nekontroluje

    static int readMeshFunctionHeader(MPI_File file,MeshFunctionType &meshFunction, int length)

    {

        MPI_Status rstatus;

        char buffer[255];

        int size=0;

        int count=0;



        MPI_File_read(file, (void *)buffer,5, MPI_CHAR, &rstatus);//MAGIC

        size+=5*sizeof(char);

        MPI_File_read(file, (void *)&count,1, MPI_INT, &rstatus);//SIZE OF DATATYPE

        size+=1*sizeof(int);

        MPI_File_read(file, (void *)&buffer,count, MPI_CHAR, &rstatus);//DATATYPE

        size+=count*sizeof(char);



        MPI_File_read(file, (void *)buffer,5, MPI_CHAR, &rstatus);//MAGIC

        size+=5*sizeof(char);

        MPI_File_read(file, (void *)&count,1, MPI_INT, &rstatus);//SIZE OF DATATYPE

        size+=1*sizeof(int);

        MPI_File_read(file, (void *)&buffer,count, MPI_CHAR, &rstatus);//DATATYPE

        size+=count*sizeof(char);

        MPI_File_read(file, (void *)&count,1, MPI_INT, &rstatus);//DATACOUNT

        size+=1*sizeof(int);

        

        if(count!=length)

        {

            std::cerr<<"Chyba načítání MeshFunction, délka dat v souboru neodpovídá očekávané délce" << std::endl;

            size=-1;

        }



        return size;

    };

    

};



};*/

#endif





}

src/TNL/Meshes/DistributedMeshes/DistributedGrid_1D.h

+36 −11
Original line number Diff line number Diff line
@@ -36,6 +36,8 @@ class DistributedMesh<Grid< 1, RealType, Device, Index >> 
        CoordinatesType localsize;

        CoordinatesType localgridsize;

        CoordinatesType overlap;

        CoordinatesType globalsize;

        CoordinatesType globalbegin;

        PointType spaceSteps;
@@ -95,6 +97,8 @@ class DistributedMesh<Grid< 1, RealType, Device, Index >> 
               localorigin=globalGrid.getOrigin();

               localsize=globalGrid.getDimensions();

               localgridsize=globalGrid.getDimensions();

               globalsize=globalGrid.getDimensions();

               globalbegin=CoordinatesType(0);

               

               localbegin=CoordinatesType(0);

               return;
@@ -107,6 +111,8 @@ class DistributedMesh<Grid< 1, RealType, Device, Index >> 
               if(rank!=nproc-1)

                   right=rank+1;



               globalsize=globalGrid.getDimensions();                 



               //compute local mesh size               

               numberoflarger=globalGrid.getDimensions().x()%nproc;
@@ -115,12 +121,17 @@ class DistributedMesh<Grid< 1, RealType, Device, Index >> 
                    localsize.x()+=1;                      

                                  

               if(numberoflarger>rank)

               {

                   globalbegin.x()=rank*localsize.x();

                   localorigin.x()=globalGrid.getOrigin().x()

                                +(rank*localsize.x()-overlap.x())*globalGrid.getSpaceSteps().x();

                                +(globalbegin.x()-overlap.x())*globalGrid.getSpaceSteps().x();

               }

               else

                   localorigin.x()=globalGrid.getOrigin().x()

                                +(numberoflarger*(localsize.x()+1)+(rank-numberoflarger)*localsize.x()-overlap.x())

                                *globalGrid.getSpaceSteps().x();

               {

                   globalbegin.x()=numberoflarger*(localsize.x()+1)+(rank-numberoflarger)*localsize.x();

                   localorigin.x()=(globalGrid.getOrigin().x()-overlap.x())

                                +globalbegin.x()*globalGrid.getSpaceSteps().x();

               }

              

              localbegin=overlap;
@@ -151,14 +162,14 @@ class DistributedMesh<Grid< 1, RealType, Device, Index >> 
           grid.SetDistMesh(this);

       };

       

       void printcoords(std::ostream& out)

       String printProcessCoords()

       {

           out<<rank<<":";

           return convertToString(rank);

       };



       void printdistr(std::ostream& out)

       String printProcessDistr()

       {

           out<<"("<<nproc<<"):";

           return convertToString(nproc);

       };       



       bool IsDistributed(void)
@@ -183,17 +194,31 @@ class DistributedMesh<Grid< 1, RealType, Device, Index >> 
           return this->overlap;

       };



       //number of elemnts of local sub domain WITHOUT overlap

       CoordinatesType getLocalSize()

       {

           return this->localsize;

       }



       //number of elements of global grid

       CoordinatesType getGlobalSize()

       {

           return this->globalsize;

       }



       //coordinates of begin of local subdomain without overlaps in global grid

       CoordinatesType getGlobalBegin()

       {

           return this->globalbegin;

       }



       //number of elemnts of local sub domain WITH overlap

       CoordinatesType getLocalGridSize()

       {

           return this->localgridsize;

       }

       

              

       //coordinates of begin of local subdomain without overlaps in local grid       

       CoordinatesType getLocalBegin()

       {

           return this->localbegin;

src/TNL/Meshes/DistributedMeshes/DistributedGrid_2D.h

+32 −17
Original line number Diff line number Diff line
@@ -38,6 +38,8 @@ class DistributedMesh<Grid< 2, RealType, Device, Index >> 
        CoordinatesType localbegin;//souradnice zacatku zpracovavane vypoctove oblasi

        CoordinatesType localgridsize;//velikost lokálního gridu včetně překryvů

        CoordinatesType overlap;

        CoordinatesType globalsize;//velikost celé sítě

        CoordinatesType globalbegin;

        

        

        IndexType Dimensions;
@@ -99,6 +101,8 @@ class DistributedMesh<Grid< 2, RealType, Device, Index >> 
               localorigin=globalGrid.getOrigin();

               localgridsize=globalGrid.getDimensions();

               localsize=globalGrid.getDimensions();

               globalsize=globalGrid.getDimensions();

               globalbegin=CoordinatesType(0);

               localbegin.x()=0;

               localbegin.y()=0;
@@ -119,10 +123,12 @@ class DistributedMesh<Grid< 2, RealType, Device, Index >> 
                  procsdistr[1]=0;

               }

               CommunicatorType::DimsCreate(nproc, 2, procsdistr);



               myproccoord[0]=rank%procsdistr[0];

               myproccoord[1]=rank/procsdistr[0];        



               //compute local mesh size

               globalsize=globalGrid.getDimensions();              

               numberoflarger[0]=globalGrid.getDimensions().x()%procsdistr[0];

               numberoflarger[1]=globalGrid.getDimensions().y()%procsdistr[1];
@@ -135,20 +141,17 @@ class DistributedMesh<Grid< 2, RealType, Device, Index >> 
                   localsize.y()+=1;



               if(numberoflarger[0]>myproccoord[0])

                   localorigin.x()=globalGrid.getOrigin().x()

                                +(myproccoord[0]*localsize.x()-overlap.x())*globalGrid.getSpaceSteps().x();

                   globalbegin.x()=myproccoord[0]*localsize.x();

               else

                   localorigin.x()=globalGrid.getOrigin().x()

                                +(numberoflarger[0]*(localsize.x()+1)+(myproccoord[0]-numberoflarger[0])*localsize.x()-overlap.x())

                                *globalGrid.getSpaceSteps().x();

                   globalbegin.x()=numberoflarger[0]*(localsize.x()+1)+(myproccoord[0]-numberoflarger[0])*localsize.x();



               if(numberoflarger[1]>myproccoord[1])

                   localorigin.y()=globalGrid.getOrigin().y()

                                +(myproccoord[1]*localsize.y()-overlap.y())*globalGrid.getSpaceSteps().y();

                   globalbegin.y()=myproccoord[1]*localsize.y();



               else

                   localorigin.y()=globalGrid.getOrigin().y()

                                +(numberoflarger[1]*(localsize.y()+1)+(myproccoord[1]-numberoflarger[1])*localsize.y()-overlap.y())

                                *globalGrid.getSpaceSteps().y();

                   globalbegin.y()=numberoflarger[1]*(localsize.y()+1)+(myproccoord[1]-numberoflarger[1])*localsize.y();



               localorigin=globalGrid.getOrigin()+TNL::Containers::tnlDotProduct(globalGrid.getSpaceSteps(),globalbegin-overlap);



               //nearnodes

               if(myproccoord[0]>0)
@@ -209,14 +212,14 @@ class DistributedMesh<Grid< 2, RealType, Device, Index >> 
           grid.SetDistMesh(this);

       };

       

       void printcoords(std::ostream& out)

       String printProcessCoords()

       {

           out<<"("<<myproccoord[0]<<","<<myproccoord[1]<<"):";

           return convertToString(myproccoord[0])+String("-")+convertToString(myproccoord[1]);

       };



       void printdistr(std::ostream& out)

       String printProcessDistr()

       {

           out<<"("<<procsdistr[0]<<","<<procsdistr[1]<<"):";

           return convertToString(procsdistr[0])+String("-")+convertToString(procsdistr[1]);

       };  

       

       bool IsDistributed(void)
@@ -240,6 +243,18 @@ class DistributedMesh<Grid< 2, RealType, Device, Index >> 
           return this->localsize;

       }



       //number of elements of global grid

       CoordinatesType getGlobalSize()

       {

           return this->globalsize;

       }



       //coordinates of begin of local subdomain without overlaps in global grid

       CoordinatesType getGlobalBegin()

       {

           return this->globalbegin;

       }



       CoordinatesType getLocalGridSize()

       {

           return this->localgridsize;

src/TNL/Meshes/DistributedMeshes/DistributedGrid_3D.h

+34 −25
Original line number Diff line number Diff line
@@ -46,7 +46,8 @@ class DistributedMesh<Grid< 3, RealType, Device, Index >> 
        CoordinatesType localgridsize;

        CoordinatesType localbegin;

        CoordinatesType overlap;

        

        CoordinatesType globalsize;

        CoordinatesType globalbegin;

        

        IndexType Dimensions;        

        bool isDistributed;
@@ -111,7 +112,9 @@ class DistributedMesh<Grid< 3, RealType, Device, Index >> 
               

               localorigin=globalGrid.getOrigin();

               localsize=globalGrid.getDimensions();

               globalsize=globalGrid.getDimensions();

               localgridsize=localsize;

               globalbegin=CoordinatesType(0);

               return;

           }

           else
@@ -136,6 +139,7 @@ class DistributedMesh<Grid< 3, RealType, Device, Index >> 
               myproccoord[0]=(rank%(procsdistr[0]*procsdistr[1]))%procsdistr[0];



               //compute local mesh size 

               globalsize=globalGrid.getDimensions();                

               numberoflarger[0]=globalGrid.getDimensions().x()%procsdistr[0];

               numberoflarger[1]=globalGrid.getDimensions().y()%procsdistr[1];

               numberoflarger[2]=globalGrid.getDimensions().z()%procsdistr[2];
@@ -152,28 +156,21 @@ class DistributedMesh<Grid< 3, RealType, Device, Index >> 
                   localsize.z()+=1;

                                  

               if(numberoflarger[0]>myproccoord[0])

                   localorigin.x()=globalGrid.getOrigin().x()

                                +(myproccoord[0]*localsize.x()-overlap.x())*globalGrid.getSpaceSteps().x();

                   globalbegin.x()=myproccoord[0]*localsize.x();

               else

                   localorigin.x()=globalGrid.getOrigin().x()

                                +(numberoflarger[0]*(localsize.x()+1)+(myproccoord[0]-numberoflarger[0])*localsize.x()-overlap.x())

                                *globalGrid.getSpaceSteps().x();

                   globalbegin.x()=numberoflarger[0]*(localsize.x()+1)+(myproccoord[0]-numberoflarger[0])*localsize.x();

                 

               if(numberoflarger[1]>myproccoord[1])

                   localorigin.y()=globalGrid.getOrigin().y()

                                +(myproccoord[1]*localsize.y()-overlap.y())*globalGrid.getSpaceSteps().y();

                   globalbegin.y()=myproccoord[1]*localsize.y();

               else

                   localorigin.y()=globalGrid.getOrigin().y()

                                +(numberoflarger[1]*(localsize.y()+1)+(myproccoord[1]-numberoflarger[1])*localsize.y()-overlap.y())

                                *globalGrid.getSpaceSteps().y();

                   globalbegin.y()=numberoflarger[1]*(localsize.y()+1)+(myproccoord[1]-numberoflarger[1])*localsize.y();

 

               if(numberoflarger[2]>myproccoord[2])

                   localorigin.z()=globalGrid.getOrigin().z()

                                +(myproccoord[2]*localsize.z()-overlap.z())*globalGrid.getSpaceSteps().z();

                   globalbegin.z()=myproccoord[2]*localsize.z();

               else

                   localorigin.z()=globalGrid.getOrigin().z()

                                +(numberoflarger[2]*(localsize.z()+1)+(myproccoord[2]-numberoflarger[2])*localsize.z()-overlap.z())

                                *globalGrid.getSpaceSteps().z();

                   globalbegin.z()=numberoflarger[2]*(localsize.z()+1)+(myproccoord[2]-numberoflarger[2])*localsize.z();



               localorigin=globalGrid.getOrigin()+TNL::Containers::tnlDotProduct(globalGrid.getSpaceSteps(),globalbegin-overlap);

               

               //nearnodes

               //X Y Z
@@ -284,14 +281,14 @@ class DistributedMesh<Grid< 3, RealType, Device, Index >> 
           grid.SetDistMesh(this);

       };

       

       void printcoords(std::ostream& out )

       String printProcessCoords()

       {

           out<<"("<<myproccoord[0]<<","<<myproccoord[1]<<","<<myproccoord[2]<<"):";

           return convertToString(myproccoord[0])+String("-")+convertToString(myproccoord[1])+String("-")+convertToString(myproccoord[2]);

       };



       void printdistr(std::ostream& out)

       String printProcessDistr()

       {

           out<<"("<<procsdistr[0]<<","<<procsdistr[1]<<","<<procsdistr[2]<<"):";

           return convertToString(procsdistr[0])+String("-")+convertToString(procsdistr[1])+String("-")+convertToString(procsdistr[2]);

       };  



       bool IsDistributed(void)
@@ -325,6 +322,18 @@ class DistributedMesh<Grid< 3, RealType, Device, Index >> 
           return this->localbegin;

       }



       //number of elements of global grid

       CoordinatesType getGlobalSize()

       {

           return this->globalsize;

       }



       //coordinates of begin of local subdomain without overlaps in global grid

       CoordinatesType getGlobalBegin()

       {

           return this->globalbegin;

       }

       

       private:

           

        int getRankOfProcCoord(int x, int y, int z)