Commit d5a2b6cd authored by Vít Hanousek's avatar Vít Hanousek
Browse files

Implementation of basic Distributed Grid IO - each node saves/load file from separate file. 

There is shor test of this stuff for 1D 2D and 3D.

Relpaced tabs with 4spaces.
parent af0c2fac

CMakeLists.txt

+1 −1
Original line number Diff line number Diff line
@@ -40,7 +40,7 @@ endif() 


# set Debug/Release options

set( CMAKE_CXX_FLAGS "-std=c++11 -pthread -Wall -Wno-unused-local-typedefs -Wno-unused-variable" )

set( CMAKE_CXX_FLAGS_DEBUG "-g" )

set( CMAKE_CXX_FLAGS_DEBUG "-g -rdynamic" )

set( CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native -mtune=native -DNDEBUG" )

#set( CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native -mtune=native -DNDEBUG -ftree-vectorizer-verbose=1 -ftree-vectorize -fopt-info-vec-missed -funroll-loops" )

# pass -rdynamic only in Debug mode

src/TNL/Assert.h

+7 −1
Original line number Diff line number Diff line
@@ -50,6 +50,7 @@ 
#include <stdio.h>



#include <TNL/Devices/CudaCallable.h>

#include <TNL/Debugging/StackBacktrace.h>



namespace TNL {

namespace Assert {
@@ -88,6 +89,9 @@ printDiagnosticsHost( const char* assertion, 
       << "Function: " << function << "\n"

       << "Line: " << line << "\n"

       << "Diagnostics:\n" << diagnostics << std::endl;



	PrintStackBacktrace;



   throw AssertionError( str.str() );

}
@@ -108,6 +112,8 @@ printDiagnosticsHost( const char* assertion, 
             << "Function: " << function << "\n"

             << "Line: " << line << "\n"

             << "Diagnostics:\n" << diagnostics << std::endl;



	PrintStackBacktrace;

}

#endif // TNL_THROW_ASSERTION_ERROR

src/TNL/Containers/StaticVector.h

+33 −0
Original line number Diff line number Diff line
@@ -460,6 +460,39 @@ Real tnlScalarProduct( const StaticVector< 3, Real >& u, 
   return u[ 0 ] * v[ 0 ] + u[ 1 ] * v[ 1 ] + u[ 2 ] * v[ 2 ];

}



template< typename T1,

          typename T2>

StaticVector<1, T1> tnlDotProduct( const StaticVector< 1, T1 >& u,

                       const StaticVector< 1, T2 >& v )

{

   StaticVector<1, T1> ret;

   ret[0]=u[0]*v[0];

   return ret;

}



template< typename T1,

          typename T2>

StaticVector<2, T1> tnlDotProduct( const StaticVector< 2, T1 >& u,

                       const StaticVector< 2, T2 >& v )

{

   StaticVector<2, T1> ret;

   ret[0]=u[0]*v[0];

   ret[1]=u[1]*v[1];

   return ret;

}



template< typename T1,

          typename T2>

StaticVector<3, T1> tnlDotProduct( const StaticVector< 3, T1 >& u,

                       const StaticVector< 3, T2 >& v )

{

   StaticVector<3, T1> ret;

   ret[0]=u[0]*v[0];

   ret[1]=u[1]*v[1];

   ret[2]=u[2]*v[2];

   return ret;

}



template< typename Real >

Real tnlTriangleArea( const StaticVector< 2, Real >& a,

                      const StaticVector< 2, Real >& b,

src/TNL/Meshes/DistributedGrid.h

+44 −18
Original line number Diff line number Diff line
@@ -80,7 +80,9 @@ class DistributedGrid <GridType,1> 
        

        GridType GlobalGrid;

        PointType localorigin;

        CoordinatesType localbegin;

        CoordinatesType localsize;

        CoordinatesType localgridsize;

        CoordinatesType overlap;
@@ -123,9 +125,13 @@ class DistributedGrid <GridType,1> 
           if(!mpiInUse)

           {

               //Without MPI



               std::cout <<"MEZ MPI"<<std::endl;

               rank=0;

               localorigin=GlobalGrid.getOrigin();

               localsize=GlobalGrid.getDimensions();

               localgridsize=GlobalGrid.getDimensions();

               localbegin=CoordinatesType(0);

               return;

           }

           else
@@ -153,15 +159,21 @@ class DistributedGrid <GridType,1> 
                                +(numberoflarger*(localsize.x()+1)+(rank-numberoflarger)*localsize.x()-overlap.x())

                                *GlobalGrid.getSpaceSteps().x();

              

              localbegin=overlap;

               

               //vlevo neni prekryv

               if(left==-1)

               {

                   localorigin.x()+=overlap.x()*GlobalGrid.getSpaceSteps().x();

                   localbegin.x()=0;

               }

               

               localgridsize=localsize;

               //add overlaps

               if(left==-1||right==-1)

                   localsize.x()+=overlap.x();

                   localgridsize.x()+=overlap.x();

               else

                   localsize.x()+=2*overlap.x();

                   localgridsize.x()+=2*overlap.x();

                         

           }                   

       };
@@ -169,7 +181,7 @@ class DistributedGrid <GridType,1> 
       void SetupGrid( GridType& grid)

       {

           grid.setOrigin(localorigin);

           grid.setDimensions(localsize);

           grid.setDimensions(localgridsize);

           //compute local proporions by sideefect

           grid.setSpaceSteps(GlobalGrid.getSpaceSteps());

           grid.SetDistGrid(this);
@@ -204,6 +216,23 @@ class DistributedGrid <GridType,1> 
       {

           return this->overlap;

       };



       CoordinatesType getLocalSize()

       {

           return this->localsize;

       }



       CoordinatesType getLocalGridSize()

       {

           return this->localgridsize;

       }

       

              

       CoordinatesType getLocalBegin()

       {

           return this->localbegin;

       }

       

};   



//========================2D======================================================
@@ -299,12 +328,10 @@ class DistributedGrid <GridType,2> 
                  procsdistr[1]=0;

               }

               MPI_Dims_create(nproc, 2, procsdistr);

               myproccoord[0]=rank%procsdistr[0]; // CO je X a co Y? --x je 0 a je to sloupec

               myproccoord[0]=rank%procsdistr[0];

               myproccoord[1]=rank/procsdistr[0];        



               

               //compute local mesh size              

               

               numberoflarger[0]=GlobalGrid.getDimensions().x()%procsdistr[0];

               numberoflarger[1]=GlobalGrid.getDimensions().y()%procsdistr[1];
@@ -350,7 +377,6 @@ class DistributedGrid <GridType,2> 
               if(myproccoord[0]<procsdistr[0]-1 && myproccoord[1]<procsdistr[1]-1)

                   neighbors[DownRight]=getRangOfProcCoord(myproccoord[0]+1,myproccoord[1]+1);

               

               

               localbegin=overlap;



               if(neighbors[Left]==-1)
@@ -543,7 +569,7 @@ class DistributedGrid <GridType,3> 
                  procsdistr[2]=0;

               }

               MPI_Dims_create(nproc, 3, procsdistr);

               myproccoord[2]=rank/(procsdistr[0]*procsdistr[1]); // CO je X, Y, Z? x je 0 y je 1 a z je 2, snad... 

               myproccoord[2]=rank/(procsdistr[0]*procsdistr[1]);

               myproccoord[1]=(rank%(procsdistr[0]*procsdistr[1]))/procsdistr[0];

               myproccoord[0]=(rank%(procsdistr[0]*procsdistr[1]))%procsdistr[0];

src/TNL/Meshes/DistributedGridIO.h

0 → 100644
+228 −0
Original line number Diff line number Diff line 
/***************************************************************************

                          DistributedGrid.h  -  description

                             -------------------

    begin                : October 5, 2017

    copyright            : (C) 2017 by Tomas Oberhuber

    email                : tomas.oberhuber@fjfi.cvut.cz

 ***************************************************************************/



/* See Copyright Notice in tnl/Copyright */



#pragma once



#include <TNL/File.h>

#include <TNL/Meshes/DistributedGrid.h>

#include <TNL/Functions/MeshFunction.h>



#include <iostream>



namespace TNL {

namespace Meshes {   





template<typename MeshFunctionType,

         int dim=MeshFunctionType::getMeshDimension()>

class CopyEntities

{

    public:

    typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;

    static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)

    {

    }



};



enum DistrGridIOTypes { Dummy = 0 , LocalCopy = 1 };

    

template<typename MeshFunctionType,

         DistrGridIOTypes type = LocalCopy> 

class DistributedGridIO

{

};



template<typename MeshFunctionType> 

class DistributedGridIO<MeshFunctionType,Dummy>

{

    bool save(File &file,MeshFunctionType meshFunction)

    {

        return true;

    };

            

    bool load(File &file,MeshFunctionType &meshFunction) 

    {

        return true;

    };

};





/*

 * This variant cerate copy of MeshFunction but smaler, reduced to local entites, without overlap. 

 * It slow and has high RAM consupation

 */

template<typename MeshFunctionType> 

class DistributedGridIO<MeshFunctionType,LocalCopy>

{



    public:



    typedef typename MeshFunctionType::MeshType MeshType;

    typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;

    typedef typename MeshFunctionType::MeshType::PointType PointType;

    typedef typename MeshFunctionType::VectorType VectorType;

    //typedef DistributedGrid< MeshType,MeshFunctionType::getMeshDimension()> DistributedGridType;

    

    static bool save(File &file,MeshFunctionType &meshFunction)

    {

        auto *distrGrid=meshFunction.getMesh().GetDistGrid();

        

        if(distrGrid==NULL) //not distributed

        {

            return meshFunction.save(file);

        }



        MeshType mesh=meshFunction.getMesh();

        

        PointType spaceSteps=mesh.getSpaceSteps();

        PointType origin=mesh.getOrigin();

                

        CoordinatesType localSize=distrGrid->getLocalSize();

        CoordinatesType localBegin=distrGrid->getLocalBegin();

 

        SharedPointer<MeshType> newMesh;

        newMesh->setDimensions(localSize);

        newMesh->setSpaceSteps(spaceSteps);

        newMesh->setOrigin(origin+TNL::Containers::tnlDotProduct(spaceSteps,localBegin));

        

        VectorType newDof(newMesh-> template getEntitiesCount< typename MeshType::Cell >());



        MeshFunctionType newMeshFunction;

        newMeshFunction.bind(newMesh,newDof);        



        CoordinatesType zeroCoord;

        zeroCoord.setValue(0);



        CopyEntities<MeshFunctionType> ::Copy(meshFunction,newMeshFunction,localBegin,zeroCoord,localSize);

        return newMeshFunction.save(file);

        

    };

            

    static bool load(File &file,MeshFunctionType &meshFunction) 

    {

        auto *distrGrid=meshFunction.getMesh().GetDistGrid();

        if(distrGrid==NULL) //not distributed

        {

            return meshFunction.boundLoad(file);

        }



        MeshType mesh=meshFunction.getMesh();

        

        PointType spaceSteps=mesh.getSpaceSteps();

        PointType origin=mesh.getOrigin();

                

        CoordinatesType localSize=distrGrid->getLocalSize();

        CoordinatesType localBegin=distrGrid->getLocalBegin();



        SharedPointer<MeshType> newMesh;

        newMesh->setDimensions(localSize);

        newMesh->setSpaceSteps(spaceSteps);

        newMesh->setOrigin(origin+TNL::Containers::tnlDotProduct(spaceSteps,localBegin));

        

        VectorType newDof(newMesh-> template getEntitiesCount< typename MeshType::Cell >());

        MeshFunctionType newMeshFunction;

        newMeshFunction.bind(newMesh,newDof); 



        CoordinatesType zeroCoord;

        zeroCoord.setValue(0);        



        bool result=newMeshFunction.boundLoad(file);

        CopyEntities<MeshFunctionType> ::Copy(newMeshFunction,meshFunction,zeroCoord,localBegin,localSize);

        

        return result;

    };

    

};





//==================================Copy Entities=========================================================

template<typename MeshFunctionType>

class CopyEntities<MeshFunctionType,1>

{

    public:

    typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;

    typedef typename MeshFunctionType::MeshType::Cell Cell;



    static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)

    {        

        Cell fromEntity(from.getMesh());

        Cell toEntity(to.getMesh());

        for(int i=0;i<size.x();i++)

        {

                        toEntity.getCoordinates().x()=toBegin.x()+i;            

                        toEntity.refresh();

                        fromEntity.getCoordinates().x()=fromBegin.x()+i;            

                        fromEntity.refresh();

            to.getData()[toEntity.getIndex()]=from.getData()[fromEntity.getIndex()];

        }

    }



};



template<typename MeshFunctionType>



class CopyEntities<MeshFunctionType,2>

{

    public:

    typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;

    typedef typename MeshFunctionType::MeshType::Cell Cell;



    static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)

    {

        Cell fromEntity(from.getMesh());

        Cell toEntity(to.getMesh());

        for(int j=0;j<size.y();j++)

            for(int i=0;i<size.x();i++)

            {

                toEntity.getCoordinates().x()=toBegin.x()+i;

                toEntity.getCoordinates().y()=toBegin.y()+j;            

                toEntity.refresh();

                fromEntity.getCoordinates().x()=fromBegin.x()+i;

                fromEntity.getCoordinates().y()=fromBegin.y()+j;            

                fromEntity.refresh();

                to.getData()[toEntity.getIndex()]=from.getData()[fromEntity.getIndex()];

            }

    }



};



template<typename MeshFunctionType>

class CopyEntities<MeshFunctionType,3>

{

    public:

    typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;

    typedef typename MeshFunctionType::MeshType::Cell Cell;



    static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)

    {

        Cell fromEntity(from.getMesh());

        Cell toEntity(to.getMesh());

        for(int k=0;k<size.z();k++)

            for(int j=0;j<size.y();j++)

                for(int i=0;i<size.x();i++)

                {

                    toEntity.getCoordinates().x()=toBegin.x()+i;

                                    toEntity.getCoordinates().y()=toBegin.y()+j;

                                    toEntity.getCoordinates().z()=toBegin.z()+k;                                

                    toEntity.refresh();

                    fromEntity.getCoordinates().x()=fromBegin.x()+i;

                    fromEntity.getCoordinates().y()=fromBegin.y()+j;

                    fromEntity.getCoordinates().z()=fromBegin.z()+k;            

                    fromEntity.refresh();

                    to.getData()[toEntity.getIndex()]=from.getData()[fromEntity.getIndex()];

                }



    }



};



}

}