CUDA aware MPI supoort for Distributed Grid file IO (50f55b20) · Commits · TNL / tnl-dev

src/TNL/Meshes/DistributedMeshes/CopyEntitiesHelper.h

0 → 100644

+133 −0

Original line number	Diff line number	Diff line
		/***************************************************************************
		CopyEntitiesHelper.h - description
		-------------------
		begin : March 8, 2018
		copyright : (C) 2018 by Tomas Oberhuber
		email : tomas.oberhuber@fjfi.cvut.cz
		***************************************************************************/

		/* See Copyright Notice in tnl/Copyright */

		#pragma once

		#include <TNL/Devices/Host.h>
		#include <TNL/Devices/Cuda.h>
		#include <TNL/ParallelFor.h>

		namespace TNL {
		namespace Meshes {
		namespace DistributedMeshes {

		template<typename MeshFunctionType,
		int dim=MeshFunctionType::getMeshDimension()>
		class CopyEntitiesHelper
		{
		public:
		typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;
		static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)
		{
		}

		};


		template<typename MeshFunctionType>
		class CopyEntitiesHelper<MeshFunctionType, 1>
		{
		public:
		typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;
		typedef typename MeshFunctionType::MeshType::Cell Cell;

		static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)
		{
		auto toData=to.getData().getData();
		auto fromData=from.getData().getData();
		auto fromMesh=from.getMesh();
		auto toMesh=to.getMesh();
		auto kernel = [fromData,toData, fromMesh, toMesh, fromBegin, toBegin] __cuda_callable__ ( int i )
		{
		Cell fromEntity(fromMesh);
		Cell toEntity(toMesh);
		toEntity.getCoordinates().x()=toBegin.x()+i;
		toEntity.refresh();
		fromEntity.getCoordinates().x()=fromBegin.x()+i;
		fromEntity.refresh();
		toData[toEntity.getIndex()]=fromData[fromEntity.getIndex()];
		};
		ParallelFor< typename MeshFunctionType::MeshType::DeviceType >::exec( 0, size.x(), kernel );

		}

		};


		template<typename MeshFunctionType>

		class CopyEntitiesHelper<MeshFunctionType,2>
		{
		public:
		typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;
		typedef typename MeshFunctionType::MeshType::Cell Cell;

		static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)
		{
		auto toData=to.getData().getData();
		auto fromData=from.getData().getData();
		auto fromMesh=from.getMesh();
		auto toMesh=to.getMesh();
		auto kernel = [fromData,toData, fromMesh, toMesh, fromBegin, toBegin] __cuda_callable__ ( int j, int i )
		{
		Cell fromEntity(fromMesh);
		Cell toEntity(toMesh);
		toEntity.getCoordinates().x()=toBegin.x()+i;
		toEntity.getCoordinates().y()=toBegin.y()+j;
		toEntity.refresh();
		fromEntity.getCoordinates().x()=fromBegin.x()+i;
		fromEntity.getCoordinates().y()=fromBegin.y()+j;
		fromEntity.refresh();
		toData[toEntity.getIndex()]=fromData[fromEntity.getIndex()];
		};
		ParallelFor2D< typename MeshFunctionType::MeshType::DeviceType >::exec( 0,0,size.y(), size.x(), kernel );

		}

		};


		template<typename MeshFunctionType>
		class CopyEntitiesHelper<MeshFunctionType,3>
		{
		public:
		typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;
		typedef typename MeshFunctionType::MeshType::Cell Cell;

		static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)
		{
		auto toData=to.getData().getData();
		auto fromData=from.getData().getData();
		auto fromMesh=from.getMesh();
		auto toMesh=to.getMesh();
		auto kernel = [fromData,toData, fromMesh, toMesh, fromBegin, toBegin] __cuda_callable__ ( int k,int j, int i )
		{
		Cell fromEntity(fromMesh);
		Cell toEntity(toMesh);
		toEntity.getCoordinates().x()=toBegin.x()+i;
		toEntity.getCoordinates().y()=toBegin.y()+j;
		toEntity.getCoordinates().z()=toBegin.z()+k;
		toEntity.refresh();
		fromEntity.getCoordinates().x()=fromBegin.x()+i;
		fromEntity.getCoordinates().y()=fromBegin.y()+j;
		fromEntity.getCoordinates().z()=fromBegin.z()+k;
		fromEntity.refresh();
		toData[toEntity.getIndex()]=fromData[fromEntity.getIndex()];
		};
		ParallelFor3D< typename MeshFunctionType::MeshType::DeviceType >::exec( 0,0,0,size.z() ,size.y(), size.x(), kernel );
		}
		};




		} // namespace DistributedMeshes
		} // namespace Meshes
		} // namespace TNL

src/TNL/Meshes/DistributedMeshes/DistributedGridIO.h

+4 −96

Original line number	Diff line number	Diff line
		@@ -12,26 +12,16 @@

		#include <TNL/File.h>
		#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
		#include <TNL/Meshes/DistributedMeshes/CopyEntitiesHelper.h>
		#include <TNL/Functions/MeshFunction.h>


		#include <iostream>

		namespace TNL {
		namespace Meshes {
		namespace DistributedMeshes {

		template<typename MeshFunctionType,
		int dim=MeshFunctionType::getMeshDimension()>
		class CopyEntities
		{
		public:
		typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;
		static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)
		{
		}

		};

		enum DistrGridIOTypes { Dummy = 0 , LocalCopy = 1 };

		template<typename MeshFunctionType,
		@@ -101,7 +91,7 @@ class DistributedGridIO<MeshFunctionType,LocalCopy>
		CoordinatesType zeroCoord;
		zeroCoord.setValue(0);

		CopyEntities<MeshFunctionType> ::Copy(meshFunction,newMeshFunction,localBegin,zeroCoord,localSize);
		CopyEntitiesHelper<MeshFunctionType>::Copy(meshFunction,newMeshFunction,localBegin,zeroCoord,localSize);
		return newMeshFunction.save(file);

		};
		@@ -135,95 +125,13 @@ class DistributedGridIO<MeshFunctionType,LocalCopy>
		zeroCoord.setValue(0);

		bool result=newMeshFunction.boundLoad(file);
		CopyEntities<MeshFunctionType> ::Copy(newMeshFunction,meshFunction,zeroCoord,localBegin,localSize);
		CopyEntitiesHelper<MeshFunctionType>::Copy(newMeshFunction,meshFunction,zeroCoord,localBegin,localSize);

		return result;
		};

		};


		//==================================Copy Entities=========================================================
		template<typename MeshFunctionType>
		class CopyEntities<MeshFunctionType,1>
		{
		public:
		typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;
		typedef typename MeshFunctionType::MeshType::Cell Cell;

		static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)
		{
		Cell fromEntity(from.getMesh());
		Cell toEntity(to.getMesh());
		for(int i=0;i<size.x();i++)
		{
		toEntity.getCoordinates().x()=toBegin.x()+i;
		toEntity.refresh();
		fromEntity.getCoordinates().x()=fromBegin.x()+i;
		fromEntity.refresh();
		to.getData()[toEntity.getIndex()]=from.getData()[fromEntity.getIndex()];
		}
		}

		};

		template<typename MeshFunctionType>

		class CopyEntities<MeshFunctionType,2>
		{
		public:
		typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;
		typedef typename MeshFunctionType::MeshType::Cell Cell;

		static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)
		{
		Cell fromEntity(from.getMesh());
		Cell toEntity(to.getMesh());
		for(int j=0;j<size.y();j++)
		for(int i=0;i<size.x();i++)
		{
		toEntity.getCoordinates().x()=toBegin.x()+i;
		toEntity.getCoordinates().y()=toBegin.y()+j;
		toEntity.refresh();
		fromEntity.getCoordinates().x()=fromBegin.x()+i;
		fromEntity.getCoordinates().y()=fromBegin.y()+j;
		fromEntity.refresh();
		to.getData()[toEntity.getIndex()]=from.getData()[fromEntity.getIndex()];
		}
		}

		};

		template<typename MeshFunctionType>
		class CopyEntities<MeshFunctionType,3>
		{
		public:
		typedef typename MeshFunctionType::MeshType::CoordinatesType CoordinatesType;
		typedef typename MeshFunctionType::MeshType::Cell Cell;

		static void Copy(MeshFunctionType &from, MeshFunctionType &to, CoordinatesType &fromBegin, CoordinatesType &toBegin, CoordinatesType &size)
		{
		Cell fromEntity(from.getMesh());
		Cell toEntity(to.getMesh());
		for(int k=0;k<size.z();k++)
		for(int j=0;j<size.y();j++)
		for(int i=0;i<size.x();i++)
		{
		toEntity.getCoordinates().x()=toBegin.x()+i;
		toEntity.getCoordinates().y()=toBegin.y()+j;
		toEntity.getCoordinates().z()=toBegin.z()+k;
		toEntity.refresh();
		fromEntity.getCoordinates().x()=fromBegin.x()+i;
		fromEntity.getCoordinates().y()=fromBegin.y()+j;
		fromEntity.getCoordinates().z()=fromBegin.z()+k;
		fromEntity.refresh();
		to.getData()[toEntity.getIndex()]=from.getData()[fromEntity.getIndex()];
		}

		}

		};

		}
		}
		}

src/UnitTests/Mpi/CMakeLists.txt

+7 −0

Original line number	Diff line number	Diff line
		@@ -43,5 +43,12 @@ ADD_TEST( CopyEntitesTest ${EXECUTABLE_OUTPUT_PATH}/CopyEntitesTest )
		SET (mpi_test_parameters_IO -np 4 "${EXECUTABLE_OUTPUT_PATH}/DistributedGridIOTest")
		ADD_TEST( NAME DistributedGridIOTest COMMAND "mpirun" ${mpi_test_parameters_IO})

		IF( BUILD_CUDA )
		CUDA_ADD_EXECUTABLE( GPUDistributedGridIOTest GPUDistributedGridIOTest.cu OPTIONS ${CXX_TESTS_FLAGS})
		TARGET_LINK_LIBRARIES( GPUDistributedGridIOTest
		${GTEST_BOTH_LIBRARIES}
		tnl )
		ENDIF( BUILD_CUDA )

		endif()

src/UnitTests/Mpi/CopyEntitiesTest.cpp

+5 −4

Original line number	Diff line number	Diff line
		@@ -6,9 +6,10 @@
		email : tomas.oberhuber@fjfi.cvut.cz
		***************************************************************************/

		#include <TNL/Meshes/DistributedMeshes/DistributedGridIO.h>
		#include <TNL/Meshes/DistributedMeshes/CopyEntitiesHelper.h>
		#include <TNL/Functions/MeshFunction.h>


		#ifdef HAVE_GTEST
		#include <gtest/gtest.h>

		@@ -199,7 +200,7 @@ class TestCopyEntities
		CoordinatesType size;
		size.setValue(8);

		CopyEntities< MeshFunctionType >::Copy(inputMeshFunction,outputMeshFunction, begin,zero, size);
		CopyEntitiesHelper< MeshFunctionType >::Copy(inputMeshFunction,outputMeshFunction, begin,zero, size);

		TestMovedMeshfunction<MeshFunctionType>::Test(outputMeshFunction);
		};
		@@ -210,7 +211,7 @@ TEST( CopyEntitiesTest, 1D )
		TestCopyEntities<1>::Test();
		}

		TEST( CopyEntitiesTest, 2D )
		/*TEST( CopyEntitiesTest, 2D )
		{
		TestCopyEntities<2>::Test();
		}
		@@ -218,7 +219,7 @@ TEST( CopyEntitiesTest, 2D )
		TEST( CopyEntitiesTest, 3D )
		{
		TestCopyEntities<3>::Test();
		}
		}*/


		#endif

src/UnitTests/Mpi/DistributedGridIOTest.cpp

+10 −372

Original line number	Diff line number	Diff line
		/***************************************************************************
		CopyEntitiesTest.cpp - description
		-------------------
		begin : Nov 1, 2017
		copyright : (C) 2017 by Tomas Oberhuber et al.
		email : tomas.oberhuber@fjfi.cvut.cz
		***************************************************************************/

		#ifdef HAVE_GTEST

		#include <gtest/gtest.h>

		#ifdef HAVE_MPI

		#include <TNL/Communicators/MpiCommunicator.h>
		#include <TNL/Meshes/DistributedMeshes/DistributedMesh.h>
		#include <TNL/Functions/MeshFunction.h>
		#include <TNL/Meshes/DistributedMeshes/DistributedGridIO.h>


		#include "Functions.h"

		using namespace TNL::Containers;
		using namespace TNL::Meshes;
		using namespace TNL::Functions;
		using namespace TNL::Devices;
		using namespace TNL::Communicators;
		using namespace TNL::Meshes::DistributedMeshes;


		//================Parameters===================================
		template <int dim>
		class ParameterProvider
		{
		public:

		typedef Grid<dim,double,Host,int> MeshType;
		typedef typename MeshType::CoordinatesType CoordinatesType;
		typedef typename MeshType::PointType PointType;

		PointType getOrigin(int rank)
		{
		};

		PointType getProportions(int rank)
		{
		};

		int* getDistr(void)
		{
		return NULL;
		};
		};

		template<>
		class ParameterProvider<1>
		{
		public:

		int distr[1];

		typedef Grid<1,double,Host,int> MeshType;
		typedef typename MeshType::CoordinatesType CoordinatesType;
		typedef typename MeshType::PointType PointType;

		PointType getOrigin(int rank)
		{
		if(rank==0)
		return PointType(-0.5);
		if(rank==1)
		return PointType(4.5);
		if(rank==2)
		return PointType(9.5);
		if(rank==3)
		return PointType(14.5);

		return PointType(0);
		};

		PointType getProportions(int rank)
		{
		if(rank==0)
		return PointType(5);
		if(rank==1)
		return PointType(5);
		if(rank==2)
		return PointType(5);
		if(rank==3)
		return PointType(5);
		return PointType(0);
		};

		int* getDistr()
		{
		distr[0]=4;
		return distr;
		};
		};

		template<>
		class ParameterProvider<2>
		{
		public:

		int distr[2];

		typedef Grid<2,double,Host,int> MeshType;
		typedef typename MeshType::CoordinatesType CoordinatesType;
		typedef typename MeshType::PointType PointType;

		PointType getOrigin(int rank)
		{
		if(rank==0)
		return PointType(-0.5,-0.5);
		if(rank==1)
		return PointType(9.5,-0.5);
		if(rank==2)
		return PointType(-0.5,9.5);
		if(rank==3)
		return PointType(9.5,9.5);

		return PointType(0,0);
		};

		PointType getProportions(int rank)
		{
		if(rank==0)
		return PointType(10,10);
		if(rank==1)
		return PointType(10,10);
		if(rank==2)
		return PointType(10,10);
		if(rank==3)
		return PointType(10,10);
		return PointType(0,0);
		};

		int* getDistr()
		{
		distr[0]=2;
		distr[1]=2;
		return distr;
		};
		};

		template<>
		class ParameterProvider<3>
		{
		public:

		int distr[3];

		typedef Grid<3,double,Host,int> MeshType;
		typedef typename MeshType::CoordinatesType CoordinatesType;
		typedef typename MeshType::PointType PointType;

		PointType getOrigin(int rank)
		{
		if(rank==0)
		return PointType(-0.5,-0.5,-0.5);
		if(rank==1)
		return PointType(9.5,-0.5,-0.5);
		if(rank==2)
		return PointType(-0.5,9.5,-0.5);
		if(rank==3)
		return PointType(9.5,9.5,-0.5);

		return PointType(0,0,0);
		};

		PointType getProportions(int rank)
		{
		if(rank==0)
		return PointType(10,10,20);
		if(rank==1)
		return PointType(10,10,20);
		if(rank==2)
		return PointType(10,10,20);
		if(rank==3)
		return PointType(10,10,20);
		return PointType(0,0,0);
		};

		int* getDistr()
		{
		distr[0]=2;
		distr[1]=2;
		distr[2]=1;
		return distr;
		};
		};

		//------------------------------------------------------------------------------

		typedef MpiCommunicator CommunicatorType;

		template <int dim>
		class TestDistributedGridIO{
		public:

		typedef Grid<dim,double,Host,int> MeshType;
		typedef MeshFunction<MeshType> MeshFunctionType;
		typedef Vector<double,Host,int> DofType;
		typedef typename MeshType::Cell Cell;
		typedef typename MeshType::IndexType IndexType;
		typedef typename MeshType::PointType PointType;
		typedef DistributedMesh<MeshType> DistributedGridType;

		typedef typename DistributedGridType::CoordinatesType CoordinatesType;
		typedef LinearFunction<double,dim> LinearFunctionType;

		static void TestSave()
		{
		SharedPointer< LinearFunctionType, Host > linearFunctionPtr;
		MeshFunctionEvaluator< MeshFunctionType, LinearFunctionType > linearFunctionEvaluator;

		ParameterProvider<dim> parametry;

		//save distributed meshfunction into files
		PointType globalOrigin;
		globalOrigin.setValue(-0.5);

		PointType globalProportions;
		globalProportions.setValue(20);


		MeshType globalGrid;
		globalGrid.setDimensions(globalProportions);
		globalGrid.setDomain(globalOrigin,globalProportions);

		int *distr=parametry.getDistr();

		CoordinatesType overlap;
		overlap.setValue(1);
		DistributedGridType distrgrid;
		distrgrid.template setGlobalGrid<CommunicatorType>(globalGrid,overlap);

		SharedPointer<MeshType> gridptr;
		SharedPointer<MeshFunctionType> meshFunctionptr;
		distrgrid.SetupGrid(*gridptr);

		DofType dof(gridptr->template getEntitiesCount< Cell >());
		dof.setValue(0);
		meshFunctionptr->bind(gridptr,dof);

		linearFunctionEvaluator.evaluateAllEntities(meshFunctionptr , linearFunctionPtr);

		File file;
		file.open( String( "/tmp/test-file.tnl-" )+convertToString(CommunicatorType::GetRank()), IOMode::write );

		DistributedGridIO<MeshFunctionType> ::save(file, *meshFunctionptr );

		file.close();

		//create similar local mesh function and evaluate linear function on it
		PointType localOrigin=parametry.getOrigin(CommunicatorType::GetRank());
		PointType localProportions=parametry.getProportions(CommunicatorType::GetRank());;

		SharedPointer<MeshType> localGridptr;
		localGridptr->setDimensions(localProportions);
		localGridptr->setDomain(localOrigin,localProportions);

		DofType localDof(localGridptr->template getEntitiesCount< Cell >());

		SharedPointer<MeshFunctionType> localMeshFunctionptr;
		localMeshFunctionptr->bind(localGridptr,localDof);
		linearFunctionEvaluator.evaluateAllEntities(localMeshFunctionptr , linearFunctionPtr);

		//load other meshfunction on same localgrid from created file
		SharedPointer<MeshType> loadGridptr;
		loadGridptr->setDimensions(localProportions);
		loadGridptr->setDomain(localOrigin,localProportions);

		DofType loadDof(localGridptr->template getEntitiesCount< Cell >());
		SharedPointer<MeshFunctionType> loadMeshFunctionptr;
		loadMeshFunctionptr->bind(loadGridptr,loadDof);

		loadDof.setValue(-1);


		file.open( String( "/tmp/test-file.tnl-" )+convertToString(CommunicatorType::GetRank()), IOMode::read );
		loadMeshFunctionptr->boundLoad(file);
		file.close();

		for(int i=0;i<localDof.getSize();i++)
		{
		EXPECT_EQ( localDof[i], loadDof[i]) << "Compare Loaded and evaluated Dof Failed for: "<< i;
		}

		}

		static void TestLoad()
		{
		SharedPointer< LinearFunctionType, Host > linearFunctionPtr;
		MeshFunctionEvaluator< MeshFunctionType, LinearFunctionType > linearFunctionEvaluator;

		ParameterProvider<dim> parametry;

		//save files from local mesh
		PointType localOrigin=parametry.getOrigin(CommunicatorType::GetRank());
		PointType localProportions=parametry.getProportions(CommunicatorType::GetRank());;

		SharedPointer<MeshType> localGridptr;
		localGridptr->setDimensions(localProportions);
		localGridptr->setDomain(localOrigin,localProportions);

		DofType localDof(localGridptr->template getEntitiesCount< Cell >());

		SharedPointer<MeshFunctionType> localMeshFunctionptr;
		localMeshFunctionptr->bind(localGridptr,localDof);
		linearFunctionEvaluator.evaluateAllEntities(localMeshFunctionptr , linearFunctionPtr);

		File file;
		file.open( String( "/tmp/test-file.tnl-" )+convertToString(CommunicatorType::GetRank()), IOMode::write );
		localMeshFunctionptr->save(file);
		file.close();

		//Crete distributed grid
		PointType globalOrigin;
		globalOrigin.setValue(-0.5);

		PointType globalProportions;
		globalProportions.setValue(20);

		MeshType globalGrid;
		globalGrid.setDimensions(globalProportions);
		globalGrid.setDomain(globalOrigin,globalProportions);

		int *distr=parametry.getDistr();

		CoordinatesType overlap;
		overlap.setValue(1);
		DistributedGridType distrgrid;
		distrgrid.template setGlobalGrid<CommunicatorType>(globalGrid,overlap, distr);

		//Crete "distributedgrid driven" grid filed by load
		SharedPointer<MeshType> loadGridptr;
		SharedPointer<MeshFunctionType> loadMeshFunctionptr;
		distrgrid.SetupGrid(*loadGridptr);

		DofType loadDof(loadGridptr->template getEntitiesCount< Cell >());
		loadDof.setValue(0);
		loadMeshFunctionptr->bind(loadGridptr,loadDof);


		file.open( String( "/tmp/test-file.tnl-" )+convertToString(CommunicatorType::GetRank()), IOMode::read );
		DistributedGridIO<MeshFunctionType> ::load(file, *loadMeshFunctionptr );
		file.close();

		loadMeshFunctionptr->template Synchronize<CommunicatorType>(); //need synchronization for overlaps to be filled corectly in loadDof


		//Crete "distributedgrid driven" grid filed by evaluated linear function
		SharedPointer<MeshType> gridptr;
		SharedPointer<MeshFunctionType> meshFunctionptr;
		distrgrid.SetupGrid(*gridptr);

		DofType dof(gridptr->template getEntitiesCount< Cell >());
		dof.setValue(-1);
		meshFunctionptr->bind(gridptr,dof);

		linearFunctionEvaluator.evaluateAllEntities(meshFunctionptr , linearFunctionPtr);
		meshFunctionptr->template Synchronize<CommunicatorType>();

		for(int i=0;i<localDof.getSize();i++)
		{
		EXPECT_EQ( dof[i], loadDof[i]) << "Compare Loaded and evaluated Dof Failed for: "<< i;
		}
		}
		};

		#include "DistributedGridIOTest.h"

		TEST( DistributedGridIO, Save_1D )
		{
		TestDistributedGridIO<1>::TestSave();
		TestDistributedGridIO<1,Host>::TestSave();
		}

		TEST( DistributedGridIO, Save_2D )
		{
		TestDistributedGridIO<2>::TestSave();
		TestDistributedGridIO<2,Host>::TestSave();
		}

		TEST( DistributedGridIO, Save_3D )
		{
		TestDistributedGridIO<3>::TestSave();
		TestDistributedGridIO<3,Host>::TestSave();
		}

		TEST( DistributedGridIO, Load_1D )
		{
		TestDistributedGridIO<1>::TestLoad();
		TestDistributedGridIO<1,Host>::TestLoad();
		}

		TEST( DistributedGridIO, Load_2D )
		{
		TestDistributedGridIO<2>::TestLoad();
		TestDistributedGridIO<2,Host>::TestLoad();
		}

		TEST( DistributedGridIO, Load_3D )
		{
		TestDistributedGridIO<3>::TestLoad();
		TestDistributedGridIO<3,Host>::TestLoad();
		}

		#else