Loading src/TNL/DistributedContainers/DistributedArray.h 0 → 100644 +142 −0 Original line number Diff line number Diff line /*************************************************************************** DistributedArray.h - description ------------------- begin : Sep 6, 2018 copyright : (C) 2018 by Tomas Oberhuber et al. email : tomas.oberhuber@fjfi.cvut.cz ***************************************************************************/ /* See Copyright Notice in tnl/Copyright */ // Implemented by: Jakub Klinkovský #pragma once #include <type_traits> // std::add_const #include <TNL/Containers/Array.h> #include <TNL/Containers/ArrayView.h> #include <TNL/Communicators/MpiCommunicator.h> #include <TNL/DistributedContainers/IndexMap.h> namespace TNL { namespace DistributedContainers { template< typename Value, typename Device = Devices::Host, typename Communicator = Communicators::MpiCommunicator, typename Index = int, typename IndexMap = Subrange< Index > > class DistributedArray : public Object { using CommunicationGroup = typename Communicator::CommunicationGroup; public: using ValueType = Value; using DeviceType = Device; using CommunicatorType = Communicator; using IndexType = Index; using IndexMapType = IndexMap; using LocalArrayType = Containers::Array< Value, Device, Index >; using LocalArrayViewType = Containers::ArrayView< Value, Device, Index >; using ConstLocalArrayViewType = Containers::ArrayView< typename std::add_const< Value >::type, Device, Index >; using HostType = DistributedArray< Value, Devices::Host, Communicator, Index, IndexMap >; using CudaType = DistributedArray< Value, Devices::Cuda, Communicator, Index, IndexMap >; DistributedArray() = default; DistributedArray( DistributedArray& ) = default; DistributedArray( IndexMap indexMap, CommunicationGroup group = Communicator::AllGroup ); void setDistribution( IndexMap indexMap, CommunicationGroup group = Communicator::AllGroup ); const IndexMap& getIndexMap() const; CommunicationGroup getCommunicationGroup() const; // we return only the view so that the user cannot resize it LocalArrayViewType getLocalArrayView(); ConstLocalArrayViewType getLocalArrayView() const; void copyFromGlobal( ConstLocalArrayViewType globalArray ); static String getType(); virtual String getTypeVirtual() const; // TODO: no getSerializationType method until there is support for serialization /* * Usual Array methods follow below. */ template< typename Array > void setLike( const Array& array ); void reset(); // TODO: swap // Returns the *global* size IndexType getSize() const; // Sets all elements of the array to the given value void setValue( ValueType value ); // Safe device-independent element setter void setElement( IndexType i, ValueType value ); // Safe device-independent element getter ValueType getElement( IndexType i ) const; // Unsafe element accessor usable only from the Device __cuda_callable__ ValueType& operator[]( IndexType i ); // Unsafe element accessor usable only from the Device __cuda_callable__ const ValueType& operator[]( IndexType i ) const; // Copy-assignment operator DistributedArray& operator=( const DistributedArray& array ); template< typename Array > DistributedArray& operator=( const Array& array ); // Comparison operators template< typename Array > bool operator==( const Array& array ) const; template< typename Array > bool operator!=( const Array& array ) const; // Checks if there is an element with given value in this array bool containsValue( ValueType value ) const; // Checks if all elements in this array have the same given value bool containsOnlyValue( ValueType value ) const; // Returns true iff non-zero size is set operator bool() const; // TODO: serialization (save, load, boundLoad) protected: IndexMap indexMap; CommunicationGroup group = Communicator::NullGroup; LocalArrayType localData; private: // TODO: disabled until they are implemented using Object::save; using Object::load; using Object::boundLoad; }; } // namespace DistributedContainers } // namespace TNL #include "DistributedArray_impl.h" src/TNL/DistributedContainers/DistributedArray_impl.h 0 → 100644 +373 −0 Original line number Diff line number Diff line /*************************************************************************** DistributedArray_impl.h - description ------------------- begin : Sep 6, 2018 copyright : (C) 2018 by Tomas Oberhuber et al. email : tomas.oberhuber@fjfi.cvut.cz ***************************************************************************/ /* See Copyright Notice in tnl/Copyright */ // Implemented by: Jakub Klinkovský #pragma once #include "DistributedArray.h" #include <TNL/ParallelFor.h> #include <TNL/Communicators/MpiDefs.h> // important only when MPI is disabled namespace TNL { namespace DistributedContainers { template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > DistributedArray< Value, Device, Communicator, Index, IndexMap >:: DistributedArray( IndexMap indexMap, CommunicationGroup group ) { setDistribution( indexMap, group ); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > void DistributedArray< Value, Device, Communicator, Index, IndexMap >:: setDistribution( IndexMap indexMap, CommunicationGroup group ) { this->indexMap = indexMap; this->group = group; if( group != Communicator::NullGroup ) localData.setSize( indexMap.getLocalSize() ); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > const IndexMap& DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getIndexMap() const { return indexMap; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > typename Communicator::CommunicationGroup DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getCommunicationGroup() const { return group; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > typename DistributedArray< Value, Device, Communicator, Index, IndexMap >::LocalArrayViewType DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getLocalArrayView() { return localData; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > typename DistributedArray< Value, Device, Communicator, Index, IndexMap >::ConstLocalArrayViewType DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getLocalArrayView() const { return localData; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > void DistributedArray< Value, Device, Communicator, Index, IndexMap >:: copyFromGlobal( ConstLocalArrayViewType globalArray ) { TNL_ASSERT_EQ( indexMap.getGlobalSize(), globalArray.getSize(), "given global array has different size than the distributed array" ); LocalArrayViewType localView( localData ); const IndexMap indexMap = getIndexMap(); auto kernel = [=] __cuda_callable__ ( IndexType i ) mutable { if( indexMap.isLocal( i ) ) localView[ indexMap.getLocalIndex( i ) ] = globalArray[ i ]; }; ParallelFor< DeviceType >::exec( (IndexType) 0, indexMap.getGlobalSize(), kernel ); } /* * Usual Array methods follow below. */ template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > String DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getType() { return String( "DistributedContainers::DistributedArray< " ) + TNL::getType< Value >() + ", " + Device::getDeviceType() + ", " + // TODO: communicators don't have a getType method "<Communicator>, " + TNL::getType< Index >() + ", " + IndexMap::getType() + " >"; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > String DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getTypeVirtual() const { return getType(); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > template< typename Array > void DistributedArray< Value, Device, Communicator, Index, IndexMap >:: setLike( const Array& array ) { indexMap = array.getIndexMap(); group = array.getCommunicationGroup(); localData.setLike( array.getLocalArrayView() ); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > void DistributedArray< Value, Device, Communicator, Index, IndexMap >:: reset() { indexMap.reset(); group = Communicator::NullGroup; localData.reset(); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > Index DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getSize() const { return indexMap.getGlobalSize(); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > void DistributedArray< Value, Device, Communicator, Index, IndexMap >:: setValue( ValueType value ) { localData.setValue( value ); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > void DistributedArray< Value, Device, Communicator, Index, IndexMap >:: setElement( IndexType i, ValueType value ) { const IndexType li = indexMap.getLocalIndex( i ); localData.setElement( li, value ); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > Value DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getElement( IndexType i ) const { const IndexType li = indexMap.getLocalIndex( i ); return localData.getElement( li ); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > __cuda_callable__ Value& DistributedArray< Value, Device, Communicator, Index, IndexMap >:: operator[]( IndexType i ) { const IndexType li = indexMap.getLocalIndex( i ); return localData[ li ]; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > __cuda_callable__ const Value& DistributedArray< Value, Device, Communicator, Index, IndexMap >:: operator[]( IndexType i ) const { const IndexType li = indexMap.getLocalIndex( i ); return localData[ li ]; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > DistributedArray< Value, Device, Communicator, Index, IndexMap >& DistributedArray< Value, Device, Communicator, Index, IndexMap >:: operator=( const DistributedArray& array ) { setLike( array ); localData = array.getLocalArrayView(); return *this; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > template< typename Array > DistributedArray< Value, Device, Communicator, Index, IndexMap >& DistributedArray< Value, Device, Communicator, Index, IndexMap >:: operator=( const Array& array ) { setLike( array ); localData = array.getLocalArrayView(); return *this; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > template< typename Array > bool DistributedArray< Value, Device, Communicator, Index, IndexMap >:: operator==( const Array& array ) const { // we can't run allreduce if the communication groups are different if( group != array.getCommunicationGroup() ) return false; const bool localResult = indexMap == array.getIndexMap() && localData == array.getLocalArrayView(); bool result = true; if( group != CommunicatorType::NullGroup ) CommunicatorType::Allreduce( &localResult, &result, 1, MPI_LAND, group ); return result; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > template< typename Array > bool DistributedArray< Value, Device, Communicator, Index, IndexMap >:: operator!=( const Array& array ) const { return ! (*this == array); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > bool DistributedArray< Value, Device, Communicator, Index, IndexMap >:: containsValue( ValueType value ) const { bool result = false; if( group != CommunicatorType::NullGroup ) { const bool localResult = localData.containsValue( value ); CommunicatorType::Allreduce( &localResult, &result, 1, MPI_LOR, group ); } return result; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > bool DistributedArray< Value, Device, Communicator, Index, IndexMap >:: containsOnlyValue( ValueType value ) const { bool result = true; if( group != CommunicatorType::NullGroup ) { const bool localResult = localData.containsOnlyValue( value ); CommunicatorType::Allreduce( &localResult, &result, 1, MPI_LAND, group ); } return result; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > DistributedArray< Value, Device, Communicator, Index, IndexMap >:: operator bool() const { return getSize() != 0; } } // namespace DistributedContainers } // namespace TNL src/TNL/DistributedContainers/IndexMap.h 0 → 100644 +130 −0 Original line number Diff line number Diff line /*************************************************************************** IndexMap.h - description ------------------- begin : Sep 6, 2018 copyright : (C) 2018 by Tomas Oberhuber et al. email : tomas.oberhuber@fjfi.cvut.cz ***************************************************************************/ /* See Copyright Notice in tnl/Copyright */ // Implemented by: Jakub Klinkovský #pragma once #include <TNL/Assert.h> #include <TNL/String.h> #include <TNL/param-types.h> namespace TNL { namespace DistributedContainers { // Specifies a subrange [begin, end) of a range [0, gloablSize). template< typename Index > class Subrange { public: using IndexType = Index; __cuda_callable__ Subrange() = default; __cuda_callable__ Subrange( Index begin, Index end, Index globalSize ) { setSubrange( begin, end, globalSize ); } // Sets the local subrange and global range size. __cuda_callable__ void setSubrange( Index begin, Index end, Index globalSize ) { TNL_ASSERT_LE( begin, end, "begin must be before end" ); TNL_ASSERT_GE( begin, 0, "begin must be non-negative" ); TNL_ASSERT_LE( end - begin, globalSize, "end of the subrange is outside of gloabl range" ); offset = begin; localSize = end - begin; this->globalSize = globalSize; } __cuda_callable__ void reset() { offset = 0; localSize = 0; globalSize = 0; } static String getType() { return "Subrange< " + TNL::getType< Index >() + " >"; } // Checks if a global index is in the set of local indices. __cuda_callable__ bool isLocal( Index i ) const { return offset <= i && i < offset + localSize; } // Gets the offset of the subrange. __cuda_callable__ Index getOffset() const { return offset; } // Gets number of local indices. __cuda_callable__ Index getLocalSize() const { return localSize; } // Gets number of global indices. __cuda_callable__ Index getGlobalSize() const { return globalSize; } // Gets local index for given global index. __cuda_callable__ Index getLocalIndex( Index i ) const { TNL_ASSERT_TRUE( isLocal( i ), "Given global index was not found in the local index set." ); return i - offset; } // Gets global index for given local index. __cuda_callable__ Index getGlobalIndex( Index i ) const { TNL_ASSERT_GE( i, 0, "Given local index was not found in the local index set." ); TNL_ASSERT_LT( i, localSize, "Given local index was not found in the local index set." ); return i + offset; } bool operator==( const Subrange& other ) const { return offset == other.offset && localSize == other.localSize && globalSize == other.globalSize; } bool operator!=( const Subrange& other ) const { return ! (*this == other); } protected: Index offset = 0; Index localSize = 0; Index globalSize = 0; }; // TODO: implement a general IndexMap class, e.g. based on collection of subranges as in deal.II: // https://www.dealii.org/8.4.0/doxygen/deal.II/classIndexSet.html } // namespace DistributedContainers } // namespace TNL src/TNL/DistributedContainers/Partitioner.h 0 → 100644 +48 −0 Original line number Diff line number Diff line /*************************************************************************** DistributedArray.h - description ------------------- begin : Sep 6, 2018 copyright : (C) 2018 by Tomas Oberhuber et al. email : tomas.oberhuber@fjfi.cvut.cz ***************************************************************************/ /* See Copyright Notice in tnl/Copyright */ // Implemented by: Jakub Klinkovský #pragma once #include "IndexMap.h" #include <TNL/Math.h> namespace TNL { namespace DistributedContainers { template< typename IndexMap, typename Communicator > class Partitioner {}; template< typename Index, typename Communicator > class Partitioner< Subrange< Index >, Communicator > { using CommunicationGroup = typename Communicator::CommunicationGroup; public: using IndexMap = Subrange< Index >; static IndexMap splitRange( Index globalSize, CommunicationGroup group ) { if( group != Communicator::NullGroup ) { const int rank = Communicator::GetRank( group ); const int partitions = Communicator::GetSize( group ); const Index begin = min( globalSize, rank * globalSize / partitions ); const Index end = min( globalSize, (rank + 1) * globalSize / partitions ); return IndexMap( begin, end, globalSize ); } else return IndexMap( 0, 0, globalSize ); } }; } // namespace DistributedContainers } // namespace TNL src/UnitTests/CMakeLists.txt +1 −0 Original line number Diff line number Diff line ADD_SUBDIRECTORY( Containers ) ADD_SUBDIRECTORY( DistributedContainers ) ADD_SUBDIRECTORY( Functions ) ADD_SUBDIRECTORY( Matrices ) ADD_SUBDIRECTORY( Meshes ) Loading Loading
src/TNL/DistributedContainers/DistributedArray.h 0 → 100644 +142 −0 Original line number Diff line number Diff line /*************************************************************************** DistributedArray.h - description ------------------- begin : Sep 6, 2018 copyright : (C) 2018 by Tomas Oberhuber et al. email : tomas.oberhuber@fjfi.cvut.cz ***************************************************************************/ /* See Copyright Notice in tnl/Copyright */ // Implemented by: Jakub Klinkovský #pragma once #include <type_traits> // std::add_const #include <TNL/Containers/Array.h> #include <TNL/Containers/ArrayView.h> #include <TNL/Communicators/MpiCommunicator.h> #include <TNL/DistributedContainers/IndexMap.h> namespace TNL { namespace DistributedContainers { template< typename Value, typename Device = Devices::Host, typename Communicator = Communicators::MpiCommunicator, typename Index = int, typename IndexMap = Subrange< Index > > class DistributedArray : public Object { using CommunicationGroup = typename Communicator::CommunicationGroup; public: using ValueType = Value; using DeviceType = Device; using CommunicatorType = Communicator; using IndexType = Index; using IndexMapType = IndexMap; using LocalArrayType = Containers::Array< Value, Device, Index >; using LocalArrayViewType = Containers::ArrayView< Value, Device, Index >; using ConstLocalArrayViewType = Containers::ArrayView< typename std::add_const< Value >::type, Device, Index >; using HostType = DistributedArray< Value, Devices::Host, Communicator, Index, IndexMap >; using CudaType = DistributedArray< Value, Devices::Cuda, Communicator, Index, IndexMap >; DistributedArray() = default; DistributedArray( DistributedArray& ) = default; DistributedArray( IndexMap indexMap, CommunicationGroup group = Communicator::AllGroup ); void setDistribution( IndexMap indexMap, CommunicationGroup group = Communicator::AllGroup ); const IndexMap& getIndexMap() const; CommunicationGroup getCommunicationGroup() const; // we return only the view so that the user cannot resize it LocalArrayViewType getLocalArrayView(); ConstLocalArrayViewType getLocalArrayView() const; void copyFromGlobal( ConstLocalArrayViewType globalArray ); static String getType(); virtual String getTypeVirtual() const; // TODO: no getSerializationType method until there is support for serialization /* * Usual Array methods follow below. */ template< typename Array > void setLike( const Array& array ); void reset(); // TODO: swap // Returns the *global* size IndexType getSize() const; // Sets all elements of the array to the given value void setValue( ValueType value ); // Safe device-independent element setter void setElement( IndexType i, ValueType value ); // Safe device-independent element getter ValueType getElement( IndexType i ) const; // Unsafe element accessor usable only from the Device __cuda_callable__ ValueType& operator[]( IndexType i ); // Unsafe element accessor usable only from the Device __cuda_callable__ const ValueType& operator[]( IndexType i ) const; // Copy-assignment operator DistributedArray& operator=( const DistributedArray& array ); template< typename Array > DistributedArray& operator=( const Array& array ); // Comparison operators template< typename Array > bool operator==( const Array& array ) const; template< typename Array > bool operator!=( const Array& array ) const; // Checks if there is an element with given value in this array bool containsValue( ValueType value ) const; // Checks if all elements in this array have the same given value bool containsOnlyValue( ValueType value ) const; // Returns true iff non-zero size is set operator bool() const; // TODO: serialization (save, load, boundLoad) protected: IndexMap indexMap; CommunicationGroup group = Communicator::NullGroup; LocalArrayType localData; private: // TODO: disabled until they are implemented using Object::save; using Object::load; using Object::boundLoad; }; } // namespace DistributedContainers } // namespace TNL #include "DistributedArray_impl.h"
src/TNL/DistributedContainers/DistributedArray_impl.h 0 → 100644 +373 −0 Original line number Diff line number Diff line /*************************************************************************** DistributedArray_impl.h - description ------------------- begin : Sep 6, 2018 copyright : (C) 2018 by Tomas Oberhuber et al. email : tomas.oberhuber@fjfi.cvut.cz ***************************************************************************/ /* See Copyright Notice in tnl/Copyright */ // Implemented by: Jakub Klinkovský #pragma once #include "DistributedArray.h" #include <TNL/ParallelFor.h> #include <TNL/Communicators/MpiDefs.h> // important only when MPI is disabled namespace TNL { namespace DistributedContainers { template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > DistributedArray< Value, Device, Communicator, Index, IndexMap >:: DistributedArray( IndexMap indexMap, CommunicationGroup group ) { setDistribution( indexMap, group ); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > void DistributedArray< Value, Device, Communicator, Index, IndexMap >:: setDistribution( IndexMap indexMap, CommunicationGroup group ) { this->indexMap = indexMap; this->group = group; if( group != Communicator::NullGroup ) localData.setSize( indexMap.getLocalSize() ); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > const IndexMap& DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getIndexMap() const { return indexMap; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > typename Communicator::CommunicationGroup DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getCommunicationGroup() const { return group; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > typename DistributedArray< Value, Device, Communicator, Index, IndexMap >::LocalArrayViewType DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getLocalArrayView() { return localData; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > typename DistributedArray< Value, Device, Communicator, Index, IndexMap >::ConstLocalArrayViewType DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getLocalArrayView() const { return localData; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > void DistributedArray< Value, Device, Communicator, Index, IndexMap >:: copyFromGlobal( ConstLocalArrayViewType globalArray ) { TNL_ASSERT_EQ( indexMap.getGlobalSize(), globalArray.getSize(), "given global array has different size than the distributed array" ); LocalArrayViewType localView( localData ); const IndexMap indexMap = getIndexMap(); auto kernel = [=] __cuda_callable__ ( IndexType i ) mutable { if( indexMap.isLocal( i ) ) localView[ indexMap.getLocalIndex( i ) ] = globalArray[ i ]; }; ParallelFor< DeviceType >::exec( (IndexType) 0, indexMap.getGlobalSize(), kernel ); } /* * Usual Array methods follow below. */ template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > String DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getType() { return String( "DistributedContainers::DistributedArray< " ) + TNL::getType< Value >() + ", " + Device::getDeviceType() + ", " + // TODO: communicators don't have a getType method "<Communicator>, " + TNL::getType< Index >() + ", " + IndexMap::getType() + " >"; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > String DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getTypeVirtual() const { return getType(); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > template< typename Array > void DistributedArray< Value, Device, Communicator, Index, IndexMap >:: setLike( const Array& array ) { indexMap = array.getIndexMap(); group = array.getCommunicationGroup(); localData.setLike( array.getLocalArrayView() ); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > void DistributedArray< Value, Device, Communicator, Index, IndexMap >:: reset() { indexMap.reset(); group = Communicator::NullGroup; localData.reset(); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > Index DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getSize() const { return indexMap.getGlobalSize(); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > void DistributedArray< Value, Device, Communicator, Index, IndexMap >:: setValue( ValueType value ) { localData.setValue( value ); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > void DistributedArray< Value, Device, Communicator, Index, IndexMap >:: setElement( IndexType i, ValueType value ) { const IndexType li = indexMap.getLocalIndex( i ); localData.setElement( li, value ); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > Value DistributedArray< Value, Device, Communicator, Index, IndexMap >:: getElement( IndexType i ) const { const IndexType li = indexMap.getLocalIndex( i ); return localData.getElement( li ); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > __cuda_callable__ Value& DistributedArray< Value, Device, Communicator, Index, IndexMap >:: operator[]( IndexType i ) { const IndexType li = indexMap.getLocalIndex( i ); return localData[ li ]; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > __cuda_callable__ const Value& DistributedArray< Value, Device, Communicator, Index, IndexMap >:: operator[]( IndexType i ) const { const IndexType li = indexMap.getLocalIndex( i ); return localData[ li ]; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > DistributedArray< Value, Device, Communicator, Index, IndexMap >& DistributedArray< Value, Device, Communicator, Index, IndexMap >:: operator=( const DistributedArray& array ) { setLike( array ); localData = array.getLocalArrayView(); return *this; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > template< typename Array > DistributedArray< Value, Device, Communicator, Index, IndexMap >& DistributedArray< Value, Device, Communicator, Index, IndexMap >:: operator=( const Array& array ) { setLike( array ); localData = array.getLocalArrayView(); return *this; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > template< typename Array > bool DistributedArray< Value, Device, Communicator, Index, IndexMap >:: operator==( const Array& array ) const { // we can't run allreduce if the communication groups are different if( group != array.getCommunicationGroup() ) return false; const bool localResult = indexMap == array.getIndexMap() && localData == array.getLocalArrayView(); bool result = true; if( group != CommunicatorType::NullGroup ) CommunicatorType::Allreduce( &localResult, &result, 1, MPI_LAND, group ); return result; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > template< typename Array > bool DistributedArray< Value, Device, Communicator, Index, IndexMap >:: operator!=( const Array& array ) const { return ! (*this == array); } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > bool DistributedArray< Value, Device, Communicator, Index, IndexMap >:: containsValue( ValueType value ) const { bool result = false; if( group != CommunicatorType::NullGroup ) { const bool localResult = localData.containsValue( value ); CommunicatorType::Allreduce( &localResult, &result, 1, MPI_LOR, group ); } return result; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > bool DistributedArray< Value, Device, Communicator, Index, IndexMap >:: containsOnlyValue( ValueType value ) const { bool result = true; if( group != CommunicatorType::NullGroup ) { const bool localResult = localData.containsOnlyValue( value ); CommunicatorType::Allreduce( &localResult, &result, 1, MPI_LAND, group ); } return result; } template< typename Value, typename Device, typename Communicator, typename Index, typename IndexMap > DistributedArray< Value, Device, Communicator, Index, IndexMap >:: operator bool() const { return getSize() != 0; } } // namespace DistributedContainers } // namespace TNL
src/TNL/DistributedContainers/IndexMap.h 0 → 100644 +130 −0 Original line number Diff line number Diff line /*************************************************************************** IndexMap.h - description ------------------- begin : Sep 6, 2018 copyright : (C) 2018 by Tomas Oberhuber et al. email : tomas.oberhuber@fjfi.cvut.cz ***************************************************************************/ /* See Copyright Notice in tnl/Copyright */ // Implemented by: Jakub Klinkovský #pragma once #include <TNL/Assert.h> #include <TNL/String.h> #include <TNL/param-types.h> namespace TNL { namespace DistributedContainers { // Specifies a subrange [begin, end) of a range [0, gloablSize). template< typename Index > class Subrange { public: using IndexType = Index; __cuda_callable__ Subrange() = default; __cuda_callable__ Subrange( Index begin, Index end, Index globalSize ) { setSubrange( begin, end, globalSize ); } // Sets the local subrange and global range size. __cuda_callable__ void setSubrange( Index begin, Index end, Index globalSize ) { TNL_ASSERT_LE( begin, end, "begin must be before end" ); TNL_ASSERT_GE( begin, 0, "begin must be non-negative" ); TNL_ASSERT_LE( end - begin, globalSize, "end of the subrange is outside of gloabl range" ); offset = begin; localSize = end - begin; this->globalSize = globalSize; } __cuda_callable__ void reset() { offset = 0; localSize = 0; globalSize = 0; } static String getType() { return "Subrange< " + TNL::getType< Index >() + " >"; } // Checks if a global index is in the set of local indices. __cuda_callable__ bool isLocal( Index i ) const { return offset <= i && i < offset + localSize; } // Gets the offset of the subrange. __cuda_callable__ Index getOffset() const { return offset; } // Gets number of local indices. __cuda_callable__ Index getLocalSize() const { return localSize; } // Gets number of global indices. __cuda_callable__ Index getGlobalSize() const { return globalSize; } // Gets local index for given global index. __cuda_callable__ Index getLocalIndex( Index i ) const { TNL_ASSERT_TRUE( isLocal( i ), "Given global index was not found in the local index set." ); return i - offset; } // Gets global index for given local index. __cuda_callable__ Index getGlobalIndex( Index i ) const { TNL_ASSERT_GE( i, 0, "Given local index was not found in the local index set." ); TNL_ASSERT_LT( i, localSize, "Given local index was not found in the local index set." ); return i + offset; } bool operator==( const Subrange& other ) const { return offset == other.offset && localSize == other.localSize && globalSize == other.globalSize; } bool operator!=( const Subrange& other ) const { return ! (*this == other); } protected: Index offset = 0; Index localSize = 0; Index globalSize = 0; }; // TODO: implement a general IndexMap class, e.g. based on collection of subranges as in deal.II: // https://www.dealii.org/8.4.0/doxygen/deal.II/classIndexSet.html } // namespace DistributedContainers } // namespace TNL
src/TNL/DistributedContainers/Partitioner.h 0 → 100644 +48 −0 Original line number Diff line number Diff line /*************************************************************************** DistributedArray.h - description ------------------- begin : Sep 6, 2018 copyright : (C) 2018 by Tomas Oberhuber et al. email : tomas.oberhuber@fjfi.cvut.cz ***************************************************************************/ /* See Copyright Notice in tnl/Copyright */ // Implemented by: Jakub Klinkovský #pragma once #include "IndexMap.h" #include <TNL/Math.h> namespace TNL { namespace DistributedContainers { template< typename IndexMap, typename Communicator > class Partitioner {}; template< typename Index, typename Communicator > class Partitioner< Subrange< Index >, Communicator > { using CommunicationGroup = typename Communicator::CommunicationGroup; public: using IndexMap = Subrange< Index >; static IndexMap splitRange( Index globalSize, CommunicationGroup group ) { if( group != Communicator::NullGroup ) { const int rank = Communicator::GetRank( group ); const int partitions = Communicator::GetSize( group ); const Index begin = min( globalSize, rank * globalSize / partitions ); const Index end = min( globalSize, (rank + 1) * globalSize / partitions ); return IndexMap( begin, end, globalSize ); } else return IndexMap( 0, 0, globalSize ); } }; } // namespace DistributedContainers } // namespace TNL
src/UnitTests/CMakeLists.txt +1 −0 Original line number Diff line number Diff line ADD_SUBDIRECTORY( Containers ) ADD_SUBDIRECTORY( DistributedContainers ) ADD_SUBDIRECTORY( Functions ) ADD_SUBDIRECTORY( Matrices ) ADD_SUBDIRECTORY( Meshes ) Loading
