Commit 7a8ba0db authored by Jakub Klinkovský's avatar Jakub Klinkovský
Browse files

Implemented DistributedArrayView and DistributedVectorView

parent fb97bcd0

src/TNL/DistributedContainers/DistributedArrayView.h

0 → 100644
+152 −0
Original line number Diff line number Diff line 
/***************************************************************************

                          DistributedArrayView.h  -  description

                             -------------------

    begin                : Sep 20, 2018

    copyright            : (C) 2018 by Tomas Oberhuber et al.

    email                : tomas.oberhuber@fjfi.cvut.cz

 ***************************************************************************/



/* See Copyright Notice in tnl/Copyright */



// Implemented by: Jakub Klinkovský



#pragma once



#include <TNL/DistributedContainers/DistributedArray.h>



namespace TNL {

namespace DistributedContainers {



template< typename Value,

          typename Device = Devices::Host,

          typename Index = int,

          typename Communicator = Communicators::MpiCommunicator >

class DistributedArrayView

{

   using CommunicationGroup = typename Communicator::CommunicationGroup;

public:

   using ValueType = Value;

   using DeviceType = Device;

   using CommunicatorType = Communicator;

   using IndexType = Index;

   using LocalRangeType = Subrange< Index >;

   using LocalArrayViewType = Containers::ArrayView< Value, Device, Index >;

   using ConstLocalArrayViewType = Containers::ArrayView< typename std::add_const< Value >::type, Device, Index >;

   using HostType = DistributedArrayView< Value, Devices::Host, Index, Communicator >;

   using CudaType = DistributedArrayView< Value, Devices::Cuda, Index, Communicator >;



   __cuda_callable__

   DistributedArrayView() = default;



   // Copy-constructor does shallow copy, so views can be passed-by-value into

   // CUDA kernels and they can be captured-by-value in __cuda_callable__

   // lambda functions.

   __cuda_callable__

   DistributedArrayView( const DistributedArrayView& ) = default;



   // "Templated copy-constructor" accepting any cv-qualification of Value

   template< typename Value_ >

   __cuda_callable__

   DistributedArrayView( const DistributedArrayView< Value_, Device, Index, Communicator >& );



   // default move-constructor

   __cuda_callable__

   DistributedArrayView( DistributedArrayView&& ) = default;



   // initialization from distributed array

   template< typename Value_ >

   DistributedArrayView( DistributedArray< Value_, Device, Index, Communicator >& array );



   // this constructor will be used only when Value is const-qualified

   // (const views are initializable by const references)

   template< typename Value_ >

   DistributedArrayView( const DistributedArray< Value_, Device, Index, Communicator >& array );



   // method for rebinding (reinitialization)

   // Note that you can also bind directly to Array and other types implicitly

   // convertible to ArrayView.

   __cuda_callable__

   void bind( DistributedArrayView view );



   // binding to local array via raw pointer

   // (local range, global size and communication group are preserved)

   template< typename Value_ >

   void bind( Value_* data, IndexType localSize );





   // Copy-assignment does deep copy, just like regular array, but the sizes

   // must match (i.e. copy-assignment cannot resize).

   DistributedArrayView& operator=( const DistributedArrayView& view );



   template< typename Array >

   DistributedArrayView& operator=( const Array& array );





   const LocalRangeType& getLocalRange() const;



   CommunicationGroup getCommunicationGroup() const;



   LocalArrayViewType getLocalArrayView();



   ConstLocalArrayViewType getLocalArrayView() const;



   void copyFromGlobal( ConstLocalArrayViewType globalArray );





   static String getType();





   /*

    * Usual ArrayView methods follow below.

    */

   void reset();



   // TODO: swap



   // Returns the *global* size

   IndexType getSize() const;



   // Sets all elements of the array to the given value

   void setValue( ValueType value );



   // Safe device-independent element setter

   void setElement( IndexType i, ValueType value );



   // Safe device-independent element getter

   ValueType getElement( IndexType i ) const;



   // Unsafe element accessor usable only from the Device

   __cuda_callable__

   ValueType& operator[]( IndexType i );



   // Unsafe element accessor usable only from the Device

   __cuda_callable__

   const ValueType& operator[]( IndexType i ) const;



   // Comparison operators

   template< typename Array >

   bool operator==( const Array& array ) const;



   template< typename Array >

   bool operator!=( const Array& array ) const;



   // Checks if there is an element with given value in this array

   bool containsValue( ValueType value ) const;



   // Checks if all elements in this array have the same given value

   bool containsOnlyValue( ValueType value ) const;



   // Returns true iff non-zero size is set

   operator bool() const;



protected:

   LocalRangeType localRange;

   IndexType globalSize = 0;

   CommunicationGroup group = Communicator::NullGroup;

   LocalArrayViewType localData;

};



} // namespace DistributedContainers

} // namespace TNL



#include "DistributedArrayView_impl.h"

src/TNL/DistributedContainers/DistributedArrayView_impl.h

0 → 100644
+371 −0
Original line number Diff line number Diff line 
/***************************************************************************

                          DistributedArrayView_impl.h  -  description

                             -------------------

    begin                : Sep 20, 2018

    copyright            : (C) 2018 by Tomas Oberhuber et al.

    email                : tomas.oberhuber@fjfi.cvut.cz

 ***************************************************************************/



/* See Copyright Notice in tnl/Copyright */



// Implemented by: Jakub Klinkovský



#pragma once



#include "DistributedArrayView.h"



namespace TNL {

namespace DistributedContainers {



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

   template< typename Value_ >

__cuda_callable__

DistributedArrayView< Value, Device, Index, Communicator >::

DistributedArrayView( const DistributedArrayView< Value_, Device, Index, Communicator >& view )

: localRange( view.getLocalRange() ),

  globalSize( view.getSize() ),

  group( view.getCommunicationGroup() ),

  localData( view.getLocalArrayView() )

{}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

   template< typename Value_ >

DistributedArrayView< Value, Device, Index, Communicator >::

DistributedArrayView( DistributedArray< Value_, Device, Index, Communicator >& array )

: localRange( array.getLocalRange() ),

  globalSize( array.getSize() ),

  group( array.getCommunicationGroup() ),

  localData( array.getLocalArrayView() )

{}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

   template< typename Value_ >

DistributedArrayView< Value, Device, Index, Communicator >::

DistributedArrayView( const DistributedArray< Value_, Device, Index, Communicator >& array )

: localRange( array.getLocalRange() ),

  globalSize( array.getSize() ),

  group( array.getCommunicationGroup() ),

  localData( array.getLocalArrayView() )

{}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

__cuda_callable__

void

DistributedArrayView< Value, Device, Index, Communicator >::

bind( DistributedArrayView view )

{

   localRange = view.getLocalRange();

   globalSize = view.getSize();

   group = view.getCommunicationGroup();

   localData.bind( view.getLocalArrayView() );

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

   template< typename Value_ >

void

DistributedArrayView< Value, Device, Index, Communicator >::

bind( Value_* data, IndexType localSize )

{

   TNL_ASSERT_EQ( localSize, localRange.getSize(),

                  "The local array size does not match the local range of the distributed array." );

   localData.bind( data, localSize );

}





template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

DistributedArrayView< Value, Device, Index, Communicator >&

DistributedArrayView< Value, Device, Index, Communicator >::

operator=( const DistributedArrayView& view )

{

   TNL_ASSERT_EQ( getSize(), view.getSize(), "The sizes of the array views must be equal, views are not resizable." );

   TNL_ASSERT_EQ( getLocalRange(), view.getLocalRange(), "The local ranges must be equal, views are not resizable." );

   TNL_ASSERT_EQ( getCommunicationGroup(), view.getCommunicationGroup(), "The communication groups of the array views must be equal." );

   localData = view.getLocalArrayView();

   return *this;

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

   template< typename Array >

DistributedArrayView< Value, Device, Index, Communicator >&

DistributedArrayView< Value, Device, Index, Communicator >::

operator=( const Array& array )

{

   TNL_ASSERT_EQ( getSize(), array.getSize(), "The global sizes must be equal, views are not resizable." );

   TNL_ASSERT_EQ( getLocalRange(), array.getLocalRange(), "The local ranges must be equal, views are not resizable." );

   TNL_ASSERT_EQ( getCommunicationGroup(), array.getCommunicationGroup(), "The communication groups must be equal." );

   localData = array.getLocalArrayView();

   return *this;

}





template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

const Subrange< Index >&

DistributedArrayView< Value, Device, Index, Communicator >::

getLocalRange() const

{

   return localRange;

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

typename Communicator::CommunicationGroup

DistributedArrayView< Value, Device, Index, Communicator >::

getCommunicationGroup() const

{

   return group;

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

typename DistributedArrayView< Value, Device, Index, Communicator >::LocalArrayViewType

DistributedArrayView< Value, Device, Index, Communicator >::

getLocalArrayView()

{

   return localData;

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

typename DistributedArrayView< Value, Device, Index, Communicator >::ConstLocalArrayViewType

DistributedArrayView< Value, Device, Index, Communicator >::

getLocalArrayView() const

{

   return localData;

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

void

DistributedArrayView< Value, Device, Index, Communicator >::

copyFromGlobal( ConstLocalArrayViewType globalArray )

{

   TNL_ASSERT_EQ( getSize(), globalArray.getSize(),

                  "given global array has different size than the distributed array view" );



   LocalArrayViewType localView( localData );

   const LocalRangeType localRange = getLocalRange();



   auto kernel = [=] __cuda_callable__ ( IndexType i ) mutable

   {

      localView[ i ] = globalArray[ localRange.getGlobalIndex( i ) ];

   };



   ParallelFor< DeviceType >::exec( (IndexType) 0, localRange.getSize(), kernel );

}





template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

String

DistributedArrayView< Value, Device, Index, Communicator >::

getType()

{

   return String( "DistributedContainers::DistributedArrayView< " ) +

          TNL::getType< Value >() + ", " +

          Device::getDeviceType() + ", " +

          TNL::getType< Index >() + ", " +

          // TODO: communicators don't have a getType method

          "<Communicator> >";

}





template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

void

DistributedArrayView< Value, Device, Index, Communicator >::

reset()

{

   localRange.reset();

   globalSize = 0;

   group = Communicator::NullGroup;

   localData.reset();

}



// TODO: swap



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

Index

DistributedArrayView< Value, Device, Index, Communicator >::

getSize() const

{

   return globalSize;

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

void

DistributedArrayView< Value, Device, Index, Communicator >::

setValue( ValueType value )

{

   localData.setValue( value );

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

void

DistributedArrayView< Value, Device, Index, Communicator >::

setElement( IndexType i, ValueType value )

{

   const IndexType li = localRange.getLocalIndex( i );

   localData.setElement( li, value );

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

Value

DistributedArrayView< Value, Device, Index, Communicator >::

getElement( IndexType i ) const

{

   const IndexType li = localRange.getLocalIndex( i );

   return localData.getElement( li );

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

__cuda_callable__

Value&

DistributedArrayView< Value, Device, Index, Communicator >::

operator[]( IndexType i )

{

   const IndexType li = localRange.getLocalIndex( i );

   return localData[ li ];

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

__cuda_callable__

const Value&

DistributedArrayView< Value, Device, Index, Communicator >::

operator[]( IndexType i ) const

{

   const IndexType li = localRange.getLocalIndex( i );

   return localData[ li ];

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

   template< typename Array >

bool

DistributedArrayView< Value, Device, Index, Communicator >::

operator==( const Array& array ) const

{

   // we can't run allreduce if the communication groups are different

   if( group != array.getCommunicationGroup() )

      return false;

   const bool localResult =

         localRange == array.getLocalRange() &&

         globalSize == array.getSize() &&

         localData == array.getLocalArrayView();

   bool result = true;

   if( group != CommunicatorType::NullGroup )

      CommunicatorType::Allreduce( &localResult, &result, 1, MPI_LAND, group );

   return result;

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

   template< typename Array >

bool

DistributedArrayView< Value, Device, Index, Communicator >::

operator!=( const Array& array ) const

{

   return ! (*this == array);

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

bool

DistributedArrayView< Value, Device, Index, Communicator >::

containsValue( ValueType value ) const

{

   bool result = false;

   if( group != CommunicatorType::NullGroup ) {

      const bool localResult = localData.containsValue( value );

      CommunicatorType::Allreduce( &localResult, &result, 1, MPI_LOR, group );

   }

   return result;

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

bool

DistributedArrayView< Value, Device, Index, Communicator >::

containsOnlyValue( ValueType value ) const

{

   bool result = true;

   if( group != CommunicatorType::NullGroup ) {

      const bool localResult = localData.containsOnlyValue( value );

      CommunicatorType::Allreduce( &localResult, &result, 1, MPI_LAND, group );

   }

   return result;

}



template< typename Value,

          typename Device,

          typename Index,

          typename Communicator >

DistributedArrayView< Value, Device, Index, Communicator >::

operator bool() const

{

   return getSize() != 0;

}



} // namespace DistributedContainers

} // namespace TNL

src/TNL/DistributedContainers/DistributedVectorView.h

0 → 100644
+137 −0
Original line number Diff line number Diff line 
/***************************************************************************

                          DistributedVectorView.h  -  description

                             -------------------

    begin                : Sep 20, 2018

    copyright            : (C) 2018 by Tomas Oberhuber et al.

    email                : tomas.oberhuber@fjfi.cvut.cz

 ***************************************************************************/



/* See Copyright Notice in tnl/Copyright */



// Implemented by: Jakub Klinkovský



#pragma once



#include <TNL/DistributedContainers/DistributedArrayView.h>

#include <TNL/Containers/VectorView.h>



namespace TNL {

namespace DistributedContainers {



template< typename Real,

          typename Device = Devices::Host,

          typename Index = int,

          typename Communicator = Communicators::MpiCommunicator >

class DistributedVectorView

: public DistributedArrayView< Real, Device, Index, Communicator >

{

   using CommunicationGroup = typename Communicator::CommunicationGroup;

   using BaseType = DistributedArrayView< Real, Device, Index, Communicator >;

   using NonConstReal = typename std::remove_const< Real >::type;

public:

   using RealType = Real;

   using DeviceType = Device;

   using CommunicatorType = Communicator;

   using IndexType = Index;

   using LocalVectorViewType = Containers::VectorView< Real, Device, Index >;

   using ConstLocalVectorViewType = Containers::VectorView< typename std::add_const< Real >::type, Device, Index >;

   using HostType = DistributedVectorView< Real, Devices::Host, Index, Communicator >;

   using CudaType = DistributedVectorView< Real, Devices::Cuda, Index, Communicator >;



   // inherit all constructors and assignment operators from ArrayView

   using BaseType::DistributedArrayView;

   using BaseType::operator=;



   LocalVectorViewType getLocalVectorView();



   ConstLocalVectorViewType getLocalVectorView() const;





   static String getType();





   /*

    * Usual Vector methods follow below.

    */

   void addElement( IndexType i,

                    RealType value );



   void addElement( IndexType i,

                    RealType value,

                    RealType thisElementMultiplicator );



   template< typename Vector >

   DistributedVectorView& operator-=( const Vector& vector );



   template< typename Vector >

   DistributedVectorView& operator+=( const Vector& vector );



   DistributedVectorView& operator*=( RealType c );



   DistributedVectorView& operator/=( RealType c );



   NonConstReal max() const;



   NonConstReal min() const;



   NonConstReal absMax() const;



   NonConstReal absMin() const;



   template< typename ResultType = NonConstReal, typename Real_ >

   ResultType lpNorm( Real_ p ) const;



   template< typename ResultType = NonConstReal >

   ResultType sum() const;



   template< typename Vector >

   NonConstReal differenceMax( const Vector& v ) const;



   template< typename Vector >

   NonConstReal differenceMin( const Vector& v ) const;



   template< typename Vector >

   NonConstReal differenceAbsMax( const Vector& v ) const;



   template< typename Vector >

   NonConstReal differenceAbsMin( const Vector& v ) const;



   template< typename ResultType = NonConstReal, typename Vector, typename Real_ >

   ResultType differenceLpNorm( const Vector& v, Real_ p ) const;



   template< typename ResultType = NonConstReal, typename Vector >

   ResultType differenceSum( const Vector& v ) const;



   void scalarMultiplication( Real alpha );



   //! Computes scalar dot product

   template< typename Vector >

   NonConstReal scalarProduct( const Vector& v ) const;



   //! Computes this = thisMultiplicator * this + alpha * x.

   template< typename Vector >

   void addVector( const Vector& x,

                   Real alpha = 1.0,

                   Real thisMultiplicator = 1.0 );



   //! Computes this = thisMultiplicator * this + multiplicator1 * v1 + multiplicator2 * v2.

   template< typename Vector >

   void addVectors( const Vector& v1,

                    Real multiplicator1,

                    const Vector& v2,

                    Real multiplicator2,

                    Real thisMultiplicator = 1.0 );



   void computePrefixSum();



   void computePrefixSum( IndexType begin, IndexType end );



   void computeExclusivePrefixSum();



   void computeExclusivePrefixSum( IndexType begin, IndexType end );

};



} // namespace DistributedContainers

} // namespace TNL



#include "DistributedVectorView_impl.h"

src/TNL/DistributedContainers/DistributedVectorView_impl.h

0 → 100644
+555 −0

File added.

Preview size limit exceeded, changes collapsed.

src/UnitTests/DistributedContainers/DistributedVectorTest.h

+97 −0

File changed.

Preview size limit exceeded, changes collapsed.