Commit bdb2b3a4 authored by Jakub Klinkovský's avatar Jakub Klinkovský
Browse files

Moved legacy implementation of DistributedSpMV to the directory of the relevant benchmark

parent aa2ab8b0

src/Benchmarks/DistSpMV/Legacy/DistributedMatrix.h

0 → 100644
+143 −0
Original line number Diff line number Diff line 
/***************************************************************************

                          DistributedMatrix.h  -  description

                             -------------------

    begin                : Sep 10, 2018

    copyright            : (C) 2018 by Tomas Oberhuber et al.

    email                : tomas.oberhuber@fjfi.cvut.cz

 ***************************************************************************/



/* See Copyright Notice in tnl/Copyright */



// Implemented by: Jakub Klinkovský



#pragma once



#include <type_traits>



#include <TNL/Containers/Subrange.h>

#include <TNL/Containers/DistributedVector.h>

#include <TNL/Containers/DistributedVectorView.h>

#include "DistributedSpMV.h"



namespace TNL {

namespace Matrices {

namespace Legacy {



// TODO: 2D distribution for dense matrices (maybe it should be in different template,

//       because e.g. setRowFast doesn't make sense for dense matrices)

template< typename Matrix >

class DistributedMatrix

{

public:

   using MatrixType = Matrix;

   using RealType = typename Matrix::RealType;

   using DeviceType = typename Matrix::DeviceType;

   using IndexType = typename Matrix::IndexType;

   using LocalRangeType = Containers::Subrange< typename Matrix::IndexType >;



   using RowsCapacitiesType = Containers::DistributedVector< IndexType, DeviceType, IndexType >;



   using MatrixRow = typename Matrix::RowView;

   using ConstMatrixRow = typename Matrix::ConstRowView;



   template< typename _Real = RealType,

             typename _Device = DeviceType,

             typename _Index = IndexType >

   using Self = DistributedMatrix< typename MatrixType::template Self< _Real, _Device, _Index > >;



   DistributedMatrix() = default;



   DistributedMatrix( DistributedMatrix& ) = default;



   DistributedMatrix( LocalRangeType localRowRange, IndexType rows, IndexType columns, MPI_Comm communicator );



   void setDistribution( LocalRangeType localRowRange, IndexType rows, IndexType columns, MPI_Comm communicator );



   const LocalRangeType& getLocalRowRange() const;



   MPI_Comm getCommunicator() const;



   const Matrix& getLocalMatrix() const;



   Matrix& getLocalMatrix();





   /*

    * Some common Matrix methods follow below.

    */



   DistributedMatrix& operator=( const DistributedMatrix& matrix );



   template< typename MatrixT >

   DistributedMatrix& operator=( const MatrixT& matrix );



   template< typename MatrixT >

   void setLike( const MatrixT& matrix );



   void reset();



   IndexType getRows() const;



   IndexType getColumns() const;



   template< typename RowCapacitiesVector >

   void setRowCapacities( const RowCapacitiesVector& rowCapacities );



   template< typename Vector >

   void getCompressedRowLengths( Vector& rowLengths ) const;



   IndexType getRowCapacity( IndexType row ) const;



   void setElement( IndexType row,

                    IndexType column,

                    RealType value );



   RealType getElement( IndexType row,

                        IndexType column ) const;



   RealType getElementFast( IndexType row,

                            IndexType column ) const;



   MatrixRow getRow( IndexType row );



   ConstMatrixRow getRow( IndexType row ) const;



   // multiplication with a global vector

   template< typename InVector,

             typename OutVector >

   typename std::enable_if< ! HasGetCommunicatorMethod< InVector >::value >::type

   vectorProduct( const InVector& inVector,

                  OutVector& outVector ) const;



   // Optimization for distributed matrix-vector multiplication

   void updateVectorProductCommunicationPattern();



   // multiplication with a distributed vector

   // (not const because it modifies internal bufers)

   template< typename InVector,

             typename OutVector >

   typename std::enable_if< HasGetCommunicatorMethod< InVector >::value >::type

   vectorProduct( const InVector& inVector,

                  OutVector& outVector ) const;



   // FIXME: does not work for distributed matrices, here only due to common interface

   template< typename Vector1, typename Vector2 >

   bool performSORIteration( const Vector1& b,

                             const IndexType row,

                             Vector2& x,

                             const RealType& omega = 1.0 ) const;



protected:

   LocalRangeType localRowRange;

   IndexType rows = 0;  // global rows count

   MPI_Comm communicator = MPI_COMM_NULL;

   Matrix localMatrix;



   DistributedSpMV< Matrix > spmv;

};



} // namespace Legacy

} // namespace Matrices

} // namespace TNL



#include "DistributedMatrix_impl.h"

src/Benchmarks/DistSpMV/Legacy/DistributedMatrix_impl.h

0 → 100644
+309 −0
Original line number Diff line number Diff line 
/***************************************************************************

                          DistributedMatrix.h  -  description

                             -------------------

    begin                : Sep 10, 2018

    copyright            : (C) 2018 by Tomas Oberhuber et al.

    email                : tomas.oberhuber@fjfi.cvut.cz

 ***************************************************************************/



/* See Copyright Notice in tnl/Copyright */



// Implemented by: Jakub Klinkovský



#pragma once



#include "DistributedMatrix.h"



namespace TNL {

namespace Matrices {

namespace Legacy {



template< typename Matrix >

DistributedMatrix< Matrix >::

DistributedMatrix( LocalRangeType localRowRange, IndexType rows, IndexType columns, MPI_Comm communicator )

{

   setDistribution( localRowRange, rows, columns, communicator );

}



template< typename Matrix >

void

DistributedMatrix< Matrix >::

setDistribution( LocalRangeType localRowRange, IndexType rows, IndexType columns, MPI_Comm communicator )

{

   this->localRowRange = localRowRange;

   this->rows = rows;

   this->communicator = communicator;

   if( communicator != MPI_COMM_NULL )

      localMatrix.setDimensions( localRowRange.getSize(), columns );



   spmv.reset();

}



template< typename Matrix >

const Containers::Subrange< typename Matrix::IndexType >&

DistributedMatrix< Matrix >::

getLocalRowRange() const

{

   return localRowRange;

}



template< typename Matrix >

MPI_Comm

DistributedMatrix< Matrix >::

getCommunicator() const

{

   return communicator;

}



template< typename Matrix >

const Matrix&

DistributedMatrix< Matrix >::

getLocalMatrix() const

{

   return localMatrix;

}



template< typename Matrix >

Matrix&

DistributedMatrix< Matrix >::

getLocalMatrix()

{

   return localMatrix;

}





/*

 * Some common Matrix methods follow below.

 */



template< typename Matrix >

DistributedMatrix< Matrix >&

DistributedMatrix< Matrix >::

operator=( const DistributedMatrix& matrix )

{

   setLike( matrix );

   localMatrix = matrix.getLocalMatrix();

   return *this;

}



template< typename Matrix >

   template< typename MatrixT >

DistributedMatrix< Matrix >&

DistributedMatrix< Matrix >::

operator=( const MatrixT& matrix )

{

   setLike( matrix );

   localMatrix = matrix.getLocalMatrix();

   return *this;

}



template< typename Matrix >

   template< typename MatrixT >

void

DistributedMatrix< Matrix >::

setLike( const MatrixT& matrix )

{

   localRowRange = matrix.getLocalRowRange();

   rows = matrix.getRows();

   communicator = matrix.getCommunicator();

   localMatrix.setLike( matrix.getLocalMatrix() );



   spmv.reset();

}



template< typename Matrix >

void

DistributedMatrix< Matrix >::

reset()

{

   localRowRange.reset();

   rows = 0;

   communicator = MPI_COMM_NULL;

   localMatrix.reset();



   spmv.reset();

}



template< typename Matrix >

typename Matrix::IndexType

DistributedMatrix< Matrix >::

getRows() const

{

   return rows;

}



template< typename Matrix >

typename Matrix::IndexType

DistributedMatrix< Matrix >::

getColumns() const

{

   return localMatrix.getColumns();

}



template< typename Matrix >

   template< typename RowCapacitiesVector >

void

DistributedMatrix< Matrix >::

setRowCapacities( const RowCapacitiesVector& rowCapacities )

{

   TNL_ASSERT_EQ( rowCapacities.getSize(), getRows(), "row lengths vector has wrong size" );

   TNL_ASSERT_EQ( rowCapacities.getLocalRange(), getLocalRowRange(), "row lengths vector has wrong distribution" );

   TNL_ASSERT_EQ( rowCapacities.getCommunicator(), getCommunicator(), "row lengths vector has wrong communicator" );



   if( getCommunicator() != MPI_COMM_NULL ) {

      localMatrix.setRowCapacities( rowCapacities.getConstLocalView() );



      spmv.reset();

   }

}



template< typename Matrix >

   template< typename Vector >

void

DistributedMatrix< Matrix >::

getCompressedRowLengths( Vector& rowLengths ) const

{

   if( getCommunicator() != MPI_COMM_NULL ) {

      rowLengths.setDistribution( getLocalRowRange(), 0, getRows(), getCommunicator() );

      auto localRowLengths = rowLengths.getLocalView();

      localMatrix.getCompressedRowLengths( localRowLengths );

   }

}



template< typename Matrix >

typename Matrix::IndexType

DistributedMatrix< Matrix >::

getRowCapacity( IndexType row ) const

{

   const IndexType localRow = localRowRange.getLocalIndex( row );

   return localMatrix.getRowCapacity( localRow );

}



template< typename Matrix >

void

DistributedMatrix< Matrix >::

setElement( IndexType row,

            IndexType column,

            RealType value )

{

   const IndexType localRow = localRowRange.getLocalIndex( row );

   localMatrix.setElement( localRow, column, value );

}



template< typename Matrix >

typename Matrix::RealType

DistributedMatrix< Matrix >::

getElement( IndexType row,

            IndexType column ) const

{

   const IndexType localRow = localRowRange.getLocalIndex( row );

   return localMatrix.getElement( localRow, column );

}



template< typename Matrix >

typename Matrix::RealType

DistributedMatrix< Matrix >::

getElementFast( IndexType row,

                IndexType column ) const

{

   const IndexType localRow = localRowRange.getLocalIndex( row );

   return localMatrix.getElementFast( localRow, column );

}



template< typename Matrix >

typename DistributedMatrix< Matrix >::MatrixRow

DistributedMatrix< Matrix >::

getRow( IndexType row )

{

   const IndexType localRow = localRowRange.getLocalIndex( row );

   return localMatrix.getRow( localRow );

}



template< typename Matrix >

typename DistributedMatrix< Matrix >::ConstMatrixRow

DistributedMatrix< Matrix >::

getRow( IndexType row ) const

{

   const IndexType localRow = localRowRange.getLocalIndex( row );

   return localMatrix.getRow( localRow );

}



template< typename Matrix >

   template< typename InVector,

             typename OutVector >

typename std::enable_if< ! HasGetCommunicatorMethod< InVector >::value >::type

DistributedMatrix< Matrix >::

vectorProduct( const InVector& inVector,

               OutVector& outVector ) const

{

   TNL_ASSERT_EQ( inVector.getSize(), getColumns(), "input vector has wrong size" );

   TNL_ASSERT_EQ( outVector.getSize(), getRows(), "output vector has wrong size" );

   TNL_ASSERT_EQ( outVector.getLocalRange(), getLocalRowRange(), "output vector has wrong distribution" );

   TNL_ASSERT_EQ( outVector.getCommunicator(), getCommunicator(), "output vector has wrong communicator" );



   auto outView = outVector.getLocalView();

   localMatrix.vectorProduct( inVector, outView );

}



template< typename Matrix >

void

DistributedMatrix< Matrix >::

updateVectorProductCommunicationPattern()

{

   if( getCommunicator() == MPI_COMM_NULL )

      return;

   spmv.updateCommunicationPattern( getLocalMatrix(), getCommunicator() );

}



template< typename Matrix >

   template< typename InVector,

             typename OutVector >

typename std::enable_if< HasGetCommunicatorMethod< InVector >::value >::type

DistributedMatrix< Matrix >::

vectorProduct( const InVector& inVector,

               OutVector& outVector ) const

{

   TNL_ASSERT_EQ( inVector.getLocalRange(), getLocalRowRange(), "input vector has wrong distribution" );

   TNL_ASSERT_EQ( inVector.getCommunicator(), getCommunicator(), "input vector has wrong communicator" );

   TNL_ASSERT_EQ( outVector.getSize(), getRows(), "output vector has wrong size" );

   TNL_ASSERT_EQ( outVector.getLocalRange(), getLocalRowRange(), "output vector has wrong distribution" );

   TNL_ASSERT_EQ( outVector.getCommunicator(), getCommunicator(), "output vector has wrong communicator" );



   if( getCommunicator() == MPI_COMM_NULL )

      return;



   if( inVector.getGhosts() == 0 ) {

      // NOTE: this branch is deprecated and kept only due to existing benchmarks

      TNL_ASSERT_EQ( inVector.getSize(), getColumns(), "input vector has wrong size" );

      const_cast< DistributedMatrix* >( this )->spmv.vectorProduct( outVector, localMatrix, localRowRange, inVector, getCommunicator() );

   }

   else {

      TNL_ASSERT_EQ( inVector.getConstLocalViewWithGhosts().getSize(), localMatrix.getColumns(), "the matrix uses non-local and non-ghost column indices" );

      TNL_ASSERT_EQ( inVector.getGhosts(), localMatrix.getColumns() - localMatrix.getRows(), "input vector has wrong ghosts size" );

      TNL_ASSERT_EQ( outVector.getGhosts(), localMatrix.getColumns() - localMatrix.getRows(), "output vector has wrong ghosts size" );

      TNL_ASSERT_EQ( outVector.getConstLocalView().getSize(), localMatrix.getRows(), "number of local matrix rows does not match the output vector local size" );



      inVector.waitForSynchronization();

      const auto inView = inVector.getConstLocalViewWithGhosts();

      auto outView = outVector.getLocalView();

      localMatrix.vectorProduct( inView, outView );

      // TODO: synchronization is not always necessary, e.g. when a preconditioning step follows

//      outVector.startSynchronization();

   }

}



template< typename Matrix >

   template< typename Vector1, typename Vector2 >

bool

DistributedMatrix< Matrix >::

performSORIteration( const Vector1& b,

                     const IndexType row,

                     Vector2& x,

                     const RealType& omega ) const

{

   return getLocalMatrix().performSORIteration( b, row, x, omega );

}



} // namespace Legacy

} // namespace Matrices

} // namespace TNL

src/TNL/Matrices/DistributedSpMV.h→src/Benchmarks/DistSpMV/Legacy/DistributedSpMV.h

+2 −0
Original line number Diff line number Diff line
@@ -32,6 +32,7 @@ 


namespace TNL {

namespace Matrices {

namespace Legacy {



template< typename Matrix >

class DistributedSpMV
@@ -256,5 +257,6 @@ protected: 
   __DistributedSpMV_impl::ThreePartVector< RealType, DeviceType, IndexType > globalBuffer;

};



} // namespace Legacy

} // namespace Matrices

} // namespace TNL

src/Benchmarks/DistSpMV/tnl-benchmark-distributed-spmv.h

+4 −2
Original line number Diff line number Diff line
@@ -23,7 +23,8 @@ 
#include <TNL/MPI/Config.h>

#include <TNL/Containers/Partitioner.h>

#include <TNL/Containers/DistributedVector.h>

#include <TNL/Matrices/DistributedMatrix.h>

//#include <TNL/Matrices/DistributedMatrix.h>

#include "Legacy/DistributedMatrix.h"

#include <TNL/Matrices/SparseOperations.h>



#include "../Benchmarks.h"
@@ -149,7 +150,8 @@ struct SpmvBenchmark 
   using VectorType = Containers::Vector< RealType, DeviceType, IndexType >;



   using Partitioner = Containers::Partitioner< IndexType >;

   using DistributedMatrix = Matrices::DistributedMatrix< MatrixType >;

//   using DistributedMatrix = Matrices::DistributedMatrix< MatrixType >;

   using DistributedMatrix = Matrices::Legacy::DistributedMatrix< MatrixType >;

   using DistributedVector = Containers::DistributedVector< RealType, DeviceType, IndexType >;

   using DistributedRowLengths = typename DistributedMatrix::RowsCapacitiesType;

src/TNL/Matrices/DistributedMatrix.h

+0 −3
Original line number Diff line number Diff line
@@ -17,7 +17,6 @@ 
#include <TNL/Containers/Subrange.h>

#include <TNL/Containers/DistributedVector.h>

#include <TNL/Containers/DistributedVectorView.h>

#include <TNL/Matrices/DistributedSpMV.h>



namespace TNL {

namespace Matrices {
@@ -131,8 +130,6 @@ protected: 
   IndexType rows = 0;  // global rows count

   MPI_Comm communicator = MPI_COMM_NULL;

   Matrix localMatrix;



   DistributedSpMV< Matrix > spmv;

};



} // namespace Matrices