Amesos2_EpetraMultiVecAdapter_decl.hpp

Go to the documentation of this file.

// @HEADER
// *****************************************************************************
//           Amesos2: Templated Direct Sparse Solver Package
//
// Copyright 2011 NTESS and the Amesos2 contributors.
// SPDX-License-Identifier: BSD-3-Clause
// *****************************************************************************
// @HEADER

#ifndef AMESOS2_EPETRA_MULTIVEC_ADAPTER_DECL_HPP
#define AMESOS2_EPETRA_MULTIVEC_ADAPTER_DECL_HPP

#include <Teuchos_RCP.hpp>
#include <Teuchos_Array.hpp>
#include <Teuchos_as.hpp>

#include <Tpetra_Vector.hpp>
#include <Tpetra_Map.hpp>

#include <Epetra_MultiVector.h>

#include "Amesos2_MultiVecAdapter_decl.hpp"
#include "Amesos2_Kokkos_View_Copy_Assign.hpp"

namespace Amesos2 {

  template <>
  class MultiVecAdapter<Epetra_MultiVector>
  {
  public:

    // public type definitions
    typedef double                                                scalar_t;
    typedef int                                            local_ordinal_t;
    typedef Tpetra::Map<>::global_ordinal_type            global_ordinal_t;
    typedef size_t                                           global_size_t;
    typedef Tpetra::Map<>::node_type                                node_t;
    typedef Epetra_MultiVector                                  multivec_t;

    friend Teuchos::RCP<MultiVecAdapter<multivec_t> > createMultiVecAdapter<>(Teuchos::RCP<multivec_t>);
    friend Teuchos::RCP<const MultiVecAdapter<multivec_t> > createConstMultiVecAdapter<>(Teuchos::RCP<const multivec_t>);


    static const char* name;


  protected:
    MultiVecAdapter( const MultiVecAdapter<multivec_t>& adapter );

    MultiVecAdapter( const Teuchos::RCP<multivec_t>& m );


  public:

    ~MultiVecAdapter() = default;


    bool isLocallyIndexed() const;

    bool isGloballyIndexed() const;


    Teuchos::RCP<Epetra_MultiVector> clone() const;

    Teuchos::RCP<const Tpetra::Map<local_ordinal_t, global_ordinal_t, node_t> >
    getMap() const;

    const Teuchos::RCP<const Teuchos::Comm<int> > getComm() const;


    size_t getLocalLength() const;


    size_t getLocalNumVectors() const;


    global_size_t getGlobalLength() const;


    size_t getGlobalNumVectors() const;


    size_t getStride() const;


    bool isConstantStride() const;


    Teuchos::RCP<const Tpetra::Vector<scalar_t,local_ordinal_t,global_ordinal_t,node_t> >
    getVector( size_t j ) const;


    Teuchos::RCP<Tpetra::Vector<scalar_t,local_ordinal_t,global_ordinal_t,node_t> >
    getVectorNonConst( size_t j );


    double * getMVPointer_impl() const;

    void get1dCopy( const Teuchos::ArrayView<scalar_t>& A,
                    size_t lda,
                    Teuchos::Ptr<
                    const Tpetra::Map<local_ordinal_t,
                    global_ordinal_t,
                    node_t> > distribution_map,
        EDistribution distribution) const;

    template<typename KV>
    bool get1dCopy_kokkos_view(bool bInitialize, KV & A,
                    size_t lda,
                    Teuchos::Ptr<
                      const Tpetra::Map<local_ordinal_t,
                      global_ordinal_t,
                      node_t> > distribution_map,
                    EDistribution distribution) const {
      Kokkos::View<double**, Kokkos::LayoutLeft, Kokkos::HostSpace> host_new_data;
      get1dCopy_kokkos_view_host(host_new_data, lda, distribution_map, distribution);
      bool bAssigned;
      deep_copy_or_assign_view(bInitialize, A, host_new_data, bAssigned);
      return false; // currently Epetra and prior get1dCopy_kokkos_view_host call cannot get direct assignment so always return false
    }

    void get1dCopy_kokkos_view_host(
                    Kokkos::View<scalar_t**, Kokkos::LayoutLeft, Kokkos::HostSpace> & new_data,
                    size_t lda,
                    Teuchos::Ptr<
                      const Tpetra::Map<local_ordinal_t,
                      global_ordinal_t,
                      node_t> > distribution_map,
                    EDistribution) const;

    Teuchos::ArrayRCP<scalar_t> get1dViewNonConst( bool local = false );


    void put1dData( const Teuchos::ArrayView<const scalar_t>& new_data,
                    size_t lda,
                    Teuchos::Ptr<
                    const Tpetra::Map<local_ordinal_t,
                    global_ordinal_t,
                    node_t> > source_map,
        EDistribution distribution );

    template<typename KV>
    void put1dData_kokkos_view(
                    KV & new_data,
                    size_t lda,
                      Teuchos::Ptr<
                      const Tpetra::Map<local_ordinal_t,
                      global_ordinal_t,
                      node_t> > source_map,
                    EDistribution distribution ) {
      Kokkos::View<scalar_t**, Kokkos::LayoutLeft, Kokkos::HostSpace> host_new_data(
        Kokkos::ViewAllocateWithoutInitializing("host_new_data"),
        new_data.extent(0), new_data.extent(1));
      Kokkos::deep_copy(host_new_data, new_data);
      put1dData_kokkos_view_host(host_new_data, lda, source_map, distribution);
    }

    void put1dData_kokkos_view_host(
                    Kokkos::View<scalar_t**, Kokkos::LayoutLeft, Kokkos::HostSpace> & new_data,
                    size_t lda,
                    Teuchos::Ptr<
                    const Tpetra::Map<local_ordinal_t,
                    global_ordinal_t,
                    node_t> > source_map,
        EDistribution distribution );

    template<typename KV, typename host_ordinal_type_array>
    int
    gather (KV& kokkos_new_view,
            host_ordinal_type_array &perm_g2l,
            host_ordinal_type_array &recvCountRows,
            host_ordinal_type_array &recvDisplRows,
            EDistribution distribution) const
    {
      auto comm = this->getComm();
      int myRank = comm->getRank();
      int nCols = this->mv_->NumVectors();
      int nRows = this->mv_->GlobalLength();
      int nRows_l = this->mv_->MyLength();
      if (myRank == 0) {
        Kokkos::resize(kokkos_new_view, nRows, nCols);
        if (int(perm_g2l.extent(0)) == nRows) {
          Kokkos::resize(this->buf_, nRows, 1);
        } else {
          Kokkos::resize(this->buf_, 0, 1);
        }
      }
      {
        for (int j=0; j<nCols; j++) {
          scalar_t * recvbuf = reinterpret_cast<scalar_t*> (myRank != 0 || this->buf_.extent(0) > 0 ? this->buf_.data() : &kokkos_new_view(0,j));
          Teuchos::gatherv<int, scalar_t> (const_cast<scalar_t*> ((*this->mv_)[j]), nRows_l,
                                           recvbuf, recvCountRows.data(), recvDisplRows.data(),
                                           0, *comm);
          if (myRank == 0 && this->buf_.extent(0) > 0) {
            for (int i=0; i<nRows; i++) kokkos_new_view(perm_g2l(i),j) = this->buf_(i,0);
          }
        }
      }
      return 0;
    }

    template<typename KV, typename host_ordinal_type_array>
    int
    scatter (KV& kokkos_new_view,
             host_ordinal_type_array &perm_g2l,
             host_ordinal_type_array &sendCountRows,
             host_ordinal_type_array &sendDisplRows,
             EDistribution distribution) const
    {
      auto comm = this->getMap()->getComm();
      int myRank = comm->getRank();
      int nCols = this->mv_->NumVectors();
      int nRows = this->mv_->GlobalLength();
      int nRows_l = this->mv_->MyLength();
      {
        for (int j=0; j<nCols; j++) {
          if (myRank == 0 && this->buf_.extent(0) > 0) {
            for (int i=0; i<nRows; i++) this->buf_(i, 0) = kokkos_new_view(perm_g2l(i),j);
          }
          scalar_t * sendbuf = reinterpret_cast<scalar_t*> (myRank != 0 || this->buf_.extent(0) > 0 ? this->buf_.data() : &kokkos_new_view(0,j));
          Teuchos::scatterv<int, scalar_t> (sendbuf, sendCountRows.data(), sendDisplRows.data(),
                                            reinterpret_cast<scalar_t*> ((*this->mv_)[j]), nRows_l,
                                            0, *comm);
        }
      }
      return 0;
    }

    std::string description() const;


    void describe( Teuchos::FancyOStream& os,
                   const Teuchos::EVerbosityLevel verbLevel = Teuchos::Describable::verbLevel_default) const;


  private:

    Teuchos::RCP<multivec_t> mv_;
    mutable Kokkos::View<scalar_t**, Kokkos::LayoutLeft, Kokkos::HostSpace> buf_;

    mutable Teuchos::RCP<Epetra_Import> importer_;
    mutable Teuchos::RCP<Epetra_Export> exporter_;

    mutable Teuchos::RCP<const Epetra_BlockMap> mv_map_;

  };                              // end class MultiVecAdapter<NewMultiVec>

} // end namespace Amesos2


#endif // AMESOS2_EPETRA_MULTIVEC_ADAPTER_DECL_HPP