Tpetra_DistObject_decl.hpp

Go to the documentation of this file.

// @HEADER
// ***********************************************************************
//
//          Tpetra: Templated Linear Algebra Services Package
//                 Copyright (2008) Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
//
// ************************************************************************
// @HEADER

#ifndef TPETRA_DISTOBJECT_DECL_HPP
#define TPETRA_DISTOBJECT_DECL_HPP


#include "Tpetra_Map.hpp"
#include "Tpetra_Import.hpp"
#include "Tpetra_Export.hpp"
#include "Tpetra_SrcDistObject.hpp"
#include "Tpetra_DistObject_fwd.hpp"
#include "Kokkos_ArithTraits.hpp"
#include <type_traits>

// #ifndef HAVE_TPETRA_TRANSFER_TIMERS
// #  define HAVE_TPETRA_TRANSFER_TIMERS 1
// #endif // HAVE_TPETRA_TRANSFER_TIMERS

#ifdef HAVE_TPETRA_TRANSFER_TIMERS
#  undef HAVE_TPETRA_TRANSFER_TIMERS
#endif // HAVE_TPETRA_TRANSFER_TIMERS

namespace KokkosClassic {
  enum ReadWriteOption {
    ReadWrite = 0, 
    WriteOnly = 1 
  };
} // namespace KokkosClassic

namespace Tpetra {

  template<class DistObjectType>
  void
  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input,
                               const Teuchos::RCP<const Map<typename DistObjectType::local_ordinal_type,
                                                            typename DistObjectType::global_ordinal_type,
                                                            typename DistObjectType::node_type> >& newMap);

  template<class DistObjectType>
  void
  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input);

  template <class Packet,
            class LocalOrdinal,
            class GlobalOrdinal,
            class Node>
  class DistObject :
    virtual public SrcDistObject,
    virtual public Teuchos::Describable
  {
  public:


    using packet_type = typename ::Kokkos::Details::ArithTraits<Packet>::val_type;
    using local_ordinal_type = LocalOrdinal;
    using global_ordinal_type = GlobalOrdinal;
    using node_type = Node;

    using device_type = typename Node::device_type;
    using execution_space = typename device_type::execution_space;

    using map_type = Map<local_ordinal_type, global_ordinal_type, node_type>;



    explicit DistObject (const Teuchos::RCP<const map_type>& map);

    DistObject (const DistObject<Packet, LocalOrdinal, GlobalOrdinal, Node>&) = default;

    DistObject& operator= (const DistObject<Packet, LocalOrdinal, GlobalOrdinal, Node>&) = default;

    DistObject (DistObject<Packet, LocalOrdinal, GlobalOrdinal, Node>&&) = default;

    DistObject& operator= (DistObject<Packet, LocalOrdinal, GlobalOrdinal, Node>&&) = default;

    virtual ~DistObject () = default;



    void
    doImport (const SrcDistObject& source,
              const Import<LocalOrdinal, GlobalOrdinal, Node>& importer,
              const CombineMode CM,
              const bool restrictedMode = false);

    void
    doExport (const SrcDistObject& source,
              const Export<LocalOrdinal, GlobalOrdinal, Node>& exporter,
              const CombineMode CM,
              const bool restrictedMode = false);

    void
    doImport (const SrcDistObject& source,
              const Export<LocalOrdinal, GlobalOrdinal, Node>& exporter,
              const CombineMode CM,
              const bool restrictedMode = false);

    void
    doExport (const SrcDistObject& source,
              const Import<LocalOrdinal, GlobalOrdinal, Node>& importer,
              const CombineMode CM,
              const bool restrictedMode = false);



    bool isDistributed () const;

    virtual Teuchos::RCP<const map_type> getMap () const { return map_; }



    void print (std::ostream& os) const;



    virtual std::string description () const;

    virtual void
    describe (Teuchos::FancyOStream &out,
              const Teuchos::EVerbosityLevel verbLevel =
              Teuchos::Describable::verbLevel_default) const;



    virtual void
    removeEmptyProcessesInPlace (const Teuchos::RCP<const map_type>& newMap);


  protected:
    enum ReverseOption {
      DoForward, //*!< Perform the transfer in forward mode.
      DoReverse  //*!< Perform the transfer in reverse mode.
    };

    virtual size_t constantNumberOfPackets () const;

    virtual void
    doTransfer (const SrcDistObject& src,
                const ::Tpetra::Details::Transfer<local_ordinal_type, global_ordinal_type, node_type>& transfer,
                const char modeString[],
                const ReverseOption revOp,
                const CombineMode CM,
                const bool restrictedMode);

    virtual bool
    reallocArraysForNumPacketsPerLid (const size_t numExportLIDs,
                                      const size_t numImportLIDs);

#ifdef TPETRA_ENABLE_DEPRECATED_CODE
    virtual void TPETRA_DEPRECATED
    doTransferOld (const SrcDistObject& src,
                   CombineMode CM,
                   size_t numSameIDs,
                   const Teuchos::ArrayView<const local_ordinal_type> &permuteToLIDs,
                   const Teuchos::ArrayView<const local_ordinal_type> &permuteFromLIDs,
                   const Teuchos::ArrayView<const local_ordinal_type> &remoteLIDs,
                   const Teuchos::ArrayView<const local_ordinal_type> &exportLIDs,
                   Distributor &distor,
                   ReverseOption revOp,
                   const bool restrictedMode);
#endif // TPETRA_ENABLE_DEPRECATED_CODE

#ifdef KOKKOS_ENABLE_CUDA
    using buffer_memory_space = typename std::conditional<
      std::is_same<typename device_type::execution_space, Kokkos::Cuda>::value,
      Kokkos::CudaSpace,
      typename device_type::memory_space>::type;
#else
    using buffer_memory_space = typename device_type::memory_space;
#endif // KOKKOS_ENABLE_CUDA

  public:
    using buffer_device_type =
      Kokkos::Device<typename device_type::execution_space,
                     buffer_memory_space>;
  protected:
    virtual void
    doTransferNew (const SrcDistObject& src,
                   const CombineMode CM,
                   const size_t numSameIDs,
                   const Kokkos::DualView<const local_ordinal_type*,
                     buffer_device_type>& permuteToLIDs,
                   const Kokkos::DualView<const local_ordinal_type*,
                     buffer_device_type>& permuteFromLIDs,
                   const Kokkos::DualView<const local_ordinal_type*,
                     buffer_device_type>& remoteLIDs,
                   const Kokkos::DualView<const local_ordinal_type*,
                     buffer_device_type>& exportLIDs,
                   Distributor& distor,
                   const ReverseOption revOp,
                   const bool commOnHost,
                   const bool restrictedMode);



    virtual bool
    checkSizes (const SrcDistObject& source) = 0;

#ifdef TPETRA_ENABLE_DEPRECATED_CODE
    virtual void
    copyAndPermuteNew (const SrcDistObject& source,
                       const size_t numSameIDs,
                       const Kokkos::DualView<const local_ordinal_type*,
                         buffer_device_type>& permuteToLIDs,
                       const Kokkos::DualView<const local_ordinal_type*,
                         buffer_device_type>& permuteFromLIDs);
#else // TPETRA_ENABLE_DEPRECATED_CODE
    virtual void
    copyAndPermute (const SrcDistObject& source,
                    const size_t numSameIDs,
                    const Kokkos::DualView<const local_ordinal_type*,
                      buffer_device_type>& permuteToLIDs,
                    const Kokkos::DualView<const local_ordinal_type*,
                      buffer_device_type>& permuteFromLIDs);
#endif // TPETRA_ENABLE_DEPRECATED_CODE

#ifdef TPETRA_ENABLE_DEPRECATED_CODE
    virtual void
    packAndPrepareNew (const SrcDistObject& source,
                       const Kokkos::DualView<const local_ordinal_type*,
                         buffer_device_type>& exportLIDs,
                       Kokkos::DualView<packet_type*,
                         buffer_device_type>& exports,
                       Kokkos::DualView<size_t*,
                         buffer_device_type> numPacketsPerLID,
                       size_t& constantNumPackets,
                       Distributor& distor);
#else // TPETRA_ENABLE_DEPRECATED_CODE
    virtual void
    packAndPrepare (const SrcDistObject& source,
                    const Kokkos::DualView<const local_ordinal_type*,
                      buffer_device_type>& exportLIDs,
                    Kokkos::DualView<packet_type*,
                      buffer_device_type>& exports,
                    Kokkos::DualView<size_t*,
                      buffer_device_type> numPacketsPerLID,
                    size_t& constantNumPackets,
                    Distributor& distor);
#endif // TPETRA_ENABLE_DEPRECATED_CODE

#ifdef TPETRA_ENABLE_DEPRECATED_CODE
    virtual void
    unpackAndCombineNew (const Kokkos::DualView<const local_ordinal_type*,
                           buffer_device_type>& importLIDs,
                         Kokkos::DualView<packet_type*,
                           buffer_device_type> imports,
                         Kokkos::DualView<size_t*,
                           buffer_device_type> numPacketsPerLID,
                         const size_t constantNumPackets,
                         Distributor& distor,
                         const CombineMode combineMode);
#else // TPETRA_ENABLE_DEPRECATED_CODE
    virtual void
    unpackAndCombine (const Kokkos::DualView<const local_ordinal_type*,
                        buffer_device_type>& importLIDs,
                      Kokkos::DualView<packet_type*,
                        buffer_device_type> imports,
                      Kokkos::DualView<size_t*,
                        buffer_device_type> numPacketsPerLID,
                      const size_t constantNumPackets,
                      Distributor& distor,
                      const CombineMode combineMode);
#endif // TPETRA_ENABLE_DEPRECATED_CODE

#ifdef TPETRA_ENABLE_DEPRECATED_CODE
    virtual bool TPETRA_DEPRECATED useNewInterface () { return true; }

    virtual void TPETRA_DEPRECATED
    copyAndPermute (const SrcDistObject& source,
                    const size_t numSameIDs,
                    const Teuchos::ArrayView<const local_ordinal_type>& permuteToLIDs,
                    const Teuchos::ArrayView<const local_ordinal_type>& permuteFromLIDs);

    virtual void TPETRA_DEPRECATED
    packAndPrepare (const SrcDistObject& source,
                    const Teuchos::ArrayView<const local_ordinal_type>& exportLIDs,
                    Teuchos::Array<packet_type>& exports,
                    const Teuchos::ArrayView<size_t>& numPacketsPerLID,
                    size_t& constantNumPackets,
                    Distributor& distor);

    virtual void TPETRA_DEPRECATED
    unpackAndCombine (const Teuchos::ArrayView<const local_ordinal_type>& importLIDs,
                      const Teuchos::ArrayView<const packet_type>& imports,
                      const Teuchos::ArrayView<size_t>& numPacketsPerLID,
                      const size_t constantNumPackets,
                      Distributor& distor,
                      const CombineMode combineMode);

    virtual void TPETRA_DEPRECATED
    createViews () const;

    virtual void TPETRA_DEPRECATED
    createViewsNonConst (KokkosClassic::ReadWriteOption rwo);

    virtual void TPETRA_DEPRECATED releaseViews () const;
#endif // TPETRA_ENABLE_DEPRECATED_CODE

    Teuchos::RCP<const map_type> map_;

  protected:
    Kokkos::DualView<packet_type*, buffer_device_type> imports_;

    bool
    reallocImportsIfNeeded (const size_t newSize,
                            const bool verbose,
                            const std::string* prefix);

    Kokkos::DualView<size_t*, buffer_device_type> numImportPacketsPerLID_;

    Kokkos::DualView<packet_type*, buffer_device_type> exports_;

    Kokkos::DualView<size_t*, buffer_device_type> numExportPacketsPerLID_;

  private:
    using this_type = DistObject<Packet, LocalOrdinal, GlobalOrdinal, Node>;

#ifdef HAVE_TPETRA_TRANSFER_TIMERS
    Teuchos::RCP<Teuchos::Time> doXferTimer_;
    Teuchos::RCP<Teuchos::Time> copyAndPermuteTimer_;
    Teuchos::RCP<Teuchos::Time> packAndPrepareTimer_;
    Teuchos::RCP<Teuchos::Time> doPostsAndWaitsTimer_;
    Teuchos::RCP<Teuchos::Time> unpackAndCombineTimer_;
#endif // HAVE_TPETRA_TRANSFER_TIMERS
  }; // class DistObject
} // namespace Tpetra

#endif // TPETRA_DISTOBJECT_DECL_HPP