Tpetra_Details_DistributorPlan.hpp

Go to the documentation of this file.

// @HEADER
// *****************************************************************************
//          Tpetra: Templated Linear Algebra Services Package
//
// Copyright 2008 NTESS and the Tpetra contributors.
// SPDX-License-Identifier: BSD-3-Clause
// *****************************************************************************
// @HEADER

#ifndef TPETRA_DETAILS_DISTRIBUTOR_PLAN_HPP
#define TPETRA_DETAILS_DISTRIBUTOR_PLAN_HPP

#include "Teuchos_ParameterListAcceptorDefaultBase.hpp"
#include "Teuchos_Array.hpp"
#include "Teuchos_Comm.hpp"
#include "Teuchos_RCP.hpp"
#include "TpetraCore_config.h"

#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
#include <mpi_advance.h>
#endif

namespace Tpetra {
namespace Details {

enum EDistributorSendType {
  DISTRIBUTOR_ISEND,    // Use MPI_Isend (Teuchos::isend)
  DISTRIBUTOR_SEND,     // Use MPI_Send (Teuchos::send)
  DISTRIBUTOR_ALLTOALL  // Use MPI_Alltoall
#if defined(HAVE_TPETRA_MPI)
  ,
  DISTRIBUTOR_IALLTOFEWV
#endif
#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
  ,
  DISTRIBUTOR_MPIADVANCE_ALLTOALL,
  DISTRIBUTOR_MPIADVANCE_NBRALLTOALLV
#endif
};

std::string
DistributorSendTypeEnumToString(EDistributorSendType sendType);

EDistributorSendType
DistributorSendTypeStringToEnum(const std::string_view s);

Teuchos::Array<std::string> distributorSendTypes();

Teuchos::Array<EDistributorSendType> distributorSendTypeEnums();

Teuchos::Array<std::string> distributorSendTypes();

const std::string& validSendTypeOrThrow(const std::string& s);

enum EDistributorHowInitialized {
  DISTRIBUTOR_NOT_INITIALIZED,                           // Not initialized yet
  DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_SENDS,          // By createFromSends
  DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_RECVS,          // By createFromRecvs
  DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_SENDS_N_RECVS,  // By createFromSendsAndRecvs
  DISTRIBUTOR_INITIALIZED_BY_REVERSE,                    // By createReverseDistributor
  DISTRIBUTOR_INITIALIZED_BY_COPY,                       // By copy constructor
};

std::string
DistributorHowInitializedEnumToString(EDistributorHowInitialized how);

class DistributorPlan : public Teuchos::ParameterListAcceptorDefaultBase {
  static constexpr int DEFAULT_MPI_TAG = 0;

 public:
  using IndexView     = std::vector<size_t>;
  using SubViewLimits = std::pair<IndexView, IndexView>;

  DistributorPlan(Teuchos::RCP<const Teuchos::Comm<int>> comm);
  DistributorPlan(const DistributorPlan& otherPlan);

  size_t createFromSends(const Teuchos::ArrayView<const int>& exportProcIDs);
  void createFromRecvs(const Teuchos::ArrayView<const int>& remoteProcIDs);
  void createFromSendsAndRecvs(const Teuchos::ArrayView<const int>& exportProcIDs,
                               const Teuchos::ArrayView<const int>& remoteProcIDs);

  void setParameterList(const Teuchos::RCP<Teuchos::ParameterList>& plist);

  Teuchos::RCP<DistributorPlan> getReversePlan() const;

  Teuchos::RCP<const Teuchos::Comm<int>> getComm() const { return comm_; }
#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
  Teuchos::RCP<MPIX_Comm*> getMPIXComm() const { return mpixComm_; }
#endif
  EDistributorSendType getSendType() const { return sendType_; }
  size_t getNumReceives() const { return numReceives_; }
  size_t getNumSends() const { return numSendsToOtherProcs_; }
  bool hasSelfMessage() const { return sendMessageToSelf_; }
  size_t getMaxSendLength() const { return maxSendLength_; }
  size_t getTotalReceiveLength() const { return totalReceiveLength_; }
  Teuchos::ArrayView<const int> getProcsFrom() const { return procsFrom_; }
  Teuchos::ArrayView<const int> getProcsTo() const { return procIdsToSendTo_; }
  Teuchos::ArrayView<const size_t> getLengthsFrom() const { return lengthsFrom_; }
  Teuchos::ArrayView<const size_t> getLengthsTo() const { return lengthsTo_; }
  Teuchos::ArrayView<const size_t> getStartsTo() const { return startsTo_; }
  Teuchos::ArrayView<const size_t> getIndicesTo() const { return indicesTo_; }
  Details::EDistributorHowInitialized howInitialized() const { return howInitialized_; }

  SubViewLimits getImportViewLimits(size_t numPackets) const;
  SubViewLimits getImportViewLimits(const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID) const;
  SubViewLimits getExportViewLimits(size_t numPackets) const;
  SubViewLimits getExportViewLimits(const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID) const;

#if defined(HAVE_TPETRA_MPI)
  const std::vector<int> getRoots() const {
    return roots_;
  }
#endif
 private:
  // after the plan has been created we have the info we need to initialize the MPI advance communicator
#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
  void initializeMpiAdvance();
#endif

#if defined(HAVE_TPETRA_MPI)
  void maybeInitializeRoots();
#endif

  Teuchos::RCP<const Teuchos::ParameterList> getValidParameters() const;

  void createReversePlan() const;

  void computeReceives();

  Teuchos::RCP<const Teuchos::Comm<int>> comm_;
#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
  Teuchos::RCP<MPIX_Comm*> mpixComm_;
#endif

  Details::EDistributorHowInitialized howInitialized_;
  mutable Teuchos::RCP<DistributorPlan> reversePlan_;


  EDistributorSendType sendType_;

  bool sendMessageToSelf_;
  size_t numSendsToOtherProcs_;
  Teuchos::Array<int> procIdsToSendTo_;

  Teuchos::Array<size_t> startsTo_;

  Teuchos::Array<size_t> lengthsTo_;

  size_t maxSendLength_;

  Teuchos::Array<size_t> indicesTo_;

  size_t numReceives_;

  size_t totalReceiveLength_;

  Teuchos::Array<size_t> lengthsFrom_;

  Teuchos::Array<int> procsFrom_;

  Teuchos::Array<size_t> startsFrom_;

  Teuchos::Array<size_t> indicesFrom_;

#if defined(HAVE_TPETRA_MPI)
  std::vector<int> roots_;

#endif
};

}  // namespace Details
}  // namespace Tpetra

#endif