10 #ifndef TPETRA_DISTRIBUTOR_HPP 
   11 #define TPETRA_DISTRIBUTOR_HPP 
   13 #include "Tpetra_Details_DistributorActor.hpp" 
   17 #include "Teuchos_as.hpp" 
   18 #include "Teuchos_Describable.hpp" 
   19 #include "Teuchos_ParameterListAcceptorDefaultBase.hpp" 
   20 #include "Teuchos_VerboseObject.hpp" 
   23 #include "KokkosCompat_View.hpp" 
   24 #include "Kokkos_Core.hpp" 
   25 #include "Kokkos_TeuchosCommAdapters.hpp" 
   28 #include <type_traits> 
   32 #if defined(TPETRA_ENABLE_DEPRECATED_CODE) 
  105                     public Teuchos::ParameterListAcceptorDefaultBase {
 
  118   explicit Distributor(
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm);
 
  131   Distributor(
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
 
  132               const Teuchos::RCP<Teuchos::FancyOStream>& out);
 
  147   Distributor(
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
 
  148               const Teuchos::RCP<Teuchos::ParameterList>& plist);
 
  166   Distributor(
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
 
  167               const Teuchos::RCP<Teuchos::FancyOStream>& out,
 
  168               const Teuchos::RCP<Teuchos::ParameterList>& plist);
 
  225   size_t createFromSends(
const Teuchos::ArrayView<const int>& exportProcIDs);
 
  260   template <
class Ordinal>
 
  263                   const Teuchos::ArrayView<const int>& remoteProcIDs,
 
  264                   Teuchos::Array<Ordinal>& exportIDs,
 
  265                   Teuchos::Array<int>& exportProcIDs);
 
  276                           const Teuchos::ArrayView<const int>& remoteProcIDs);
 
  311   Teuchos::ArrayView<const int> 
getProcsTo() 
const;
 
  336     return plan_.howInitialized();
 
  353   Teuchos::RCP<Distributor> 
getReverse(
bool create = 
true) 
const;
 
  395   template <
class ExpView, 
class ImpView>
 
  396   typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  398       const ExpView& exports,
 
  400       const ImpView& imports);
 
  423   template <
class ExpView, 
class ImpView>
 
  424   typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  426                   const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
 
  427                   const ImpView& imports,
 
  428                   const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
 
  454   template <
class ExpView, 
class ImpView>
 
  455   typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  456   doPosts(
const ExpView& exports,
 
  458           const ImpView& imports);
 
  478   template <
class ExpView, 
class ImpView>
 
  479   typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  480   doPosts(
const ExpView& exports,
 
  481           const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
 
  482           const ImpView& imports,
 
  483           const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
 
  489   template <
class ExpView, 
class ImpView>
 
  490   typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  493                          const ImpView& imports);
 
  499   template <
class ExpView, 
class ImpView>
 
  500   typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  502                          const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
 
  503                          const ImpView& imports,
 
  504                          const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
 
  510   template <
class ExpView, 
class ImpView>
 
  511   typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  514                  const ImpView& imports);
 
  520   template <
class ExpView, 
class ImpView>
 
  521   typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  523                  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
 
  524                  const ImpView& imports,
 
  525                  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
 
  556   describe(Teuchos::FancyOStream& out,
 
  557            const Teuchos::EVerbosityLevel verbLevel =
 
  558                Teuchos::Describable::verbLevel_default) 
const;
 
  569   Details::DistributorActor actor_;
 
  575   static bool getVerbose();
 
  581   std::unique_ptr<std::string>
 
  582   createPrefix(
const char methodName[]) 
const;
 
  585   bool verbose_ = getVerbose();
 
  592   mutable Teuchos::RCP<Distributor> reverseDistributor_;
 
  606   template <
class Ordinal>
 
  607   void computeSends(
const Teuchos::ArrayView<const Ordinal>& remoteGIDs,
 
  608                     const Teuchos::ArrayView<const int>& remoteProcIDs,
 
  609                     Teuchos::Array<Ordinal>& exportGIDs,
 
  610                     Teuchos::Array<int>& exportProcIDs);
 
  613   void createReverseDistributor() 
const;
 
  620   localDescribeToString(
const Teuchos::EVerbosityLevel vl) 
const;
 
  623 template <
class ExpView, 
class ImpView>
 
  624 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  626     doPostsAndWaits(
const ExpView& exports,
 
  628                     const ImpView& imports) {
 
  629   actor_.doPostsAndWaits(plan_, exports, numPackets, imports);
 
  632 template <
class ExpView, 
class ImpView>
 
  633 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  635     doPostsAndWaits(
const ExpView& exports,
 
  636                     const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
 
  637                     const ImpView& imports,
 
  638                     const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID) {
 
  639   actor_.
doPostsAndWaits(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
 
  642 template <
class ExpView, 
class ImpView>
 
  643 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  645     doPosts(
const ExpView& exports,
 
  647             const ImpView& imports) {
 
  648   actor_.
doPosts(plan_, exports, numPackets, imports);
 
  651 template <
class ExpView, 
class ImpView>
 
  652 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  654     doPosts(
const ExpView& exports,
 
  655             const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
 
  656             const ImpView& imports,
 
  657             const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID) {
 
  658   actor_.
doPosts(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
 
  661 template <
class ExpView, 
class ImpView>
 
  662 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  664     doReversePostsAndWaits(
const ExpView& exports,
 
  666                            const ImpView& imports) {
 
  667   doReversePosts(exports, numPackets, imports);
 
  671 template <
class ExpView, 
class ImpView>
 
  672 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  674     doReversePostsAndWaits(
const ExpView& exports,
 
  675                            const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
 
  676                            const ImpView& imports,
 
  677                            const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID) {
 
  678   doReversePosts(exports, numExportPacketsPerLID, imports,
 
  679                  numImportPacketsPerLID);
 
  683 template <
class ExpView, 
class ImpView>
 
  684 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  686     doReversePosts(
const ExpView& exports,
 
  688                    const ImpView& imports) {
 
  690   TEUCHOS_TEST_FOR_EXCEPTION(
 
  691       !plan_.getIndicesTo().is_null(), std::runtime_error,
 
  692       "Tpetra::Distributor::doReversePosts(3 args): Can only do " 
  693       "reverse communication when original data are blocked by process.");
 
  694   if (reverseDistributor_.is_null()) {
 
  695     createReverseDistributor();
 
  697   reverseDistributor_->doPosts(exports, numPackets, imports);
 
  700 template <
class ExpView, 
class ImpView>
 
  701 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
 
  703     doReversePosts(
const ExpView& exports,
 
  704                    const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
 
  705                    const ImpView& imports,
 
  706                    const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID) {
 
  708   TEUCHOS_TEST_FOR_EXCEPTION(
 
  709       !plan_.getIndicesTo().is_null(), std::runtime_error,
 
  710       "Tpetra::Distributor::doReversePosts(3 args): Can only do " 
  711       "reverse communication when original data are blocked by process.");
 
  712   if (reverseDistributor_.is_null()) {
 
  713     createReverseDistributor();
 
  715   reverseDistributor_->doPosts(exports, numExportPacketsPerLID,
 
  716                                imports, numImportPacketsPerLID);
 
  719 template <
class OrdinalType>
 
  721     computeSends(
const Teuchos::ArrayView<const OrdinalType>& importGIDs,
 
  722                  const Teuchos::ArrayView<const int>& importProcIDs,
 
  723                  Teuchos::Array<OrdinalType>& exportGIDs,
 
  724                  Teuchos::Array<int>& exportProcIDs) {
 
  733   using Teuchos::ArrayView;
 
  734   using size_type        = 
typename ArrayView<const OrdinalType>::size_type;
 
  735   const char errPrefix[] = 
"Tpetra::Distributor::computeSends: ";
 
  736   const char suffix[] =
 
  737       "  Please report this bug to the Tpetra developers.";
 
  739   const int myRank = plan_.getComm()->getRank();
 
  741   TEUCHOS_TEST_FOR_EXCEPTION(importGIDs.size() != importProcIDs.size(),
 
  742                              std::invalid_argument, errPrefix << 
"On Process " << myRank << 
": importProcIDs.size()=" << importProcIDs.size() << 
" != importGIDs.size()=" << importGIDs.size() << 
".");
 
  744   const size_type numImports = importProcIDs.size();
 
  745   Kokkos::View<size_t*, Kokkos::HostSpace> importObjs(
"importObjs", 2 * numImports);
 
  747   for (size_type i = 0; i < numImports; ++i) {
 
  748     importObjs[2 * i]     = 
static_cast<size_t>(importGIDs[i]);
 
  749     importObjs[2 * i + 1] = 
static_cast<size_t>(myRank);
 
  758   const size_t numExportsAsSizeT =
 
  759       tempPlan.createFromSends(importProcIDs);
 
  760   const size_type numExports =
 
  761       static_cast<size_type
>(numExportsAsSizeT);
 
  762   TEUCHOS_TEST_FOR_EXCEPTION(numExports < 0, std::logic_error, errPrefix << 
"tempPlan.createFromSends() returned numExports=" << numExportsAsSizeT << 
" as a size_t, which overflows to " << numExports << 
" when cast to " << Teuchos::TypeNameTraits<size_type>::name() << 
"." << suffix);
 
  763   TEUCHOS_TEST_FOR_EXCEPTION(size_type(tempPlan.getTotalReceiveLength()) != numExports,
 
  764                              std::logic_error, errPrefix << 
"tempPlan.getTotalReceiveLength()=" << tempPlan.getTotalReceiveLength() << 
" != numExports=" << numExports << 
"." << suffix);
 
  766   if (numExports > 0) {
 
  767     exportGIDs.resize(numExports);
 
  768     exportProcIDs.resize(numExports);
 
  779   static_assert(
sizeof(
size_t) >= 
sizeof(OrdinalType),
 
  780                 "Tpetra::Distributor::computeSends: " 
  781                 "sizeof(size_t) < sizeof(OrdinalType).");
 
  783   TEUCHOS_TEST_FOR_EXCEPTION(tempPlan.getTotalReceiveLength() < size_t(numExports),
 
  785                              errPrefix << 
"tempPlan.getTotalReceiveLength()=" 
  786                                        << tempPlan.getTotalReceiveLength() << 
" < numExports=" 
  787                                        << numExports << 
"." << suffix);
 
  789   Kokkos::View<size_t*, Kokkos::HostSpace> exportObjs(
"exportObjs", tempPlan.getTotalReceiveLength() * 2);
 
  790   tempPlan.doPostsAndWaits(importObjs, 2, exportObjs);
 
  793   for (size_type i = 0; i < numExports; ++i) {
 
  794     exportGIDs[i]    = 
static_cast<OrdinalType
>(exportObjs[2 * i]);
 
  795     exportProcIDs[i] = 
static_cast<int>(exportObjs[2 * i + 1]);
 
  799   plan_ = tempPlan.plan_;
 
  802 template <
class OrdinalType>
 
  804     createFromRecvs(
const Teuchos::ArrayView<const OrdinalType>& remoteGIDs,
 
  805                     const Teuchos::ArrayView<const int>& remoteProcIDs,
 
  806                     Teuchos::Array<OrdinalType>& exportGIDs,
 
  807                     Teuchos::Array<int>& exportProcIDs) {
 
  809   const char errPrefix[] = 
"Tpetra::Distributor::createFromRecvs: ";
 
  810   const int myRank       = plan_.getComm()->getRank();
 
  812   std::unique_ptr<std::string> prefix;
 
  814     prefix = createPrefix(
"createFromRecvs");
 
  815     std::ostringstream os;
 
  816     os << *prefix << 
"Start" << endl;
 
  817     std::cerr << os.str();
 
  822     using Teuchos::outArg;
 
  823     using Teuchos::REDUCE_MAX;
 
  824     using Teuchos::reduceAll;
 
  828         (remoteGIDs.size() != remoteProcIDs.size()) ? myRank : -1;
 
  830     reduceAll(*plan_.getComm(), REDUCE_MAX, errProc, outArg(maxErrProc));
 
  831     TEUCHOS_TEST_FOR_EXCEPTION(maxErrProc != -1, std::runtime_error, errPrefix << 
"Lists " 
  832                                                                                   "of remote IDs and remote process IDs must have the same " 
  833                                                                                   "size on all participating processes.  Maximum process ID " 
  835                                                                                << maxErrProc << 
".");
 
  839     TEUCHOS_TEST_FOR_EXCEPTION(remoteGIDs.size() != remoteProcIDs.size(), std::runtime_error,
 
  840                                errPrefix << 
"On Process " << myRank << 
": " 
  842                                          << remoteGIDs.size() << 
" != remoteProcIDs.size()=" << remoteProcIDs.size() << 
".");
 
  845   computeSends(remoteGIDs, remoteProcIDs, exportGIDs, exportProcIDs);
 
  847   plan_.createFromRecvs(remoteProcIDs);
 
  850     std::ostringstream os;
 
  851     os << *prefix << 
"Done" << endl;
 
  852     std::cerr << os.str();
 
  858 #endif  // TPETRA_DISTRIBUTOR_HPP 
const Details::DistributorPlan & getPlan() const 
Get this Distributor's DistributorPlan. 
 
size_t getNumReceives() const 
The number of processes from which we will receive data. 
 
std::string description() const 
Return a one-line description of this object. 
 
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the reverse communication plan. 
 
Teuchos::RCP< Distributor > getReverse(bool create=true) const 
A reverse communication plan Distributor. 
 
EDistributorHowInitialized
Enum indicating how and whether a Distributor was initialized. 
 
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const 
List of valid Distributor parameters. 
 
virtual ~Distributor()=default
Destructor (virtual for memory safety). 
 
static bool debug()
Whether Tpetra is in debug mode. 
 
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a forward plan, but do not execute the waits yet. 
 
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this. 
 
Teuchos::ArrayView< const size_t > getLengthsFrom() const 
Number of values this process will receive from each process. 
 
Teuchos::ArrayView< const int > getProcsFrom() const 
Ranks of the processes sending values to this process. 
 
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send. 
 
Details::EDistributorHowInitialized howInitialized() const 
Return an enum indicating whether and how a Distributor was initialized. 
 
Teuchos::ArrayView< const int > getProcsTo() const 
Ranks of the processes to which this process will send values. 
 
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
 
bool hasSelfMessage() const 
Whether the calling process will send or receive messages to itself. 
 
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a reverse plan, but do not execute the waits yet. 
 
Sets up and executes a communication plan for a Tpetra DistObject. 
 
size_t getTotalReceiveLength() const 
Total number of values this process will receive from other processes. 
 
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters. 
 
Teuchos::ArrayView< const size_t > getLengthsTo() const 
Number of values this process will send to each process. 
 
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the (forward) communication plan. 
 
Stand-alone utility functions and macros. 
 
Teuchos::Array< std::string > distributorSendTypes()
Valid string values for Distributor's "Send type" parameter. 
 
size_t getNumSends() const 
The number of processes to which we will send data. 
 
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const 
Describe this object in a human-readable way to the given output stream. 
 
size_t getMaxSendLength() const 
Maximum number of values this process will send to another single process. 
 
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive. 
 
Distributor(const Teuchos::RCP< const Teuchos::Comm< int > > &comm)
Construct using the specified communicator and default parameters. 
 
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.