10 #ifndef TPETRA_DISTRIBUTOR_HPP
11 #define TPETRA_DISTRIBUTOR_HPP
13 #include "Tpetra_Details_DistributorActor.hpp"
17 #include "Teuchos_as.hpp"
18 #include "Teuchos_Describable.hpp"
19 #include "Teuchos_ParameterListAcceptorDefaultBase.hpp"
20 #include "Teuchos_VerboseObject.hpp"
23 #include "KokkosCompat_View.hpp"
24 #include "Kokkos_Core.hpp"
25 #include "Kokkos_TeuchosCommAdapters.hpp"
28 #include <type_traits>
32 #if defined(TPETRA_ENABLE_DEPRECATED_CODE)
105 public Teuchos::ParameterListAcceptorDefaultBase {
118 explicit Distributor(
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm);
131 Distributor(
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
132 const Teuchos::RCP<Teuchos::FancyOStream>& out);
147 Distributor(
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
148 const Teuchos::RCP<Teuchos::ParameterList>& plist);
166 Distributor(
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
167 const Teuchos::RCP<Teuchos::FancyOStream>& out,
168 const Teuchos::RCP<Teuchos::ParameterList>& plist);
225 size_t createFromSends(
const Teuchos::ArrayView<const int>& exportProcIDs);
260 template <
class Ordinal>
263 const Teuchos::ArrayView<const int>& remoteProcIDs,
264 Teuchos::Array<Ordinal>& exportIDs,
265 Teuchos::Array<int>& exportProcIDs);
276 const Teuchos::ArrayView<const int>& remoteProcIDs);
311 Teuchos::ArrayView<const int>
getProcsTo()
const;
336 return plan_.howInitialized();
353 Teuchos::RCP<Distributor>
getReverse(
bool create =
true)
const;
395 template <
class ExpView,
class ImpView>
396 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
398 const ExpView& exports,
400 const ImpView& imports);
423 template <
class ExpView,
class ImpView>
424 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
426 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
427 const ImpView& imports,
428 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
454 template <
class ExpView,
class ImpView>
455 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
456 doPosts(
const ExpView& exports,
458 const ImpView& imports);
478 template <
class ExpView,
class ImpView>
479 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
480 doPosts(
const ExpView& exports,
481 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
482 const ImpView& imports,
483 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
489 template <
class ExpView,
class ImpView>
490 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
493 const ImpView& imports);
499 template <
class ExpView,
class ImpView>
500 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
502 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
503 const ImpView& imports,
504 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
510 template <
class ExpView,
class ImpView>
511 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
514 const ImpView& imports);
520 template <
class ExpView,
class ImpView>
521 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
523 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
524 const ImpView& imports,
525 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
556 describe(Teuchos::FancyOStream& out,
557 const Teuchos::EVerbosityLevel verbLevel =
558 Teuchos::Describable::verbLevel_default)
const;
569 Details::DistributorActor actor_;
575 static bool getVerbose();
581 std::unique_ptr<std::string>
582 createPrefix(
const char methodName[])
const;
585 bool verbose_ = getVerbose();
592 mutable Teuchos::RCP<Distributor> reverseDistributor_;
606 template <
class Ordinal>
607 void computeSends(
const Teuchos::ArrayView<const Ordinal>& remoteGIDs,
608 const Teuchos::ArrayView<const int>& remoteProcIDs,
609 Teuchos::Array<Ordinal>& exportGIDs,
610 Teuchos::Array<int>& exportProcIDs);
613 void createReverseDistributor()
const;
620 localDescribeToString(
const Teuchos::EVerbosityLevel vl)
const;
623 template <
class ExpView,
class ImpView>
624 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
626 doPostsAndWaits(
const ExpView& exports,
628 const ImpView& imports) {
629 actor_.doPostsAndWaits(plan_, exports, numPackets, imports);
632 template <
class ExpView,
class ImpView>
633 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
635 doPostsAndWaits(
const ExpView& exports,
636 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
637 const ImpView& imports,
638 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID) {
639 actor_.
doPostsAndWaits(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
642 template <
class ExpView,
class ImpView>
643 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
645 doPosts(
const ExpView& exports,
647 const ImpView& imports) {
648 actor_.
doPosts(plan_, exports, numPackets, imports);
651 template <
class ExpView,
class ImpView>
652 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
654 doPosts(
const ExpView& exports,
655 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
656 const ImpView& imports,
657 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID) {
658 actor_.
doPosts(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
661 template <
class ExpView,
class ImpView>
662 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
664 doReversePostsAndWaits(
const ExpView& exports,
666 const ImpView& imports) {
667 doReversePosts(exports, numPackets, imports);
671 template <
class ExpView,
class ImpView>
672 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
674 doReversePostsAndWaits(
const ExpView& exports,
675 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
676 const ImpView& imports,
677 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID) {
678 doReversePosts(exports, numExportPacketsPerLID, imports,
679 numImportPacketsPerLID);
683 template <
class ExpView,
class ImpView>
684 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
686 doReversePosts(
const ExpView& exports,
688 const ImpView& imports) {
690 TEUCHOS_TEST_FOR_EXCEPTION(
691 !plan_.getIndicesTo().is_null(), std::runtime_error,
692 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
693 "reverse communication when original data are blocked by process.");
694 if (reverseDistributor_.is_null()) {
695 createReverseDistributor();
697 reverseDistributor_->doPosts(exports, numPackets, imports);
700 template <
class ExpView,
class ImpView>
701 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
703 doReversePosts(
const ExpView& exports,
704 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
705 const ImpView& imports,
706 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID) {
708 TEUCHOS_TEST_FOR_EXCEPTION(
709 !plan_.getIndicesTo().is_null(), std::runtime_error,
710 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
711 "reverse communication when original data are blocked by process.");
712 if (reverseDistributor_.is_null()) {
713 createReverseDistributor();
715 reverseDistributor_->doPosts(exports, numExportPacketsPerLID,
716 imports, numImportPacketsPerLID);
719 template <
class OrdinalType>
721 computeSends(
const Teuchos::ArrayView<const OrdinalType>& importGIDs,
722 const Teuchos::ArrayView<const int>& importProcIDs,
723 Teuchos::Array<OrdinalType>& exportGIDs,
724 Teuchos::Array<int>& exportProcIDs) {
733 using Teuchos::ArrayView;
734 using size_type =
typename ArrayView<const OrdinalType>::size_type;
735 const char errPrefix[] =
"Tpetra::Distributor::computeSends: ";
736 const char suffix[] =
737 " Please report this bug to the Tpetra developers.";
739 const int myRank = plan_.getComm()->getRank();
741 TEUCHOS_TEST_FOR_EXCEPTION(importGIDs.size() != importProcIDs.size(),
742 std::invalid_argument, errPrefix <<
"On Process " << myRank <<
": importProcIDs.size()=" << importProcIDs.size() <<
" != importGIDs.size()=" << importGIDs.size() <<
".");
744 const size_type numImports = importProcIDs.size();
745 Kokkos::View<size_t*, Kokkos::HostSpace> importObjs(
"importObjs", 2 * numImports);
747 for (size_type i = 0; i < numImports; ++i) {
748 importObjs[2 * i] =
static_cast<size_t>(importGIDs[i]);
749 importObjs[2 * i + 1] =
static_cast<size_t>(myRank);
758 const size_t numExportsAsSizeT =
759 tempPlan.createFromSends(importProcIDs);
760 const size_type numExports =
761 static_cast<size_type
>(numExportsAsSizeT);
762 TEUCHOS_TEST_FOR_EXCEPTION(numExports < 0, std::logic_error, errPrefix <<
"tempPlan.createFromSends() returned numExports=" << numExportsAsSizeT <<
" as a size_t, which overflows to " << numExports <<
" when cast to " << Teuchos::TypeNameTraits<size_type>::name() <<
"." << suffix);
763 TEUCHOS_TEST_FOR_EXCEPTION(size_type(tempPlan.getTotalReceiveLength()) != numExports,
764 std::logic_error, errPrefix <<
"tempPlan.getTotalReceiveLength()=" << tempPlan.getTotalReceiveLength() <<
" != numExports=" << numExports <<
"." << suffix);
766 if (numExports > 0) {
767 exportGIDs.resize(numExports);
768 exportProcIDs.resize(numExports);
779 static_assert(
sizeof(
size_t) >=
sizeof(OrdinalType),
780 "Tpetra::Distributor::computeSends: "
781 "sizeof(size_t) < sizeof(OrdinalType).");
783 TEUCHOS_TEST_FOR_EXCEPTION(tempPlan.getTotalReceiveLength() < size_t(numExports),
785 errPrefix <<
"tempPlan.getTotalReceiveLength()="
786 << tempPlan.getTotalReceiveLength() <<
" < numExports="
787 << numExports <<
"." << suffix);
789 Kokkos::View<size_t*, Kokkos::HostSpace> exportObjs(
"exportObjs", tempPlan.getTotalReceiveLength() * 2);
790 tempPlan.doPostsAndWaits(importObjs, 2, exportObjs);
793 for (size_type i = 0; i < numExports; ++i) {
794 exportGIDs[i] =
static_cast<OrdinalType
>(exportObjs[2 * i]);
795 exportProcIDs[i] =
static_cast<int>(exportObjs[2 * i + 1]);
799 plan_ = tempPlan.plan_;
802 template <
class OrdinalType>
804 createFromRecvs(
const Teuchos::ArrayView<const OrdinalType>& remoteGIDs,
805 const Teuchos::ArrayView<const int>& remoteProcIDs,
806 Teuchos::Array<OrdinalType>& exportGIDs,
807 Teuchos::Array<int>& exportProcIDs) {
809 const char errPrefix[] =
"Tpetra::Distributor::createFromRecvs: ";
810 const int myRank = plan_.getComm()->getRank();
812 std::unique_ptr<std::string> prefix;
814 prefix = createPrefix(
"createFromRecvs");
815 std::ostringstream os;
816 os << *prefix <<
"Start" << endl;
817 std::cerr << os.str();
822 using Teuchos::outArg;
823 using Teuchos::REDUCE_MAX;
824 using Teuchos::reduceAll;
828 (remoteGIDs.size() != remoteProcIDs.size()) ? myRank : -1;
830 reduceAll(*plan_.getComm(), REDUCE_MAX, errProc, outArg(maxErrProc));
831 TEUCHOS_TEST_FOR_EXCEPTION(maxErrProc != -1, std::runtime_error, errPrefix <<
"Lists "
832 "of remote IDs and remote process IDs must have the same "
833 "size on all participating processes. Maximum process ID "
835 << maxErrProc <<
".");
839 TEUCHOS_TEST_FOR_EXCEPTION(remoteGIDs.size() != remoteProcIDs.size(), std::runtime_error,
840 errPrefix <<
"On Process " << myRank <<
": "
842 << remoteGIDs.size() <<
" != remoteProcIDs.size()=" << remoteProcIDs.size() <<
".");
845 computeSends(remoteGIDs, remoteProcIDs, exportGIDs, exportProcIDs);
847 plan_.createFromRecvs(remoteProcIDs);
850 std::ostringstream os;
851 os << *prefix <<
"Done" << endl;
852 std::cerr << os.str();
858 #endif // TPETRA_DISTRIBUTOR_HPP
const Details::DistributorPlan & getPlan() const
Get this Distributor's DistributorPlan.
size_t getNumReceives() const
The number of processes from which we will receive data.
std::string description() const
Return a one-line description of this object.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the reverse communication plan.
Teuchos::RCP< Distributor > getReverse(bool create=true) const
A reverse communication plan Distributor.
EDistributorHowInitialized
Enum indicating how and whether a Distributor was initialized.
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const
List of valid Distributor parameters.
virtual ~Distributor()=default
Destructor (virtual for memory safety).
static bool debug()
Whether Tpetra is in debug mode.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a forward plan, but do not execute the waits yet.
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
Details::EDistributorHowInitialized howInitialized() const
Return an enum indicating whether and how a Distributor was initialized.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a reverse plan, but do not execute the waits yet.
Sets up and executes a communication plan for a Tpetra DistObject.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters.
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the (forward) communication plan.
Stand-alone utility functions and macros.
Teuchos::Array< std::string > distributorSendTypes()
Valid string values for Distributor's "Send type" parameter.
size_t getNumSends() const
The number of processes to which we will send data.
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
size_t getMaxSendLength() const
Maximum number of values this process will send to another single process.
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
Distributor(const Teuchos::RCP< const Teuchos::Comm< int > > &comm)
Construct using the specified communicator and default parameters.
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.