10 #ifndef TPETRA_DISTRIBUTOR_HPP
11 #define TPETRA_DISTRIBUTOR_HPP
13 #include "Tpetra_Details_DistributorActor.hpp"
17 #include "Teuchos_as.hpp"
18 #include "Teuchos_Describable.hpp"
19 #include "Teuchos_ParameterListAcceptorDefaultBase.hpp"
20 #include "Teuchos_VerboseObject.hpp"
23 #include "KokkosCompat_View.hpp"
24 #include "Kokkos_Core.hpp"
25 #include "Kokkos_TeuchosCommAdapters.hpp"
28 #include <type_traits>
32 #if defined (TPETRA_ENABLE_DEPRECATED_CODE)
105 public Teuchos::Describable,
106 public Teuchos::ParameterListAcceptorDefaultBase {
119 explicit Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm);
132 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
133 const Teuchos::RCP<Teuchos::FancyOStream>& out);
148 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
149 const Teuchos::RCP<Teuchos::ParameterList>& plist);
167 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
168 const Teuchos::RCP<Teuchos::FancyOStream>& out,
169 const Teuchos::RCP<Teuchos::ParameterList>& plist);
226 size_t createFromSends (
const Teuchos::ArrayView<const int>& exportProcIDs);
261 template <
class Ordinal>
264 const Teuchos::ArrayView<const int>& remoteProcIDs,
265 Teuchos::Array<Ordinal>& exportIDs,
266 Teuchos::Array<int>& exportProcIDs);
277 const Teuchos::ArrayView<const int>& remoteProcIDs);
312 Teuchos::ArrayView<const int>
getProcsTo()
const;
337 return plan_.howInitialized();
354 Teuchos::RCP<Distributor>
getReverse(
bool create=
true)
const;
396 template <
class ExpView,
class ImpView>
397 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
399 const ExpView &exports,
401 const ImpView &imports);
424 template <
class ExpView,
class ImpView>
425 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
427 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
428 const ImpView &imports,
429 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
455 template <
class ExpView,
class ImpView>
456 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
457 doPosts (
const ExpView &exports,
459 const ImpView &imports);
479 template <
class ExpView,
class ImpView>
480 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
481 doPosts (
const ExpView &exports,
482 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
483 const ImpView &imports,
484 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
490 template <
class ExpView,
class ImpView>
491 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
494 const ImpView &imports);
500 template <
class ExpView,
class ImpView>
501 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
503 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
504 const ImpView &imports,
505 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
511 template <
class ExpView,
class ImpView>
512 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
515 const ImpView &imports);
521 template <
class ExpView,
class ImpView>
522 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
524 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
525 const ImpView &imports,
526 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
557 describe (Teuchos::FancyOStream& out,
558 const Teuchos::EVerbosityLevel verbLevel =
559 Teuchos::Describable::verbLevel_default)
const;
569 Details::DistributorActor actor_;
575 static bool getVerbose();
581 std::unique_ptr<std::string>
582 createPrefix(
const char methodName[])
const;
585 bool verbose_ = getVerbose();
592 mutable Teuchos::RCP<Distributor> reverseDistributor_;
606 template <
class Ordinal>
607 void computeSends (
const Teuchos::ArrayView<const Ordinal> &remoteGIDs,
608 const Teuchos::ArrayView<const int> &remoteProcIDs,
609 Teuchos::Array<Ordinal> &exportGIDs,
610 Teuchos::Array<int> &exportProcIDs);
613 void createReverseDistributor()
const;
621 localDescribeToString (
const Teuchos::EVerbosityLevel vl)
const;
624 template <
class ExpView,
class ImpView>
625 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
627 doPostsAndWaits (
const ExpView& exports,
629 const ImpView& imports)
631 actor_.doPostsAndWaits(plan_, exports, numPackets, imports);
634 template <
class ExpView,
class ImpView>
635 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
637 doPostsAndWaits(
const ExpView& exports,
638 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
639 const ImpView& imports,
640 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
642 actor_.
doPostsAndWaits(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
646 template <
class ExpView,
class ImpView>
647 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
649 doPosts (
const ExpView &exports,
651 const ImpView &imports)
653 actor_.
doPosts(plan_, exports, numPackets, imports);
656 template <
class ExpView,
class ImpView>
657 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
659 doPosts (
const ExpView &exports,
660 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
661 const ImpView &imports,
662 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
664 actor_.
doPosts(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
667 template <
class ExpView,
class ImpView>
668 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
670 doReversePostsAndWaits (
const ExpView& exports,
672 const ImpView& imports)
674 doReversePosts (exports, numPackets, imports);
678 template <
class ExpView,
class ImpView>
679 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
681 doReversePostsAndWaits (
const ExpView& exports,
682 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
683 const ImpView& imports,
684 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
686 doReversePosts (exports, numExportPacketsPerLID, imports,
687 numImportPacketsPerLID);
691 template <
class ExpView,
class ImpView>
692 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
694 doReversePosts (
const ExpView &exports,
696 const ImpView &imports)
699 TEUCHOS_TEST_FOR_EXCEPTION(
700 ! plan_.getIndicesTo().is_null(), std::runtime_error,
701 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
702 "reverse communication when original data are blocked by process.");
703 if (reverseDistributor_.is_null ()) {
704 createReverseDistributor ();
706 reverseDistributor_->doPosts (exports, numPackets, imports);
709 template <
class ExpView,
class ImpView>
710 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
712 doReversePosts (
const ExpView &exports,
713 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
714 const ImpView &imports,
715 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
718 TEUCHOS_TEST_FOR_EXCEPTION(
719 ! plan_.getIndicesTo().is_null(), std::runtime_error,
720 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
721 "reverse communication when original data are blocked by process.");
722 if (reverseDistributor_.is_null ()) {
723 createReverseDistributor ();
725 reverseDistributor_->doPosts (exports, numExportPacketsPerLID,
726 imports, numImportPacketsPerLID);
729 template <
class OrdinalType>
731 computeSends(
const Teuchos::ArrayView<const OrdinalType>& importGIDs,
732 const Teuchos::ArrayView<const int>& importProcIDs,
733 Teuchos::Array<OrdinalType>& exportGIDs,
734 Teuchos::Array<int>& exportProcIDs)
743 using Teuchos::ArrayView;
745 using size_type =
typename ArrayView<const OrdinalType>::size_type;
746 const char errPrefix[] =
"Tpetra::Distributor::computeSends: ";
747 const char suffix[] =
748 " Please report this bug to the Tpetra developers.";
750 const int myRank = plan_.getComm()->getRank ();
752 TEUCHOS_TEST_FOR_EXCEPTION
753 (importGIDs.size () != importProcIDs.size (),
754 std::invalid_argument, errPrefix <<
"On Process " << myRank
755 <<
": importProcIDs.size()=" << importProcIDs.size()
756 <<
" != importGIDs.size()=" << importGIDs.size() <<
".");
758 const size_type numImports = importProcIDs.size();
759 Kokkos::View<size_t*, Kokkos::HostSpace> importObjs(
"importObjs", 2*numImports);
761 for (size_type i = 0; i < numImports; ++i) {
762 importObjs[2*i] =
static_cast<size_t>(importGIDs[i]);
763 importObjs[2*i+1] =
static_cast<size_t>(myRank);
772 const size_t numExportsAsSizeT =
773 tempPlan.createFromSends(importProcIDs);
774 const size_type numExports =
775 static_cast<size_type
>(numExportsAsSizeT);
776 TEUCHOS_TEST_FOR_EXCEPTION
777 (numExports < 0, std::logic_error, errPrefix <<
778 "tempPlan.createFromSends() returned numExports="
779 << numExportsAsSizeT <<
" as a size_t, which overflows to "
780 << numExports <<
" when cast to " <<
781 Teuchos::TypeNameTraits<size_type>::name () <<
"." << suffix);
782 TEUCHOS_TEST_FOR_EXCEPTION
783 (size_type(tempPlan.getTotalReceiveLength()) != numExports,
784 std::logic_error, errPrefix <<
"tempPlan.getTotalReceiveLength()="
785 << tempPlan.getTotalReceiveLength () <<
" != numExports="
786 << numExports <<
"." << suffix);
788 if (numExports > 0) {
789 exportGIDs.resize(numExports);
790 exportProcIDs.resize(numExports);
801 static_assert(
sizeof(
size_t) >=
sizeof(OrdinalType),
802 "Tpetra::Distributor::computeSends: "
803 "sizeof(size_t) < sizeof(OrdinalType).");
805 TEUCHOS_TEST_FOR_EXCEPTION
806 (tempPlan.getTotalReceiveLength () < size_t(numExports),
808 errPrefix <<
"tempPlan.getTotalReceiveLength()="
809 << tempPlan.getTotalReceiveLength() <<
" < numExports="
810 << numExports <<
"." << suffix);
812 Kokkos::View<size_t*, Kokkos::HostSpace> exportObjs(
"exportObjs", tempPlan.getTotalReceiveLength() * 2);
813 tempPlan.doPostsAndWaits(importObjs, 2, exportObjs);
816 for (size_type i = 0; i < numExports; ++i) {
817 exportGIDs[i] =
static_cast<OrdinalType
> (exportObjs[2*i]);
818 exportProcIDs[i] =
static_cast<int> (exportObjs[2*i+1]);
822 plan_ = tempPlan.plan_;
825 template <
class OrdinalType>
827 createFromRecvs (
const Teuchos::ArrayView<const OrdinalType> &remoteGIDs,
828 const Teuchos::ArrayView<const int> &remoteProcIDs,
829 Teuchos::Array<OrdinalType> &exportGIDs,
830 Teuchos::Array<int> &exportProcIDs)
833 const char errPrefix[] =
"Tpetra::Distributor::createFromRecvs: ";
834 const int myRank = plan_.getComm()->getRank();
836 std::unique_ptr<std::string> prefix;
838 prefix = createPrefix(
"createFromRecvs");
839 std::ostringstream os;
840 os << *prefix <<
"Start" << endl;
841 std::cerr << os.str();
846 using Teuchos::outArg;
847 using Teuchos::REDUCE_MAX;
848 using Teuchos::reduceAll;
852 (remoteGIDs.size () != remoteProcIDs.size ()) ? myRank : -1;
854 reduceAll(*plan_.getComm(), REDUCE_MAX, errProc, outArg(maxErrProc));
855 TEUCHOS_TEST_FOR_EXCEPTION
856 (maxErrProc != -1, std::runtime_error, errPrefix <<
"Lists "
857 "of remote IDs and remote process IDs must have the same "
858 "size on all participating processes. Maximum process ID "
859 "with error: " << maxErrProc <<
".");
864 TEUCHOS_TEST_FOR_EXCEPTION
865 (remoteGIDs.size() != remoteProcIDs.size(), std::runtime_error,
866 errPrefix <<
"On Process " << myRank <<
": "
867 "remoteGIDs.size()=" << remoteGIDs.size() <<
868 " != remoteProcIDs.size()=" << remoteProcIDs.size() <<
".");
871 computeSends(remoteGIDs, remoteProcIDs, exportGIDs, exportProcIDs);
873 plan_.createFromRecvs(remoteProcIDs);
876 std::ostringstream os;
877 os << *prefix <<
"Done" << endl;
878 std::cerr << os.str();
884 #endif // TPETRA_DISTRIBUTOR_HPP
const Details::DistributorPlan & getPlan() const
Get this Distributor's DistributorPlan.
size_t getNumReceives() const
The number of processes from which we will receive data.
std::string description() const
Return a one-line description of this object.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the reverse communication plan.
Teuchos::RCP< Distributor > getReverse(bool create=true) const
A reverse communication plan Distributor.
EDistributorHowInitialized
Enum indicating how and whether a Distributor was initialized.
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const
List of valid Distributor parameters.
virtual ~Distributor()=default
Destructor (virtual for memory safety).
static bool debug()
Whether Tpetra is in debug mode.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a forward plan, but do not execute the waits yet.
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
Details::EDistributorHowInitialized howInitialized() const
Return an enum indicating whether and how a Distributor was initialized.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a reverse plan, but do not execute the waits yet.
Sets up and executes a communication plan for a Tpetra DistObject.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters.
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the (forward) communication plan.
Stand-alone utility functions and macros.
Teuchos::Array< std::string > distributorSendTypes()
Valid string values for Distributor's "Send type" parameter.
size_t getNumSends() const
The number of processes to which we will send data.
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
size_t getMaxSendLength() const
Maximum number of values this process will send to another single process.
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
Distributor(const Teuchos::RCP< const Teuchos::Comm< int > > &comm)
Construct using the specified communicator and default parameters.
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.