10 #ifndef TPETRA_DISTRIBUTOR_HPP
11 #define TPETRA_DISTRIBUTOR_HPP
13 #include "Tpetra_Details_DistributorActor.hpp"
17 #include "Teuchos_as.hpp"
18 #include "Teuchos_Describable.hpp"
19 #include "Teuchos_ParameterListAcceptorDefaultBase.hpp"
20 #include "Teuchos_VerboseObject.hpp"
23 #include "KokkosCompat_View.hpp"
24 #include "Kokkos_Core.hpp"
25 #include "Kokkos_TeuchosCommAdapters.hpp"
28 #include <type_traits>
103 public Teuchos::Describable,
104 public Teuchos::ParameterListAcceptorDefaultBase {
117 explicit Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm);
130 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
131 const Teuchos::RCP<Teuchos::FancyOStream>& out);
146 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
147 const Teuchos::RCP<Teuchos::ParameterList>& plist);
165 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
166 const Teuchos::RCP<Teuchos::FancyOStream>& out,
167 const Teuchos::RCP<Teuchos::ParameterList>& plist);
224 size_t createFromSends (
const Teuchos::ArrayView<const int>& exportProcIDs);
259 template <
class Ordinal>
262 const Teuchos::ArrayView<const int>& remoteProcIDs,
263 Teuchos::Array<Ordinal>& exportIDs,
264 Teuchos::Array<int>& exportProcIDs);
275 const Teuchos::ArrayView<const int>& remoteProcIDs);
310 Teuchos::ArrayView<const int>
getProcsTo()
const;
335 return plan_.howInitialized();
352 Teuchos::RCP<Distributor>
getReverse(
bool create=
true)
const;
394 template <
class ExpView,
class ImpView>
395 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
397 const ExpView &exports,
399 const ImpView &imports);
422 template <
class ExpView,
class ImpView>
423 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
425 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
426 const ImpView &imports,
427 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
453 template <
class ExpView,
class ImpView>
454 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
455 doPosts (
const ExpView &exports,
457 const ImpView &imports);
477 template <
class ExpView,
class ImpView>
478 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
479 doPosts (
const ExpView &exports,
480 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
481 const ImpView &imports,
482 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
488 template <
class ExpView,
class ImpView>
489 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
492 const ImpView &imports);
498 template <
class ExpView,
class ImpView>
499 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
501 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
502 const ImpView &imports,
503 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
509 template <
class ExpView,
class ImpView>
510 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
513 const ImpView &imports);
519 template <
class ExpView,
class ImpView>
520 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
522 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
523 const ImpView &imports,
524 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
555 describe (Teuchos::FancyOStream& out,
556 const Teuchos::EVerbosityLevel verbLevel =
557 Teuchos::Describable::verbLevel_default)
const;
567 Details::DistributorActor actor_;
573 static bool getVerbose();
579 std::unique_ptr<std::string>
580 createPrefix(
const char methodName[])
const;
583 bool verbose_ = getVerbose();
590 mutable Teuchos::RCP<Distributor> reverseDistributor_;
604 template <
class Ordinal>
605 void computeSends (
const Teuchos::ArrayView<const Ordinal> &remoteGIDs,
606 const Teuchos::ArrayView<const int> &remoteProcIDs,
607 Teuchos::Array<Ordinal> &exportGIDs,
608 Teuchos::Array<int> &exportProcIDs);
611 void createReverseDistributor()
const;
619 localDescribeToString (
const Teuchos::EVerbosityLevel vl)
const;
622 template <
class ExpView,
class ImpView>
623 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
625 doPostsAndWaits (
const ExpView& exports,
627 const ImpView& imports)
629 actor_.doPostsAndWaits(plan_, exports, numPackets, imports);
632 template <
class ExpView,
class ImpView>
633 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
635 doPostsAndWaits(
const ExpView& exports,
636 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
637 const ImpView& imports,
638 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
640 actor_.
doPostsAndWaits(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
644 template <
class ExpView,
class ImpView>
645 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
647 doPosts (
const ExpView &exports,
649 const ImpView &imports)
651 actor_.
doPosts(plan_, exports, numPackets, imports);
654 template <
class ExpView,
class ImpView>
655 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
657 doPosts (
const ExpView &exports,
658 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
659 const ImpView &imports,
660 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
662 actor_.
doPosts(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
665 template <
class ExpView,
class ImpView>
666 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
668 doReversePostsAndWaits (
const ExpView& exports,
670 const ImpView& imports)
672 doReversePosts (exports, numPackets, imports);
676 template <
class ExpView,
class ImpView>
677 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
679 doReversePostsAndWaits (
const ExpView& exports,
680 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
681 const ImpView& imports,
682 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
684 doReversePosts (exports, numExportPacketsPerLID, imports,
685 numImportPacketsPerLID);
689 template <
class ExpView,
class ImpView>
690 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
692 doReversePosts (
const ExpView &exports,
694 const ImpView &imports)
697 TEUCHOS_TEST_FOR_EXCEPTION(
698 ! plan_.getIndicesTo().is_null(), std::runtime_error,
699 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
700 "reverse communication when original data are blocked by process.");
701 if (reverseDistributor_.is_null ()) {
702 createReverseDistributor ();
704 reverseDistributor_->doPosts (exports, numPackets, imports);
707 template <
class ExpView,
class ImpView>
708 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
710 doReversePosts (
const ExpView &exports,
711 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
712 const ImpView &imports,
713 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
716 TEUCHOS_TEST_FOR_EXCEPTION(
717 ! plan_.getIndicesTo().is_null(), std::runtime_error,
718 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
719 "reverse communication when original data are blocked by process.");
720 if (reverseDistributor_.is_null ()) {
721 createReverseDistributor ();
723 reverseDistributor_->doPosts (exports, numExportPacketsPerLID,
724 imports, numImportPacketsPerLID);
727 template <
class OrdinalType>
729 computeSends(
const Teuchos::ArrayView<const OrdinalType>& importGIDs,
730 const Teuchos::ArrayView<const int>& importProcIDs,
731 Teuchos::Array<OrdinalType>& exportGIDs,
732 Teuchos::Array<int>& exportProcIDs)
741 using Teuchos::ArrayView;
743 using size_type =
typename ArrayView<const OrdinalType>::size_type;
744 const char errPrefix[] =
"Tpetra::Distributor::computeSends: ";
745 const char suffix[] =
746 " Please report this bug to the Tpetra developers.";
748 const int myRank = plan_.getComm()->getRank ();
750 TEUCHOS_TEST_FOR_EXCEPTION
751 (importGIDs.size () != importProcIDs.size (),
752 std::invalid_argument, errPrefix <<
"On Process " << myRank
753 <<
": importProcIDs.size()=" << importProcIDs.size()
754 <<
" != importGIDs.size()=" << importGIDs.size() <<
".");
756 const size_type numImports = importProcIDs.size();
757 Kokkos::View<size_t*, Kokkos::HostSpace> importObjs(
"importObjs", 2*numImports);
759 for (size_type i = 0; i < numImports; ++i) {
760 importObjs[2*i] =
static_cast<size_t>(importGIDs[i]);
761 importObjs[2*i+1] =
static_cast<size_t>(myRank);
770 const size_t numExportsAsSizeT =
771 tempPlan.createFromSends(importProcIDs);
772 const size_type numExports =
773 static_cast<size_type
>(numExportsAsSizeT);
774 TEUCHOS_TEST_FOR_EXCEPTION
775 (numExports < 0, std::logic_error, errPrefix <<
776 "tempPlan.createFromSends() returned numExports="
777 << numExportsAsSizeT <<
" as a size_t, which overflows to "
778 << numExports <<
" when cast to " <<
779 Teuchos::TypeNameTraits<size_type>::name () <<
"." << suffix);
780 TEUCHOS_TEST_FOR_EXCEPTION
781 (size_type(tempPlan.getTotalReceiveLength()) != numExports,
782 std::logic_error, errPrefix <<
"tempPlan.getTotalReceiveLength()="
783 << tempPlan.getTotalReceiveLength () <<
" != numExports="
784 << numExports <<
"." << suffix);
786 if (numExports > 0) {
787 exportGIDs.resize(numExports);
788 exportProcIDs.resize(numExports);
799 static_assert(
sizeof(
size_t) >=
sizeof(OrdinalType),
800 "Tpetra::Distributor::computeSends: "
801 "sizeof(size_t) < sizeof(OrdinalType).");
803 TEUCHOS_TEST_FOR_EXCEPTION
804 (tempPlan.getTotalReceiveLength () < size_t(numExports),
806 errPrefix <<
"tempPlan.getTotalReceiveLength()="
807 << tempPlan.getTotalReceiveLength() <<
" < numExports="
808 << numExports <<
"." << suffix);
810 Kokkos::View<size_t*, Kokkos::HostSpace> exportObjs(
"exportObjs", tempPlan.getTotalReceiveLength() * 2);
811 tempPlan.doPostsAndWaits(importObjs, 2, exportObjs);
814 for (size_type i = 0; i < numExports; ++i) {
815 exportGIDs[i] =
static_cast<OrdinalType
> (exportObjs[2*i]);
816 exportProcIDs[i] =
static_cast<int> (exportObjs[2*i+1]);
820 template <
class OrdinalType>
822 createFromRecvs (
const Teuchos::ArrayView<const OrdinalType> &remoteGIDs,
823 const Teuchos::ArrayView<const int> &remoteProcIDs,
824 Teuchos::Array<OrdinalType> &exportGIDs,
825 Teuchos::Array<int> &exportProcIDs)
828 const char errPrefix[] =
"Tpetra::Distributor::createFromRecvs: ";
829 const int myRank = plan_.getComm()->getRank();
831 std::unique_ptr<std::string> prefix;
833 prefix = createPrefix(
"createFromRecvs");
834 std::ostringstream os;
835 os << *prefix <<
"Start" << endl;
836 std::cerr << os.str();
841 using Teuchos::outArg;
842 using Teuchos::REDUCE_MAX;
843 using Teuchos::reduceAll;
847 (remoteGIDs.size () != remoteProcIDs.size ()) ? myRank : -1;
849 reduceAll(*plan_.getComm(), REDUCE_MAX, errProc, outArg(maxErrProc));
850 TEUCHOS_TEST_FOR_EXCEPTION
851 (maxErrProc != -1, std::runtime_error, errPrefix <<
"Lists "
852 "of remote IDs and remote process IDs must have the same "
853 "size on all participating processes. Maximum process ID "
854 "with error: " << maxErrProc <<
".");
859 TEUCHOS_TEST_FOR_EXCEPTION
860 (remoteGIDs.size() != remoteProcIDs.size(), std::runtime_error,
861 errPrefix <<
"On Process " << myRank <<
": "
862 "remoteGIDs.size()=" << remoteGIDs.size() <<
863 " != remoteProcIDs.size()=" << remoteProcIDs.size() <<
".");
866 computeSends(remoteGIDs, remoteProcIDs, exportGIDs, exportProcIDs);
868 plan_.createFromRecvs(remoteProcIDs);
871 std::ostringstream os;
872 os << *prefix <<
"Done" << endl;
873 std::cerr << os.str();
879 #endif // TPETRA_DISTRIBUTOR_HPP
const Details::DistributorPlan & getPlan() const
Get this Distributor's DistributorPlan.
size_t getNumReceives() const
The number of processes from which we will receive data.
std::string description() const
Return a one-line description of this object.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the reverse communication plan.
Teuchos::RCP< Distributor > getReverse(bool create=true) const
A reverse communication plan Distributor.
EDistributorHowInitialized
Enum indicating how and whether a Distributor was initialized.
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const
List of valid Distributor parameters.
virtual ~Distributor()=default
Destructor (virtual for memory safety).
static bool debug()
Whether Tpetra is in debug mode.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a forward plan, but do not execute the waits yet.
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
Details::EDistributorHowInitialized howInitialized() const
Return an enum indicating whether and how a Distributor was initialized.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a reverse plan, but do not execute the waits yet.
Sets up and executes a communication plan for a Tpetra DistObject.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters.
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
Teuchos::Array< std::string > distributorSendTypes()
Valid values for Distributor's "Send type" parameter.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the (forward) communication plan.
Stand-alone utility functions and macros.
size_t getNumSends() const
The number of processes to which we will send data.
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
size_t getMaxSendLength() const
Maximum number of values this process will send to another single process.
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
Distributor(const Teuchos::RCP< const Teuchos::Comm< int > > &comm)
Construct using the specified communicator and default parameters.
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.