12 #include "Teuchos_StandardParameterEntryValidators.hpp"
23 if (sendType == DISTRIBUTOR_ISEND) {
26 else if (sendType == DISTRIBUTOR_SEND) {
29 else if (sendType == DISTRIBUTOR_ALLTOALL) {
32 #if defined(HAVE_TPETRACORE_MPI_ADVANCE)
33 else if (sendType == DISTRIBUTOR_MPIADVANCE_ALLTOALL) {
34 return "MpiAdvanceAlltoall";
36 else if (sendType == DISTRIBUTOR_MPIADVANCE_NBRALLTOALLV) {
37 return "MpiAdvanceNbralltoallv";
41 TEUCHOS_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Invalid "
42 "EDistributorSendType enum value " << sendType <<
".");
49 if (s ==
"Isend")
return DISTRIBUTOR_ISEND;
50 if (s ==
"Send")
return DISTRIBUTOR_SEND;
51 if (s ==
"Alltoall")
return DISTRIBUTOR_ALLTOALL;
52 #if defined(HAVE_TPETRACORE_MPI_ADVANCE)
53 if (s ==
"MpiAdvanceAlltoall")
return DISTRIBUTOR_MPIADVANCE_ALLTOALL;
54 if (s ==
"MpiAdvanceNbralltoallv")
return DISTRIBUTOR_MPIADVANCE_NBRALLTOALLV;
56 TEUCHOS_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Invalid string to convert to EDistributorSendType enum value: " << s);
62 if (std::find(valids.begin(), valids.end(), s) == valids.end()) {
63 TEUCHOS_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Invalid string for EDistributorSendType enum value: " << s);
72 case Details::DISTRIBUTOR_NOT_INITIALIZED:
73 return "Not initialized yet";
74 case Details::DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_SENDS:
75 return "By createFromSends";
76 case Details::DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_RECVS:
77 return "By createFromRecvs";
78 case Details::DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_SENDS_N_RECVS:
79 return "By createFromSendsAndRecvs";
80 case Details::DISTRIBUTOR_INITIALIZED_BY_REVERSE:
81 return "By createReverseDistributor";
82 case Details::DISTRIBUTOR_INITIALIZED_BY_COPY:
83 return "By copy constructor";
89 DistributorPlan::DistributorPlan(Teuchos::RCP<
const Teuchos::Comm<int>> comm)
91 #if defined(HAVE_TPETRACORE_MPI_ADVANCE)
92 mpixComm_(Teuchos::null),
94 howInitialized_(DISTRIBUTOR_NOT_INITIALIZED),
95 reversePlan_(Teuchos::null),
97 sendMessageToSelf_(false),
98 numSendsToOtherProcs_(0),
101 totalReceiveLength_(0)
104 DistributorPlan::DistributorPlan(
const DistributorPlan& otherPlan)
105 : comm_(otherPlan.comm_),
106 #if defined(HAVE_TPETRACORE_MPI_ADVANCE)
107 mpixComm_(otherPlan.mpixComm_),
109 howInitialized_(DISTRIBUTOR_INITIALIZED_BY_COPY),
110 reversePlan_(otherPlan.reversePlan_),
111 sendType_(otherPlan.sendType_),
112 sendMessageToSelf_(otherPlan.sendMessageToSelf_),
113 numSendsToOtherProcs_(otherPlan.numSendsToOtherProcs_),
114 procIdsToSendTo_(otherPlan.procIdsToSendTo_),
115 startsTo_(otherPlan.startsTo_),
116 lengthsTo_(otherPlan.lengthsTo_),
117 maxSendLength_(otherPlan.maxSendLength_),
118 indicesTo_(otherPlan.indicesTo_),
119 numReceives_(otherPlan.numReceives_),
120 totalReceiveLength_(otherPlan.totalReceiveLength_),
121 lengthsFrom_(otherPlan.lengthsFrom_),
122 procsFrom_(otherPlan.procsFrom_),
123 startsFrom_(otherPlan.startsFrom_),
124 indicesFrom_(otherPlan.indicesFrom_)
127 size_t DistributorPlan::createFromSends(
const Teuchos::ArrayView<const int>& exportProcIDs) {
128 using Teuchos::outArg;
129 using Teuchos::REDUCE_MAX;
130 using Teuchos::reduceAll;
132 const char rawPrefix[] =
"Tpetra::DistributorPlan::createFromSends";
134 const size_t numExports = exportProcIDs.size();
135 const int myProcID = comm_->getRank();
136 const int numProcs = comm_->getSize();
180 for (
size_t i = 0; i < numExports; ++i) {
181 const int exportID = exportProcIDs[i];
182 if (exportID >= numProcs || exportID < 0) {
188 reduceAll<int, int> (*comm_, REDUCE_MAX, badID, outArg (gbl_badID));
189 TEUCHOS_TEST_FOR_EXCEPTION
190 (gbl_badID >= 0, std::runtime_error, rawPrefix <<
"Proc "
191 << gbl_badID <<
", perhaps among other processes, got a bad "
207 Teuchos::Array<size_t> starts (numProcs + 1, 0);
210 size_t numActive = 0;
211 int needSendBuff = 0;
213 for (
size_t i = 0; i < numExports; ++i) {
214 const int exportID = exportProcIDs[i];
229 if (needSendBuff == 0 && starts[exportID] > 1 &&
230 exportID != exportProcIDs[i-1]) {
237 #if defined(HAVE_TPETRA_PRINT_EFFICIENCY_WARNINGS)
239 int global_needSendBuff;
240 reduceAll<int, int> (*comm_, REDUCE_MAX, needSendBuff,
241 outArg (global_needSendBuff));
243 global_needSendBuff != 0,
244 "::createFromSends: Grouping export IDs together by process rank often "
245 "improves performance.");
251 if (starts[myProcID] != 0) {
252 sendMessageToSelf_ =
true;
255 sendMessageToSelf_ =
false;
258 if (! needSendBuff) {
260 numSendsToOtherProcs_ = 0;
263 for (
int i = 0; i < numProcs; ++i) {
265 ++numSendsToOtherProcs_;
271 indicesTo_.resize(0);
274 procIdsToSendTo_.assign(numSendsToOtherProcs_,0);
275 startsTo_.assign(numSendsToOtherProcs_,0);
276 lengthsTo_.assign(numSendsToOtherProcs_,0);
283 size_t procIndex = 0;
284 for (
size_t i = 0; i < numSendsToOtherProcs_; ++i) {
285 while (exportProcIDs[procIndex] < 0) {
288 startsTo_[i] = procIndex;
289 int procID = exportProcIDs[procIndex];
290 procIdsToSendTo_[i] = procID;
291 procIndex += starts[procID];
296 if (numSendsToOtherProcs_ > 0) {
297 sort2(procIdsToSendTo_.begin(), procIdsToSendTo_.end(), startsTo_.begin());
301 for (
size_t i = 0; i < numSendsToOtherProcs_; ++i) {
302 int procID = procIdsToSendTo_[i];
303 lengthsTo_[i] = starts[procID];
304 if ((procID != myProcID) && (lengthsTo_[i] > maxSendLength_)) {
305 maxSendLength_ = lengthsTo_[i];
316 if (starts[0] == 0 ) {
317 numSendsToOtherProcs_ = 0;
320 numSendsToOtherProcs_ = 1;
322 for (Teuchos::Array<size_t>::iterator i=starts.begin()+1,
324 i != starts.end(); ++i)
326 if (*i != 0) ++numSendsToOtherProcs_;
332 for (Teuchos::Array<size_t>::reverse_iterator ip1=starts.rbegin(),
334 i != starts.rend(); ++i)
343 indicesTo_.resize(numActive);
345 for (
size_t i = 0; i < numExports; ++i) {
346 if (exportProcIDs[i] >= 0) {
348 indicesTo_[starts[exportProcIDs[i]]] = i;
350 ++starts[exportProcIDs[i]];
362 for (
int proc = numProcs-1; proc != 0; --proc) {
363 starts[proc] = starts[proc-1];
366 starts[numProcs] = numActive;
373 procIdsToSendTo_.resize(numSendsToOtherProcs_);
374 startsTo_.resize(numSendsToOtherProcs_);
375 lengthsTo_.resize(numSendsToOtherProcs_);
382 for (
int proc = 0; proc < numProcs; ++proc ) {
383 if (starts[proc+1] != starts[proc]) {
384 lengthsTo_[snd] = starts[proc+1] - starts[proc];
385 startsTo_[snd] = starts[proc];
387 if ((proc != myProcID) && (lengthsTo_[snd] > maxSendLength_)) {
388 maxSendLength_ = lengthsTo_[snd];
390 procIdsToSendTo_[snd] = proc;
396 if (sendMessageToSelf_) {
397 --numSendsToOtherProcs_;
403 #if defined(HAVE_TPETRACORE_MPI_ADVANCE)
404 initializeMpiAdvance();
409 howInitialized_ = Details::DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_SENDS;
411 return totalReceiveLength_;
414 void DistributorPlan::createFromRecvs(
const Teuchos::ArrayView<const int>& remoteProcIDs)
416 *
this = *getReversePlan();
417 howInitialized_ = Details::DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_RECVS;
420 void DistributorPlan::createFromSendsAndRecvs(
const Teuchos::ArrayView<const int>& exportProcIDs,
421 const Teuchos::ArrayView<const int>& remoteProcIDs)
430 howInitialized_ = Tpetra::Details::DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_SENDS_N_RECVS;
432 int myProcID = comm_->getRank ();
433 int numProcs = comm_->getSize();
435 const size_t numExportIDs = exportProcIDs.size();
436 Teuchos::Array<size_t> starts (numProcs + 1, 0);
438 size_t numActive = 0;
439 int needSendBuff = 0;
441 for(
size_t i = 0; i < numExportIDs; i++ )
443 if( needSendBuff==0 && i && (exportProcIDs[i] < exportProcIDs[i-1]) )
445 if( exportProcIDs[i] >= 0 )
447 ++starts[ exportProcIDs[i] ];
452 sendMessageToSelf_ = ( starts[myProcID] != 0 ) ? 1 : 0;
454 numSendsToOtherProcs_ = 0;
458 if (starts[0] == 0 ) {
459 numSendsToOtherProcs_ = 0;
462 numSendsToOtherProcs_ = 1;
464 for (Teuchos::Array<size_t>::iterator i=starts.begin()+1,
466 i != starts.end(); ++i)
468 if (*i != 0) ++numSendsToOtherProcs_;
474 for (Teuchos::Array<size_t>::reverse_iterator ip1=starts.rbegin(),
476 i != starts.rend(); ++i)
485 indicesTo_.resize(numActive);
487 for (
size_t i = 0; i < numExportIDs; ++i) {
488 if (exportProcIDs[i] >= 0) {
490 indicesTo_[starts[exportProcIDs[i]]] = i;
492 ++starts[exportProcIDs[i]];
495 for (
int proc = numProcs-1; proc != 0; --proc) {
496 starts[proc] = starts[proc-1];
499 starts[numProcs] = numActive;
500 procIdsToSendTo_.resize(numSendsToOtherProcs_);
501 startsTo_.resize(numSendsToOtherProcs_);
502 lengthsTo_.resize(numSendsToOtherProcs_);
505 for (
int proc = 0; proc < numProcs; ++proc ) {
506 if (starts[proc+1] != starts[proc]) {
507 lengthsTo_[snd] = starts[proc+1] - starts[proc];
508 startsTo_[snd] = starts[proc];
510 if ((proc != myProcID) && (lengthsTo_[snd] > maxSendLength_)) {
511 maxSendLength_ = lengthsTo_[snd];
513 procIdsToSendTo_[snd] = proc;
520 numSendsToOtherProcs_ = 0;
523 for (
int i = 0; i < numProcs; ++i) {
525 ++numSendsToOtherProcs_;
531 indicesTo_.resize(0);
534 procIdsToSendTo_.assign(numSendsToOtherProcs_,0);
535 startsTo_.assign(numSendsToOtherProcs_,0);
536 lengthsTo_.assign(numSendsToOtherProcs_,0);
543 size_t procIndex = 0;
544 for (
size_t i = 0; i < numSendsToOtherProcs_; ++i) {
545 while (exportProcIDs[procIndex] < 0) {
548 startsTo_[i] = procIndex;
549 int procID = exportProcIDs[procIndex];
550 procIdsToSendTo_[i] = procID;
551 procIndex += starts[procID];
556 if (numSendsToOtherProcs_ > 0) {
557 sort2(procIdsToSendTo_.begin(), procIdsToSendTo_.end(), startsTo_.begin());
561 for (
size_t i = 0; i < numSendsToOtherProcs_; ++i) {
562 int procID = procIdsToSendTo_[i];
563 lengthsTo_[i] = starts[procID];
564 if ((procID != myProcID) && (lengthsTo_[i] > maxSendLength_)) {
565 maxSendLength_ = lengthsTo_[i];
571 numSendsToOtherProcs_ -= sendMessageToSelf_;
572 std::vector<int> recv_list;
573 recv_list.reserve(numSendsToOtherProcs_);
576 for(
int i=0; i<remoteProcIDs.size(); i++) {
577 if(remoteProcIDs[i]>last_pid) {
578 recv_list.push_back(remoteProcIDs[i]);
579 last_pid = remoteProcIDs[i];
581 else if (remoteProcIDs[i]<last_pid)
582 throw std::runtime_error(
"Tpetra::Distributor:::createFromSendsAndRecvs expected RemotePIDs to be in sorted order");
584 numReceives_ = recv_list.size();
586 procsFrom_.assign(numReceives_,0);
587 lengthsFrom_.assign(numReceives_,0);
588 indicesFrom_.assign(numReceives_,0);
589 startsFrom_.assign(numReceives_,0);
591 for(
size_t i=0,j=0; i<numReceives_; ++i) {
593 procsFrom_[i] = recv_list[i];
595 for( ; j<(size_t)remoteProcIDs.size() &&
596 remoteProcIDs[jlast]==remoteProcIDs[j] ; j++){;}
597 lengthsFrom_[i] = j-jlast;
599 totalReceiveLength_ = remoteProcIDs.size();
600 indicesFrom_.clear ();
601 numReceives_-=sendMessageToSelf_;
603 #if defined(HAVE_TPETRACORE_MPI_ADVANCE)
604 initializeMpiAdvance();
608 Teuchos::RCP<DistributorPlan> DistributorPlan::getReversePlan()
const {
609 if (reversePlan_.is_null()) createReversePlan();
613 void DistributorPlan::createReversePlan()
const
615 reversePlan_ = Teuchos::rcp(
new DistributorPlan(comm_));
616 reversePlan_->howInitialized_ = Details::DISTRIBUTOR_INITIALIZED_BY_REVERSE;
617 reversePlan_->sendType_ = sendType_;
622 size_t totalSendLength =
623 std::accumulate(lengthsTo_.begin(), lengthsTo_.end(), 0);
628 size_t maxReceiveLength = 0;
629 const int myProcID = comm_->getRank();
630 for (
size_t i=0; i < numReceives_; ++i) {
631 if (procsFrom_[i] != myProcID) {
633 if (lengthsFrom_[i] > maxReceiveLength) {
634 maxReceiveLength = lengthsFrom_[i];
639 reversePlan_->sendMessageToSelf_ = sendMessageToSelf_;
640 reversePlan_->numSendsToOtherProcs_ = numReceives_;
641 reversePlan_->procIdsToSendTo_ = procsFrom_;
642 reversePlan_->startsTo_ = startsFrom_;
643 reversePlan_->lengthsTo_ = lengthsFrom_;
644 reversePlan_->maxSendLength_ = maxReceiveLength;
645 reversePlan_->indicesTo_ = indicesFrom_;
646 reversePlan_->numReceives_ = numSendsToOtherProcs_;
647 reversePlan_->totalReceiveLength_ = totalSendLength;
648 reversePlan_->lengthsFrom_ = lengthsTo_;
649 reversePlan_->procsFrom_ = procIdsToSendTo_;
650 reversePlan_->startsFrom_ = startsTo_;
651 reversePlan_->indicesFrom_ = indicesTo_;
653 #if defined(HAVE_TPETRACORE_MPI_ADVANCE)
655 reversePlan_->initializeMpiAdvance();
659 void DistributorPlan::computeReceives()
661 using Teuchos::Array;
662 using Teuchos::ArrayRCP;
664 using Teuchos::CommStatus;
665 using Teuchos::CommRequest;
666 using Teuchos::ireceive;
669 using Teuchos::REDUCE_SUM;
670 using Teuchos::receive;
671 using Teuchos::reduce;
672 using Teuchos::scatter;
674 using Teuchos::waitAll;
676 const int myRank = comm_->getRank();
677 const int numProcs = comm_->getSize();
679 const int mpiTag = DEFAULT_MPI_TAG;
687 Array<int> toProcsFromMe (numProcs, 0);
688 #ifdef HAVE_TPETRA_DEBUG
689 bool counting_error =
false;
690 #endif // HAVE_TPETRA_DEBUG
691 for (
size_t i = 0; i < (numSendsToOtherProcs_ + (sendMessageToSelf_ ? 1 : 0)); ++i) {
692 #ifdef HAVE_TPETRA_DEBUG
693 if (toProcsFromMe[procIdsToSendTo_[i]] != 0) {
694 counting_error =
true;
696 #endif // HAVE_TPETRA_DEBUG
697 toProcsFromMe[procIdsToSendTo_[i]] = 1;
699 #ifdef HAVE_TPETRA_DEBUG
702 "Tpetra::Distributor::computeReceives: There was an error on at least "
703 "one process in counting the number of messages send by that process to "
704 "the other processs. Please report this bug to the Tpetra developers.",
706 #endif // HAVE_TPETRA_DEBUG
761 Array<int> numRecvsOnEachProc;
762 if (myRank == root) {
763 numRecvsOnEachProc.resize (numProcs);
765 int numReceivesAsInt = 0;
766 reduce<int, int> (toProcsFromMe.getRawPtr (),
767 numRecvsOnEachProc.getRawPtr (),
768 numProcs, REDUCE_SUM, root, *comm_);
769 scatter<int, int> (numRecvsOnEachProc.getRawPtr (), 1,
770 &numReceivesAsInt, 1, root, *comm_);
771 numReceives_ =
static_cast<size_t> (numReceivesAsInt);
777 lengthsFrom_.assign (numReceives_, 0);
778 procsFrom_.assign (numReceives_, 0);
794 const size_t actualNumReceives = numReceives_ - (sendMessageToSelf_ ? 1 : 0);
800 Array<RCP<CommRequest<int> > > requests (actualNumReceives);
801 Array<ArrayRCP<size_t> > lengthsFromBuffers (actualNumReceives);
802 Array<RCP<CommStatus<int> > > statuses (actualNumReceives);
807 const int anySourceProc = MPI_ANY_SOURCE;
809 const int anySourceProc = -1;
813 for (
size_t i = 0; i < actualNumReceives; ++i) {
818 lengthsFromBuffers[i].resize (1);
819 lengthsFromBuffers[i][0] = as<size_t> (0);
820 requests[i] = ireceive<int, size_t> (lengthsFromBuffers[i], anySourceProc,
832 for (
size_t i = 0; i < numSendsToOtherProcs_ + (sendMessageToSelf_ ? 1 : 0); ++i) {
833 if (procIdsToSendTo_[i] != myRank) {
837 const size_t*
const lengthsTo_i = &lengthsTo_[i];
838 send<int, size_t> (lengthsTo_i, 1, as<int> (procIdsToSendTo_[i]), mpiTag, *comm_);
847 lengthsFrom_[numReceives_-1] = lengthsTo_[i];
848 procsFrom_[numReceives_-1] = myRank;
858 waitAll (*comm_, requests (), statuses ());
859 for (
size_t i = 0; i < actualNumReceives; ++i) {
860 lengthsFrom_[i] = *lengthsFromBuffers[i];
861 procsFrom_[i] = statuses[i]->getSourceRank ();
867 sort2 (procsFrom_.begin(), procsFrom_.end(), lengthsFrom_.begin());
870 totalReceiveLength_ =
871 std::accumulate (lengthsFrom_.begin (), lengthsFrom_.end (), 0);
872 indicesFrom_.clear ();
874 startsFrom_.clear ();
875 startsFrom_.reserve (numReceives_);
876 for (
size_t i = 0, j = 0; i < numReceives_; ++i) {
877 startsFrom_.push_back(j);
878 j += lengthsFrom_[i];
881 if (sendMessageToSelf_) {
886 void DistributorPlan::setParameterList(
const Teuchos::RCP<Teuchos::ParameterList>& plist)
888 using Teuchos::FancyOStream;
889 using Teuchos::getIntegralValue;
890 using Teuchos::ParameterList;
891 using Teuchos::parameterList;
895 if (! plist.is_null()) {
896 RCP<const ParameterList> validParams = getValidParameters ();
897 plist->validateParametersAndSetDefaults (*validParams);
900 getIntegralValue<Details::EDistributorSendType> (*plist,
"Send type");
903 sendType_ = sendType;
907 this->setMyParamList (plist);
913 Teuchos::Array<std::string> sendTypes;
914 sendTypes.push_back (
"Isend");
915 sendTypes.push_back (
"Send");
916 sendTypes.push_back (
"Alltoall");
917 #if defined(HAVE_TPETRACORE_MPI_ADVANCE)
918 sendTypes.push_back (
"MpiAdvanceAlltoall");
919 sendTypes.push_back (
"MpiAdvanceNbralltoallv");
925 Teuchos::Array<EDistributorSendType> res;
926 res.push_back (DISTRIBUTOR_ISEND);
927 res.push_back (DISTRIBUTOR_SEND);
928 res.push_back (DISTRIBUTOR_ALLTOALL);
929 #if defined(HAVE_TPETRACORE_MPI_ADVANCE)
930 res.push_back (DISTRIBUTOR_MPIADVANCE_ALLTOALL);
931 res.push_back (DISTRIBUTOR_MPIADVANCE_NBRALLTOALLV);
936 Teuchos::RCP<const Teuchos::ParameterList>
937 DistributorPlan::getValidParameters()
const
939 using Teuchos::Array;
940 using Teuchos::ParameterList;
941 using Teuchos::parameterList;
943 using Teuchos::setStringToIntegralParameter;
945 Array<std::string> sendTypes = distributorSendTypes ();
946 const Array<Details::EDistributorSendType> sendTypeEnums = distributorSendTypeEnums ();
948 const std::string validatedSendType = validSendTypeOrThrow(Behavior::defaultSendType());
950 RCP<ParameterList> plist = parameterList (
"Tpetra::Distributor");
952 setStringToIntegralParameter<Details::EDistributorSendType> (
"Send type",
953 validatedSendType,
"When using MPI, the variant of send to use in "
954 "do[Reverse]Posts()", sendTypes(), sendTypeEnums(), plist.getRawPtr());
955 plist->set (
"Timer Label",
"",
"Label for Time Monitor output");
957 return Teuchos::rcp_const_cast<
const ParameterList> (plist);
960 #if defined(HAVE_TPETRACORE_MPI_ADVANCE)
963 struct MpixCommDeallocator {
964 void free(MPIX_Comm **comm)
const {
965 MPIX_Comm_free(*comm);
969 void DistributorPlan::initializeMpiAdvance() {
972 TEUCHOS_ASSERT(mpixComm_.is_null());
975 Teuchos::RCP<const Teuchos::MpiComm<int> > mpiComm = Teuchos::rcp_dynamic_cast<
const Teuchos::MpiComm<int> >(comm_);
976 Teuchos::RCP<const Teuchos::OpaqueWrapper<MPI_Comm> > rawComm = mpiComm->getRawMpiComm();
978 if (sendType_ == DISTRIBUTOR_MPIADVANCE_ALLTOALL) {
979 MPIX_Comm **mpixComm =
new(MPIX_Comm*);
980 err = MPIX_Comm_init(mpixComm, (*rawComm)());
981 mpixComm_ = Teuchos::RCP(mpixComm,
982 MpixCommDeallocator(),
986 else if (sendType_ == DISTRIBUTOR_MPIADVANCE_NBRALLTOALLV) {
987 int numRecvs = (int)(numReceives_ + (sendMessageToSelf_ ? 1 : 0));
988 int *sourceRanks = procsFrom_.data();
991 const int *sourceWeights = MPI_UNWEIGHTED;
992 int numSends = (int)(numSendsToOtherProcs_ + (sendMessageToSelf_ ? 1 : 0));
993 int *destRanks = procIdsToSendTo_.data();
996 const int *destWeights = MPI_UNWEIGHTED;
998 MPIX_Comm **mpixComm =
new(MPIX_Comm*);
999 err = MPIX_Dist_graph_create_adjacent((*rawComm)(), numRecvs, sourceRanks, sourceWeights, numSends, destRanks, destWeights, MPI_INFO_NULL,
false, mpixComm);
1000 mpixComm_ = Teuchos::RCP(mpixComm,
1001 MpixCommDeallocator(),
1006 TEUCHOS_ASSERT(err == 0);
1011 DistributorPlan::SubViewLimits DistributorPlan::getImportViewLimits(
size_t numPackets)
const {
1012 const size_t actualNumReceives = getNumReceives() + (hasSelfMessage() ? 1 : 0);
1014 IndexView importStarts(actualNumReceives);
1015 IndexView importLengths(actualNumReceives);
1018 for (
size_t i = 0; i < actualNumReceives; ++i) {
1019 importStarts[i] = offset;
1020 offset += getLengthsFrom()[i] * numPackets;
1021 importLengths[i] = getLengthsFrom()[i] * numPackets;
1023 return std::make_pair(importStarts, importLengths);
1026 DistributorPlan::SubViewLimits DistributorPlan::getImportViewLimits(
const Teuchos::ArrayView<const size_t> &numImportPacketsPerLID)
const {
1028 const size_t actualNumReceives = getNumReceives() + (hasSelfMessage() ? 1 : 0);
1030 IndexView importStarts(actualNumReceives);
1031 IndexView importLengths(actualNumReceives);
1034 size_t curLIDoffset = 0;
1035 for (
size_t i = 0; i < actualNumReceives; ++i) {
1036 size_t totalPacketsFrom_i = 0;
1037 for (
size_t j = 0; j < getLengthsFrom()[i]; ++j) {
1038 totalPacketsFrom_i += numImportPacketsPerLID[curLIDoffset + j];
1040 curLIDoffset += getLengthsFrom()[i];
1041 importStarts[i] = offset;
1042 offset += totalPacketsFrom_i;
1043 importLengths[i] = totalPacketsFrom_i;
1045 return std::make_pair(importStarts, importLengths);
1049 DistributorPlan::SubViewLimits DistributorPlan::getExportViewLimits(
size_t numPackets)
const {
1050 if (getIndicesTo().is_null()) {
1052 const size_t actualNumSends = getNumSends() + (hasSelfMessage() ? 1 : 0);
1053 IndexView exportStarts(actualNumSends);
1054 IndexView exportLengths(actualNumSends);
1055 for (
size_t pp = 0; pp < actualNumSends; ++pp) {
1056 exportStarts[pp] = getStartsTo()[pp] * numPackets;
1057 exportLengths[pp] = getLengthsTo()[pp] * numPackets;
1059 return std::make_pair(exportStarts, exportLengths);
1061 const size_t numIndices = getIndicesTo().size();
1062 IndexView exportStarts(numIndices);
1063 IndexView exportLengths(numIndices);
1064 for (
size_t j = 0; j < numIndices; ++j) {
1065 exportStarts[j] = getIndicesTo()[j]*numPackets;
1066 exportLengths[j] = numPackets;
1068 return std::make_pair(exportStarts, exportLengths);
1072 DistributorPlan::SubViewLimits DistributorPlan::getExportViewLimits(
const Teuchos::ArrayView<const size_t> &numExportPacketsPerLID)
const {
1073 if (getIndicesTo().is_null()) {
1074 const size_t actualNumSends = getNumSends() + (hasSelfMessage() ? 1 : 0);
1075 IndexView exportStarts(actualNumSends);
1076 IndexView exportLengths(actualNumSends);
1078 for (
size_t pp = 0; pp < actualNumSends; ++pp) {
1079 size_t numPackets = 0;
1080 for (
size_t j = getStartsTo()[pp];
1081 j < getStartsTo()[pp] + getLengthsTo()[pp]; ++j) {
1082 numPackets += numExportPacketsPerLID[j];
1084 exportStarts[pp] = offset;
1085 offset += numPackets;
1086 exportLengths[pp] = numPackets;
1088 return std::make_pair(exportStarts, exportLengths);
1090 const size_t numIndices = getIndicesTo().size();
1091 IndexView exportStarts(numIndices);
1092 IndexView exportLengths(numIndices);
1094 for (
size_t j = 0; j < numIndices; ++j) {
1095 exportStarts[j] = offset;
1096 offset += numExportPacketsPerLID[j];
1097 exportLengths[j] = numExportPacketsPerLID[j];
1099 return std::make_pair(exportStarts, exportLengths);
EDistributorHowInitialized
Enum indicating how and whether a Distributor was initialized.
static bool debug()
Whether Tpetra is in debug mode.
const std::string & validSendTypeOrThrow(const std::string &s)
Valid enum values of distributor send types.
std::string DistributorSendTypeEnumToString(EDistributorSendType sendType)
Convert an EDistributorSendType enum value to a string.
#define TPETRA_EFFICIENCY_WARNING(throw_exception_test, msg)
Print or throw an efficency warning.
EDistributorSendType DistributorSendTypeStringToEnum(const std::string_view s)
Convert a string to an EDistributorSendType. Throw on error.
Teuchos::Array< EDistributorSendType > distributorSendTypeEnums()
Valid enum values of distributor send types.
void sort2(const IT1 &first1, const IT1 &last1, const IT2 &first2, const bool stableSort=false)
Sort the first array, and apply the resulting permutation to the second array.
std::string DistributorHowInitializedEnumToString(EDistributorHowInitialized how)
Convert an EDistributorHowInitialized enum value to a string.
Stand-alone utility functions and macros.
Teuchos::Array< std::string > distributorSendTypes()
Valid string values for Distributor's "Send type" parameter.
#define SHARED_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg, comm)
Test for exception, with reduction over the given communicator.
EDistributorSendType
The type of MPI send that Distributor should use.
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.