10 #ifndef TPETRA_DIRECTORYIMPL_DEF_HPP
11 #define TPETRA_DIRECTORYIMPL_DEF_HPP
16 #include "Tpetra_Distributor.hpp"
17 #include "Tpetra_Map.hpp"
20 #include "Tpetra_Details_FixedHashTable.hpp"
21 #include "Teuchos_Comm.hpp"
26 #ifdef HAVE_TPETRACORE_MPI
28 #endif // HAVE_TPETRACORE_MPI
33 template <
class LO,
class GO,
class NT>
37 const Teuchos::ArrayView<const GO>& globalIDs,
38 const Teuchos::ArrayView<int>& nodeIDs,
39 const Teuchos::ArrayView<LO>& localIDs,
40 const bool computeLIDs)
const {
43 TEUCHOS_TEST_FOR_EXCEPTION(nodeIDs.size() != globalIDs.size(),
44 std::invalid_argument, Teuchos::typeName(*
this) <<
"::getEntries(): "
45 "Output arrays do not have the right sizes. nodeIDs.size() = "
46 << nodeIDs.size() <<
" != globalIDs.size() = " << globalIDs.size() <<
".");
47 TEUCHOS_TEST_FOR_EXCEPTION(
48 computeLIDs && localIDs.size() != globalIDs.size(),
49 std::invalid_argument, Teuchos::typeName(*
this) <<
"::getEntries(): "
50 "Output array do not have the right sizes. localIDs.size() = "
51 << localIDs.size() <<
" != globalIDs.size() = " << globalIDs.size() <<
".");
58 std::fill(nodeIDs.begin(), nodeIDs.end(), -1);
60 std::fill(localIDs.begin(), localIDs.end(),
61 Teuchos::OrdinalTraits<LO>::invalid());
64 return this->getEntriesImpl(map, globalIDs, nodeIDs, localIDs, computeLIDs);
67 template <
class LO,
class GO,
class NT>
70 : numProcs_(map.getComm()->getSize()) {}
72 template <
class LO,
class GO,
class NT>
78 return (numProcs_ == 1);
81 template <
class LO,
class GO,
class NT>
84 std::ostringstream os;
85 os <<
"ReplicatedDirectory"
86 <<
"<" << Teuchos::TypeNameTraits<LO>::name()
87 <<
", " << Teuchos::TypeNameTraits<GO>::name()
88 <<
", " << Teuchos::TypeNameTraits<NT>::name() <<
">";
92 template <
class LO,
class GO,
class NT>
95 TEUCHOS_TEST_FOR_EXCEPTION(!map.isContiguous(), std::invalid_argument,
96 Teuchos::typeName(*
this) <<
" constructor: Map is not contiguous.");
97 TEUCHOS_TEST_FOR_EXCEPTION(!map.isUniform(), std::invalid_argument,
98 Teuchos::typeName(*
this) <<
" constructor: Map is not uniform.");
101 template <
class LO,
class GO,
class NT>
104 std::ostringstream os;
105 os <<
"ContiguousUniformDirectory"
106 <<
"<" << Teuchos::TypeNameTraits<LO>::name()
107 <<
", " << Teuchos::TypeNameTraits<GO>::name()
108 <<
", " << Teuchos::TypeNameTraits<NT>::name() <<
">";
112 template <
class LO,
class GO,
class NT>
116 const Teuchos::ArrayView<const GO>& globalIDs,
117 const Teuchos::ArrayView<int>& nodeIDs,
118 const Teuchos::ArrayView<LO>& localIDs,
119 const bool computeLIDs)
const {
122 typedef typename Teuchos::ArrayView<const GO>::size_type size_type;
123 const LO invalidLid = Teuchos::OrdinalTraits<LO>::invalid();
126 RCP<const Comm<int> > comm = map.
getComm();
150 const size_type N_G =
152 const size_type P =
static_cast<size_type
>(comm->getSize());
153 const size_type N_L = N_G / P;
154 const size_type R = N_G - N_L * P;
155 const size_type N_R = R * (N_L +
static_cast<size_type
>(1));
157 #ifdef HAVE_TPETRA_DEBUG
158 TEUCHOS_TEST_FOR_EXCEPTION(
159 N_G != P * N_L + R, std::logic_error,
160 "Tpetra::ContiguousUniformDirectory::getEntriesImpl: "
162 << N_G <<
" != P*N_L + R = " << P <<
"*" << N_L <<
" + " << R
163 <<
" = " << P * N_L + R <<
". "
164 "Please report this bug to the Tpetra developers.");
165 #endif // HAVE_TPETRA_DEBUG
167 const size_type numGids = globalIDs.size();
170 const GO ONE =
static_cast<GO
>(1);
173 for (size_type k = 0; k < numGids; ++k) {
174 const GO g_0 = globalIDs[k] - g_min;
180 if (g_0 + ONE < ONE || g_0 >= static_cast<GO>(N_G)) {
182 localIDs[k] = invalidLid;
184 }
else if (g_0 < static_cast<GO>(N_R)) {
186 nodeIDs[k] =
static_cast<int>(g_0 /
static_cast<GO
>(N_L + 1));
187 localIDs[k] =
static_cast<LO
>(g_0 %
static_cast<GO
>(N_L + 1));
188 }
else if (g_0 >= static_cast<GO>(N_R)) {
190 const GO g_R = g_0 -
static_cast<GO
>(N_R);
191 nodeIDs[k] =
static_cast<int>(R + g_R / N_L);
192 localIDs[k] =
static_cast<int>(g_R % N_L);
194 #ifdef HAVE_TPETRA_DEBUG
196 TEUCHOS_TEST_FOR_EXCEPTION(
true, std::logic_error,
197 "Tpetra::ContiguousUniformDirectory::getEntriesImpl: "
198 "should never get here. "
199 "Please report this bug to the Tpetra developers.");
201 #endif // HAVE_TPETRA_DEBUG
204 for (size_type k = 0; k < numGids; ++k) {
205 const GO g_0 = globalIDs[k] - g_min;
210 if (g_0 + ONE < ONE || g_0 >= static_cast<GO>(N_G)) {
213 }
else if (g_0 < static_cast<GO>(N_R)) {
215 nodeIDs[k] =
static_cast<int>(g_0 /
static_cast<GO
>(N_L + 1));
216 }
else if (g_0 >= static_cast<GO>(N_R)) {
218 const GO g_R = g_0 -
static_cast<GO
>(N_R);
219 nodeIDs[k] =
static_cast<int>(R + g_R / N_L);
221 #ifdef HAVE_TPETRA_DEBUG
223 TEUCHOS_TEST_FOR_EXCEPTION(
true, std::logic_error,
224 "Tpetra::ContiguousUniformDirectory::getEntriesImpl: "
225 "should never get here. "
226 "Please report this bug to the Tpetra developers.");
228 #endif // HAVE_TPETRA_DEBUG
234 template <
class LO,
class GO,
class NT>
238 using Teuchos::gatherAll;
241 RCP<const Teuchos::Comm<int> > comm = map.getComm();
243 TEUCHOS_TEST_FOR_EXCEPTION(!map.isDistributed(), std::invalid_argument,
244 Teuchos::typeName(*
this) <<
" constructor: Map is not distributed.");
245 TEUCHOS_TEST_FOR_EXCEPTION(!map.isContiguous(), std::invalid_argument,
246 Teuchos::typeName(*
this) <<
" constructor: Map is not contiguous.");
248 const int numProcs = comm->getSize();
252 allMinGIDs_ = arcp<GO>(numProcs + 1);
254 GO minMyGID = map.getMinGlobalIndex();
265 #ifdef HAVE_TPETRACORE_MPI
266 MPI_Datatype rawMpiType = MPI_INT;
267 bool useRawMpi =
true;
268 if (
typeid(GO) ==
typeid(
int)) {
269 rawMpiType = MPI_INT;
270 }
else if (
typeid(GO) ==
typeid(
long)) {
271 rawMpiType = MPI_LONG;
276 using Teuchos::MpiComm;
277 using Teuchos::rcp_dynamic_cast;
278 RCP<const MpiComm<int> > mpiComm =
279 rcp_dynamic_cast<
const MpiComm<int> >(comm);
282 if (!comm.is_null()) {
283 MPI_Comm rawMpiComm = *(mpiComm->getRawMpiComm());
285 MPI_Allgather(&minMyGID, 1, rawMpiType,
286 allMinGIDs_.getRawPtr(), 1, rawMpiType,
288 TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
289 "Tpetra::DistributedContiguousDirectory: MPI_Allgather failed");
291 gatherAll<int, GO>(*comm, 1, &minMyGID, numProcs, allMinGIDs_.getRawPtr());
294 gatherAll<int, GO>(*comm, 1, &minMyGID, numProcs, allMinGIDs_.getRawPtr());
296 #else // NOT HAVE_TPETRACORE_MPI
297 gatherAll<int, GO>(*comm, 1, &minMyGID, numProcs, allMinGIDs_.getRawPtr());
298 #endif // HAVE_TPETRACORE_MPI
305 allMinGIDs_[numProcs] = map.getMaxAllGlobalIndex() + Teuchos::OrdinalTraits<GO>::one();
308 template <
class LO,
class GO,
class NT>
311 std::ostringstream os;
312 os <<
"DistributedContiguousDirectory"
313 <<
"<" << Teuchos::TypeNameTraits<LO>::name()
314 <<
", " << Teuchos::TypeNameTraits<GO>::name()
315 <<
", " << Teuchos::TypeNameTraits<NT>::name() <<
">";
319 template <
class LO,
class GO,
class NT>
323 const Teuchos::ArrayView<const GO>& globalIDs,
324 const Teuchos::ArrayView<int>& nodeIDs,
325 const Teuchos::ArrayView<LO>& localIDs,
326 const bool computeLIDs)
const {
327 using Teuchos::Array;
328 using Teuchos::ArrayRCP;
329 using Teuchos::ArrayView;
335 RCP<const Teuchos::Comm<int> > comm = map.
getComm();
336 const int myRank = comm->getRank();
339 typename ArrayView<int>::iterator procIter = nodeIDs.begin();
340 typename ArrayView<LO>::iterator lidIter = localIDs.begin();
341 typename ArrayView<const GO>::iterator gidIter;
342 for (gidIter = globalIDs.begin(); gidIter != globalIDs.end(); ++gidIter) {
344 *procIter++ = myRank;
360 template <
class LO,
class GO,
class NT>
364 const Teuchos::ArrayView<const GO>& globalIDs,
365 const Teuchos::ArrayView<int>& nodeIDs,
366 const Teuchos::ArrayView<LO>& localIDs,
367 const bool computeLIDs)
const {
368 using Teuchos::Array;
369 using Teuchos::ArrayRCP;
370 using Teuchos::ArrayView;
375 RCP<const Teuchos::Comm<int> > comm = map.
getComm();
376 const int numProcs = comm->getSize();
377 const LO LINVALID = Teuchos::OrdinalTraits<LO>::invalid();
378 const GO noGIDsOnProc = std::numeric_limits<GO>::max();
382 int firstProcWithGIDs;
383 for (firstProcWithGIDs = 0; firstProcWithGIDs < numProcs;
384 firstProcWithGIDs++) {
385 if (allMinGIDs_[firstProcWithGIDs] != noGIDsOnProc)
break;
391 if (firstProcWithGIDs == numProcs) {
394 std::fill(nodeIDs.begin(), nodeIDs.end(), -1);
396 std::fill(localIDs.begin(), localIDs.end(), LINVALID);
400 const GO one = as<GO>(1);
401 const GO nOverP = as<GO>(map.
getGlobalNumElements() / as<global_size_t>(numProcs - firstProcWithGIDs));
404 typename ArrayView<int>::iterator procIter = nodeIDs.begin();
405 typename ArrayView<LO>::iterator lidIter = localIDs.begin();
406 typename ArrayView<const GO>::iterator gidIter;
407 for (gidIter = globalIDs.begin(); gidIter != globalIDs.end(); ++gidIter) {
416 const GO firstGuess = firstProcWithGIDs + GID / std::max(nOverP, one);
417 curRank = as<int>(std::min(firstGuess, as<GO>(numProcs - 1)));
421 while (allMinGIDs_[curRank] == noGIDsOnProc) curRank++;
424 while (curRank >= firstProcWithGIDs && GID < allMinGIDs_[curRank]) {
426 while (curRank >= firstProcWithGIDs &&
427 allMinGIDs_[curRank] == noGIDsOnProc) curRank--;
429 if (curRank < firstProcWithGIDs) {
432 }
else if (curRank == numProcs) {
437 int above = curRank + 1;
438 while (allMinGIDs_[above] == noGIDsOnProc) above++;
440 while (GID >= allMinGIDs_[above]) {
442 if (curRank == numProcs) {
448 while (allMinGIDs_[above] == noGIDsOnProc) above++;
454 LID = as<LO>(GID - allMinGIDs_[image]);
466 template <
class LO,
class GO,
class NT>
469 : oneToOneResult_(ONE_TO_ONE_NOT_CALLED_YET)
471 locallyOneToOne_(true)
473 useHashTables_(false)
478 template <
class LO,
class GO,
class NT>
479 DistributedNoncontiguousDirectory<LO, GO, NT>::
480 DistributedNoncontiguousDirectory(
const map_type& map,
481 const tie_break_type& tie_break)
482 : oneToOneResult_(ONE_TO_ONE_NOT_CALLED_YET)
484 locallyOneToOne_(true)
486 useHashTables_(false)
488 initialize(map, Teuchos::ptrFromRef(tie_break));
491 template <
class LO,
class GO,
class NT>
492 void DistributedNoncontiguousDirectory<LO, GO, NT>::
493 initialize(
const map_type& map,
494 Teuchos::Ptr<const tie_break_type> tie_break) {
498 using Teuchos::Array;
499 using Teuchos::ArrayRCP;
500 using Teuchos::ArrayView;
504 using Teuchos::typeName;
505 using Teuchos::TypeNameTraits;
506 typedef Array<int>::size_type size_type;
516 TEUCHOS_TEST_FOR_EXCEPTION(
sizeof(
global_size_t) <
sizeof(GO),
517 std::logic_error, typeName(*
this) <<
": sizeof(Tpetra::"
521 << TypeNameTraits<LO>::name() <<
") = " <<
sizeof(GO) <<
".");
522 TEUCHOS_TEST_FOR_EXCEPTION(
sizeof(
global_size_t) <
sizeof(
int),
523 std::logic_error, typeName(*
this) <<
": sizeof(Tpetra::"
525 <<
sizeof(
global_size_t) <<
" < sizeof(int) = " <<
sizeof(
int) <<
".");
526 TEUCHOS_TEST_FOR_EXCEPTION(
sizeof(
global_size_t) <
sizeof(LO),
527 std::logic_error, typeName(*
this) <<
": sizeof(Tpetra::"
531 << TypeNameTraits<LO>::name() <<
") = " <<
sizeof(LO) <<
".");
533 RCP<const Teuchos::Comm<int> > comm = map.getComm();
534 const LO LINVALID = Teuchos::OrdinalTraits<LO>::invalid();
535 const GO minAllGID = map.getMinAllGlobalIndex();
536 const GO maxAllGID = map.getMaxAllGlobalIndex();
543 const global_size_t numGlobalEntries = maxAllGID - minAllGID + 1;
553 directoryMap_ = rcp(
new map_type(numGlobalEntries, minAllGID, comm,
554 GloballyDistributed));
556 const size_t dir_numMyEntries = directoryMap_->getLocalNumElements();
580 const size_t inverseSparsityThreshold = 10;
582 (dir_numMyEntries >= inverseSparsityThreshold * map.getLocalNumElements());
587 const int myRank = comm->getRank();
588 const size_t numMyEntries = map.getLocalNumElements();
589 Array<int> sendImageIDs(numMyEntries);
590 ArrayView<const GO> myGlobalEntries = map.getLocalElementList();
595 directoryMap_->getRemoteIndexList(myGlobalEntries, sendImageIDs);
596 TEUCHOS_TEST_FOR_EXCEPTION(
597 lookupStatus ==
IDNotPresent, std::logic_error, Teuchos::typeName(*
this) <<
" constructor: the Directory Map could not find out where one or "
598 "more of my Map's indices should go. The input to getRemoteIndexList "
600 << Teuchos::toString(myGlobalEntries) <<
", and the output is " << Teuchos::toString(sendImageIDs()) <<
". The input Map itself has "
601 "the following entries on the calling process "
602 << map.getComm()->getRank() <<
": " << Teuchos::toString(map.getLocalElementList()) <<
", and has " << map.getGlobalNumElements() <<
" total global indices in [" << map.getMinAllGlobalIndex() <<
"," << map.getMaxAllGlobalIndex() <<
"]. The Directory Map has " << directoryMap_->getGlobalNumElements() <<
" total global indices in "
604 << directoryMap_->getMinAllGlobalIndex() <<
"," << directoryMap_->getMaxAllGlobalIndex() <<
"], and the calling process "
606 << directoryMap_->getMinGlobalIndex() <<
"," << directoryMap_->getMaxGlobalIndex() <<
"]. "
607 "This probably means there is a bug in Map or Directory. "
608 "Please report this bug to the Tpetra developers.");
615 Distributor distor(comm);
616 const size_t numReceives = distor.createFromSends(sendImageIDs);
630 const int packetSize = 3;
631 Kokkos::View<GO*, Kokkos::HostSpace> exportEntries(
"exportEntries", packetSize * numMyEntries);
633 size_t exportIndex = 0;
634 for (
size_t i = 0; i < numMyEntries; ++i) {
635 exportEntries[exportIndex++] = myGlobalEntries[i];
636 exportEntries[exportIndex++] = as<GO>(myRank);
637 exportEntries[exportIndex++] = as<GO>(i);
643 Kokkos::View<GO*, Kokkos::HostSpace> importElements(
"importElements", packetSize * distor.getTotalReceiveLength());
646 distor.doPostsAndWaits(exportEntries, packetSize, importElements);
653 if (useHashTables_) {
668 LO* tableKeysRaw = NULL;
669 LO* tableLidsRaw = NULL;
670 int* tablePidsRaw = NULL;
672 tableKeysRaw =
new LO[numReceives];
673 tableLidsRaw =
new LO[numReceives];
674 tablePidsRaw =
new int[numReceives];
676 if (tableKeysRaw != NULL) {
677 delete[] tableKeysRaw;
679 if (tableLidsRaw != NULL) {
680 delete[] tableLidsRaw;
682 if (tablePidsRaw != NULL) {
683 delete[] tablePidsRaw;
687 ArrayRCP<LO> tableKeys(tableKeysRaw, 0, numReceives,
true);
688 ArrayRCP<LO> tableLids(tableLidsRaw, 0, numReceives,
true);
689 ArrayRCP<int> tablePids(tablePidsRaw, 0, numReceives,
true);
691 if (tie_break.is_null()) {
693 size_type importIndex = 0;
694 for (size_type i = 0; i < static_cast<size_type>(numReceives); ++i) {
695 const GO curGID = importElements[importIndex++];
696 const LO curLID = directoryMap_->getLocalElement(curGID);
697 TEUCHOS_TEST_FOR_EXCEPTION(
698 curLID == LINVALID, std::logic_error,
699 Teuchos::typeName(*
this) <<
" constructor: Incoming global index "
700 << curGID <<
" does not have a corresponding local index in the "
701 "Directory Map. Please report this bug to the Tpetra developers.");
702 tableKeys[i] = curLID;
703 tablePids[i] = importElements[importIndex++];
704 tableLids[i] = importElements[importIndex++];
710 rcp(
new lidToPidTable_type(tableKeys(), tablePids()));
711 locallyOneToOne_ = !(lidToPidTable_->hasDuplicateKeys());
713 rcp(
new lidToLidTable_type(tableKeys(), tableLids()));
721 typedef std::map<LO, std::vector<std::pair<int, LO> > > pair_table_type;
722 pair_table_type ownedPidLidPairs;
726 size_type importIndex = 0;
727 for (size_type i = 0; i < static_cast<size_type>(numReceives); ++i) {
728 const GO curGID = importElements[importIndex++];
729 const LO dirMapLid = directoryMap_->getLocalElement(curGID);
730 TEUCHOS_TEST_FOR_EXCEPTION(
731 dirMapLid == LINVALID, std::logic_error,
732 Teuchos::typeName(*
this) <<
" constructor: Incoming global index "
733 << curGID <<
" does not have a corresponding local index in the "
734 "Directory Map. Please report this bug to the Tpetra developers.");
735 tableKeys[i] = dirMapLid;
736 const int PID = importElements[importIndex++];
737 const int LID = importElements[importIndex++];
747 ownedPidLidPairs[dirMapLid].push_back(std::make_pair(PID, LID));
760 for (size_type i = 0; i < static_cast<size_type>(numReceives); ++i) {
761 const LO dirMapLid = tableKeys[i];
762 const std::vector<std::pair<int, LO> >& pidLidList =
763 ownedPidLidPairs[dirMapLid];
764 const size_t listLen = pidLidList.size();
765 if (listLen == 0)
continue;
766 const GO dirMapGid = directoryMap_->getGlobalElement(dirMapLid);
768 locallyOneToOne_ =
false;
777 const size_type index =
778 static_cast<size_type
>(tie_break->selectedIndex(dirMapGid,
780 tablePids[i] = pidLidList[index].first;
781 tableLids[i] = pidLidList[index].second;
786 rcp(
new lidToPidTable_type(tableKeys(), tablePids()));
788 rcp(
new lidToLidTable_type(tableKeys(), tableLids()));
791 if (tie_break.is_null()) {
796 PIDs_ = arcp<int>(dir_numMyEntries);
797 std::fill(PIDs_.begin(), PIDs_.end(), -1);
798 LIDs_ = arcp<LO>(dir_numMyEntries);
799 std::fill(LIDs_.begin(), LIDs_.end(), LINVALID);
801 size_type importIndex = 0;
802 for (size_type i = 0; i < static_cast<size_type>(numReceives); ++i) {
803 const GO curGID = importElements[importIndex++];
804 const LO curLID = directoryMap_->getLocalElement(curGID);
805 TEUCHOS_TEST_FOR_EXCEPTION(curLID == LINVALID, std::logic_error,
806 Teuchos::typeName(*
this) <<
" constructor: Incoming global index "
807 << curGID <<
" does not have a corresponding local index in the "
808 "Directory Map. Please report this bug to the Tpetra developers.");
813 if (PIDs_[curLID] != -1) {
814 locallyOneToOne_ =
false;
816 PIDs_[curLID] = importElements[importIndex++];
817 LIDs_[curLID] = importElements[importIndex++];
820 PIDs_ = arcp<int>(dir_numMyEntries);
821 LIDs_ = arcp<LO>(dir_numMyEntries);
822 std::fill(PIDs_.begin(), PIDs_.end(), -1);
831 Array<std::vector<std::pair<int, LO> > > ownedPidLidPairs(dir_numMyEntries);
832 size_t importIndex = 0;
833 for (
size_t i = 0; i < numReceives; ++i) {
834 const GO GID = importElements[importIndex++];
835 const int PID = importElements[importIndex++];
836 const LO LID = importElements[importIndex++];
838 const LO dirMapLid = directoryMap_->getLocalElement(GID);
839 TEUCHOS_TEST_FOR_EXCEPTION(
840 dirMapLid == LINVALID, std::logic_error,
841 Teuchos::typeName(*
this) <<
" constructor: Incoming global index "
842 << GID <<
" does not have a corresponding local index in the "
843 "Directory Map. Please report this bug to the Tpetra developers.");
844 ownedPidLidPairs[dirMapLid].push_back(std::make_pair(PID, LID));
856 for (
size_t i = 0; i < dir_numMyEntries; ++i) {
857 const std::vector<std::pair<int, LO> >& pidLidList =
859 const size_t listLen = pidLidList.size();
860 if (listLen == 0)
continue;
862 const LO dirMapLid =
static_cast<LO
>(i);
863 const GO dirMapGid = directoryMap_->getGlobalElement(dirMapLid);
865 locallyOneToOne_ =
false;
874 const size_type index =
875 static_cast<size_type
>(tie_break->selectedIndex(dirMapGid,
877 PIDs_[i] = pidLidList[index].first;
878 LIDs_[i] = pidLidList[index].second;
884 template <
class LO,
class GO,
class NT>
887 std::ostringstream os;
888 os <<
"DistributedNoncontiguousDirectory"
889 <<
"<" << Teuchos::TypeNameTraits<LO>::name()
890 <<
", " << Teuchos::TypeNameTraits<GO>::name()
891 <<
", " << Teuchos::TypeNameTraits<NT>::name() <<
">";
895 template <
class LO,
class GO,
class NT>
899 const Teuchos::ArrayView<const GO>& globalIDs,
900 const Teuchos::ArrayView<int>& nodeIDs,
901 const Teuchos::ArrayView<LO>& localIDs,
902 const bool computeLIDs)
const {
907 using Teuchos::Array;
908 using Teuchos::ArrayRCP;
909 using Teuchos::ArrayView;
912 using Teuchos::toString;
913 using size_type =
typename Array<GO>::size_type;
914 const char funcPrefix[] =
916 "DistributedNoncontiguousDirectory::getEntriesImpl: ";
917 const char errSuffix[] =
918 " Please report this bug to the Tpetra developers.";
920 RCP<const Teuchos::Comm<int> > comm = map.getComm();
925 std::unique_ptr<std::string> procPrefix;
927 std::ostringstream os;
929 if (map.getComm().is_null()) {
932 os << map.getComm()->getRank();
935 procPrefix = std::unique_ptr<std::string>(
936 new std::string(os.str()));
937 os << funcPrefix <<
"{";
943 os <<
", computeLIDs: "
944 << (computeLIDs ?
"true" :
"false") <<
"}" << endl;
948 const size_t numEntries = globalIDs.size();
949 const LO LINVALID = Teuchos::OrdinalTraits<LO>::invalid();
958 const int packetSize = computeLIDs ? 3 : 2;
964 Array<int> dirImages(numEntries);
966 std::ostringstream os;
967 os << *procPrefix <<
"Call directoryMap_->getRemoteIndexList"
971 res = directoryMap_->getRemoteIndexList(globalIDs, dirImages());
973 std::ostringstream os;
974 os << *procPrefix <<
"Director Map getRemoteIndexList out ";
981 size_t numMissing = 0;
983 for (
size_t i = 0; i < numEntries; ++i) {
984 if (dirImages[i] == -1) {
987 localIDs[i] = LINVALID;
995 Array<int> sendImages;
997 std::ostringstream os;
998 os << *procPrefix <<
"Call Distributor::createFromRecvs"
1004 std::ostringstream os;
1005 os << *procPrefix <<
"Distributor::createFromRecvs result: "
1013 const size_type numSends = sendGIDs.size();
1046 Kokkos::View<global_size_t*, Kokkos::HostSpace> exports(
"exports", packetSize * numSends);
1059 size_t exportsIndex = 0;
1061 if (useHashTables_) {
1063 std::ostringstream os;
1064 os << *procPrefix <<
"Pack exports (useHashTables_ true)"
1068 for (size_type gidIndex = 0; gidIndex < numSends; ++gidIndex) {
1069 const GO curGID = sendGIDs[gidIndex];
1071 exports[exportsIndex++] =
static_cast<global_size_t>(curGID);
1072 const LO curLID = directoryMap_->getLocalElement(curGID);
1073 TEUCHOS_TEST_FOR_EXCEPTION(curLID == LINVALID, std::logic_error, funcPrefix <<
"Directory Map's global index " << curGID <<
" lacks "
1074 "a corresponding local index."
1077 exports[exportsIndex++] =
1081 exports[exportsIndex++] =
1087 std::ostringstream os;
1088 os << *procPrefix <<
"Pack exports (useHashTables_ false)"
1092 for (size_type gidIndex = 0; gidIndex < numSends; ++gidIndex) {
1093 const GO curGID = sendGIDs[gidIndex];
1095 exports[exportsIndex++] =
static_cast<global_size_t>(curGID);
1096 const LO curLID = directoryMap_->getLocalElement(curGID);
1097 TEUCHOS_TEST_FOR_EXCEPTION(curLID == LINVALID, std::logic_error, funcPrefix <<
"Directory Map's global index " << curGID <<
" lacks "
1098 "a corresponding local index."
1101 exports[exportsIndex++] =
1105 exports[exportsIndex++] =
1111 TEUCHOS_TEST_FOR_EXCEPTION(exportsIndex > exports.size(), std::logic_error,
1112 funcPrefix <<
"On Process " << comm->getRank() <<
", "
1114 << exportsIndex <<
" > exports.size() = "
1115 << exports.size() <<
"." << errSuffix);
1118 TEUCHOS_TEST_FOR_EXCEPTION(numEntries < numMissing, std::logic_error, funcPrefix <<
"On Process " << comm->getRank() <<
", numEntries = " << numEntries <<
" < numMissing = " << numMissing <<
"." << errSuffix);
1124 const size_t numRecv = numEntries - numMissing;
1128 const size_t requiredImportLen = numRecv * packetSize;
1129 const int myRank = comm->getRank();
1130 TEUCHOS_TEST_FOR_EXCEPTION(
1131 importLen < requiredImportLen, std::logic_error,
1132 "Tpetra::Details::DistributedNoncontiguousDirectory::getEntriesImpl: "
1134 << myRank <<
": The 'imports' array must have length "
1136 << requiredImportLen <<
", but its actual length is " << importLen <<
". numRecv: " << numRecv <<
", packetSize: " << packetSize <<
", numEntries (# GIDs): " << numEntries <<
", numMissing: " << numMissing <<
": distor.getTotalReceiveLength(): "
1138 <<
"Distributor description: " << distor.
description() <<
". "
1140 <<
"Please report this bug to the Tpetra developers.");
1143 Kokkos::View<global_size_t*, Kokkos::HostSpace> imports(
"imports", packetSize * distor.
getTotalReceiveLength());
1148 std::ostringstream os;
1149 os << *procPrefix <<
"Call doPostsAndWaits: {"
1150 <<
"packetSize: " << packetSize <<
", ";
1157 std::ostringstream os;
1158 os << *procPrefix <<
"doPostsAndWaits result: ";
1164 Array<GO> sortedIDs(globalIDs);
1165 Array<GO> offset(numEntries);
1166 for (GO ii = 0; ii < static_cast<GO>(numEntries); ++ii) {
1169 sort2(sortedIDs.begin(), sortedIDs.begin() + numEntries, offset.begin());
1171 std::ostringstream os;
1180 size_t importsIndex = 0;
1185 for (
size_t i = 0; i < numRecv; ++i) {
1186 const GO curGID =
static_cast<GO
>(imports[importsIndex++]);
1187 auto p1 = std::equal_range(sortedIDs.begin(),
1188 sortedIDs.end(), curGID);
1189 if (p1.first != p1.second) {
1190 const size_t j = p1.first - sortedIDs.begin();
1191 nodeIDs[offset[j]] =
1192 static_cast<int>(imports[importsIndex++]);
1194 localIDs[offset[j]] =
1195 static_cast<LO
>(imports[importsIndex++]);
1197 if (nodeIDs[offset[j]] == -1) {
1203 TEUCHOS_TEST_FOR_EXCEPTION(
size_t(importsIndex) >
size_t(imports.size()),
1204 std::logic_error, funcPrefix <<
"On Process " << comm->getRank() <<
": importsIndex = " << importsIndex <<
" > imports.size() = " << imports.size() <<
". "
1206 << numRecv <<
", packetSize: " << packetSize <<
", "
1207 "numEntries (# GIDs): "
1208 << numEntries <<
", numMissing: " << numMissing <<
": distor.getTotalReceiveLength(): " << distor.
getTotalReceiveLength() <<
"." << errSuffix);
1210 std::ostringstream os;
1211 os << *procPrefix << funcPrefix <<
"Done!" << endl;
1217 template <
class LO,
class GO,
class NT>
1220 if (oneToOneResult_ == ONE_TO_ONE_NOT_CALLED_YET) {
1221 const int lcl121 = isLocallyOneToOne() ? 1 : 0;
1223 Teuchos::reduceAll<int, int>(comm, Teuchos::REDUCE_MIN, lcl121,
1224 Teuchos::outArg(gbl121));
1225 oneToOneResult_ = (gbl121 == 1) ? ONE_TO_ONE_TRUE : ONE_TO_ONE_FALSE;
1227 return (oneToOneResult_ == ONE_TO_ONE_TRUE);
1237 #define TPETRA_DIRECTORYIMPL_INSTANT(LO, GO, NODE) \
1238 namespace Details { \
1239 template class Directory<LO, GO, NODE>; \
1240 template class ReplicatedDirectory<LO, GO, NODE>; \
1241 template class ContiguousUniformDirectory<LO, GO, NODE>; \
1242 template class DistributedContiguousDirectory<LO, GO, NODE>; \
1243 template class DistributedNoncontiguousDirectory<LO, GO, NODE>; \
1246 #endif // TPETRA_DIRECTORYIMPL_DEF_HPP
void initialize(int *argc, char ***argv)
Initialize Tpetra.
std::string description() const override
A one-line human-readable description of this object.
Interface for breaking ties in ownership.
std::string description() const
Return a one-line description of this object.
LookupStatus
Return status of Map remote index lookup (getRemoteIndexList()).
Implementation of Directory for a distributed noncontiguous Map.
Teuchos::RCP< const Teuchos::Comm< int > > getComm() const
Accessors for the Teuchos::Comm and Kokkos Node objects.
bool isOneToOne(const Teuchos::Comm< int > &comm) const override
Whether the Directory's input Map is (globally) one to one.
Implementation of Directory for a distributed contiguous Map.
void verbosePrintArray(std::ostream &out, const ArrayType &x, const char name[], const size_t maxNumToPrint)
Print min(x.size(), maxNumToPrint) entries of x.
LookupStatus getEntriesImpl(const map_type &map, const Teuchos::ArrayView< const GlobalOrdinal > &globalIDs, const Teuchos::ArrayView< int > &nodeIDs, const Teuchos::ArrayView< LocalOrdinal > &localIDs, const bool computeLIDs) const override
Find process IDs and (optionally) local IDs for the given global IDs.
bool isOneToOne(const Teuchos::Comm< int > &comm) const override
Whether the Directory's input Map is (globally) one to one.
size_t global_size_t
Global size_t object.
LookupStatus getEntriesImpl(const map_type &map, const Teuchos::ArrayView< const GlobalOrdinal > &globalIDs, const Teuchos::ArrayView< int > &nodeIDs, const Teuchos::ArrayView< LocalOrdinal > &localIDs, const bool computeLIDs) const override
Find process IDs and (optionally) local IDs for the given global IDs.
void sort2(const IT1 &first1, const IT1 &last1, const IT2 &first2, const bool stableSort=false)
Sort the first array, and apply the resulting permutation to the second array.
LookupStatus getEntries(const map_type &map, const Teuchos::ArrayView< const GlobalOrdinal > &globalIDs, const Teuchos::ArrayView< int > &nodeIDs, const Teuchos::ArrayView< LocalOrdinal > &localIDs, const bool computeLIDs) const
bool isNodeGlobalElement(global_ordinal_type globalIndex) const
Whether the given global index is owned by this Map on the calling process.
Sets up and executes a communication plan for a Tpetra DistObject.
static bool verbose()
Whether Tpetra is in verbose mode.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
std::string description() const override
A one-line human-readable description of this object.
A parallel distribution of indices over processes.
ReplicatedDirectory()=default
Constructor (that takes no arguments).
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the (forward) communication plan.
local_ordinal_type getLocalElement(global_ordinal_type globalIndex) const
The local index corresponding to the given global index.
Stand-alone utility functions and macros.
static size_t verbosePrintCountThreshold()
Number of entries below which arrays, lists, etc. will be printed in debug mode.
LookupStatus getEntriesImpl(const map_type &map, const Teuchos::ArrayView< const GlobalOrdinal > &globalIDs, const Teuchos::ArrayView< int > &nodeIDs, const Teuchos::ArrayView< LocalOrdinal > &localIDs, const bool computeLIDs) const override
Find process IDs and (optionally) local IDs for the given global IDs.
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
std::string description() const override
A one-line human-readable description of this object.
global_ordinal_type getMinAllGlobalIndex() const
The minimum global index over all processes in the communicator.
global_size_t getGlobalNumElements() const
The number of elements in this Map.
Description of Tpetra's behavior.