40 #ifndef TPETRA_DIRECTORYIMPL_DEF_HPP
41 #define TPETRA_DIRECTORYIMPL_DEF_HPP
46 #include "Tpetra_Distributor.hpp"
47 #include "Tpetra_Map.hpp"
50 #include "Tpetra_Details_FixedHashTable.hpp"
51 #include "Teuchos_Comm.hpp"
56 #ifdef HAVE_TPETRACORE_MPI
58 #endif // HAVE_TPETRACORE_MPI
63 template<
class LO,
class GO,
class NT>
67 const Teuchos::ArrayView<const GO> &globalIDs,
68 const Teuchos::ArrayView<int> &nodeIDs,
69 const Teuchos::ArrayView<LO> &localIDs,
70 const bool computeLIDs)
const
74 TEUCHOS_TEST_FOR_EXCEPTION(nodeIDs.size() != globalIDs.size(),
75 std::invalid_argument, Teuchos::typeName(*
this) <<
"::getEntries(): "
76 "Output arrays do not have the right sizes. nodeIDs.size() = "
77 << nodeIDs.size() <<
" != globalIDs.size() = " << globalIDs.size()
79 TEUCHOS_TEST_FOR_EXCEPTION(
80 computeLIDs && localIDs.size() != globalIDs.size(),
81 std::invalid_argument, Teuchos::typeName(*
this) <<
"::getEntries(): "
82 "Output array do not have the right sizes. localIDs.size() = "
83 << localIDs.size() <<
" != globalIDs.size() = " << globalIDs.size()
91 std::fill (nodeIDs.begin(), nodeIDs.end(), -1);
93 std::fill (localIDs.begin(), localIDs.end(),
94 Teuchos::OrdinalTraits<LO>::invalid ());
97 return this->getEntriesImpl (map, globalIDs, nodeIDs, localIDs, computeLIDs);
101 template<
class LO,
class GO,
class NT>
104 numProcs_ (map.getComm ()->getSize ())
108 template<
class LO,
class GO,
class NT>
116 return (numProcs_ == 1);
120 template<
class LO,
class GO,
class NT>
124 std::ostringstream os;
125 os <<
"ReplicatedDirectory"
126 <<
"<" << Teuchos::TypeNameTraits<LO>::name ()
127 <<
", " << Teuchos::TypeNameTraits<GO>::name ()
128 <<
", " << Teuchos::TypeNameTraits<NT>::name () <<
">";
133 template<
class LO,
class GO,
class NT>
137 TEUCHOS_TEST_FOR_EXCEPTION(! map.isContiguous (), std::invalid_argument,
138 Teuchos::typeName (*
this) <<
" constructor: Map is not contiguous.");
139 TEUCHOS_TEST_FOR_EXCEPTION(! map.isUniform (), std::invalid_argument,
140 Teuchos::typeName (*
this) <<
" constructor: Map is not uniform.");
144 template<
class LO,
class GO,
class NT>
148 std::ostringstream os;
149 os <<
"ContiguousUniformDirectory"
150 <<
"<" << Teuchos::TypeNameTraits<LO>::name ()
151 <<
", " << Teuchos::TypeNameTraits<GO>::name ()
152 <<
", " << Teuchos::TypeNameTraits<NT>::name () <<
">";
157 template<
class LO,
class GO,
class NT>
161 const Teuchos::ArrayView<const GO> &globalIDs,
162 const Teuchos::ArrayView<int> &nodeIDs,
163 const Teuchos::ArrayView<LO> &localIDs,
164 const bool computeLIDs)
const
168 typedef typename Teuchos::ArrayView<const GO>::size_type size_type;
169 const LO invalidLid = Teuchos::OrdinalTraits<LO>::invalid ();
172 RCP<const Comm<int> > comm = map.
getComm ();
196 const size_type N_G =
198 const size_type P =
static_cast<size_type
> (comm->getSize ());
199 const size_type N_L = N_G / P;
200 const size_type R = N_G - N_L * P;
201 const size_type N_R = R * (N_L +
static_cast<size_type
> (1));
203 #ifdef HAVE_TPETRA_DEBUG
204 TEUCHOS_TEST_FOR_EXCEPTION(
205 N_G != P*N_L + R, std::logic_error,
206 "Tpetra::ContiguousUniformDirectory::getEntriesImpl: "
207 "N_G = " << N_G <<
" != P*N_L + R = " << P <<
"*" << N_L <<
" + " << R
208 <<
" = " << P*N_L + R <<
". "
209 "Please report this bug to the Tpetra developers.");
210 #endif // HAVE_TPETRA_DEBUG
212 const size_type numGids = globalIDs.size ();
215 const GO ONE =
static_cast<GO
> (1);
218 for (size_type k = 0; k < numGids; ++k) {
219 const GO g_0 = globalIDs[k] - g_min;
225 if (g_0 + ONE < ONE || g_0 >= static_cast<GO> (N_G)) {
227 localIDs[k] = invalidLid;
230 else if (g_0 < static_cast<GO> (N_R)) {
232 nodeIDs[k] =
static_cast<int> (g_0 /
static_cast<GO
> (N_L + 1));
233 localIDs[k] =
static_cast<LO
> (g_0 %
static_cast<GO
> (N_L + 1));
235 else if (g_0 >= static_cast<GO> (N_R)) {
237 const GO g_R = g_0 -
static_cast<GO
> (N_R);
238 nodeIDs[k] =
static_cast<int> (R + g_R / N_L);
239 localIDs[k] =
static_cast<int> (g_R % N_L);
241 #ifdef HAVE_TPETRA_DEBUG
243 TEUCHOS_TEST_FOR_EXCEPTION(
true, std::logic_error,
244 "Tpetra::ContiguousUniformDirectory::getEntriesImpl: "
245 "should never get here. "
246 "Please report this bug to the Tpetra developers.");
248 #endif // HAVE_TPETRA_DEBUG
252 for (size_type k = 0; k < numGids; ++k) {
253 const GO g_0 = globalIDs[k] - g_min;
258 if (g_0 + ONE < ONE || g_0 >= static_cast<GO> (N_G)) {
262 else if (g_0 < static_cast<GO> (N_R)) {
264 nodeIDs[k] =
static_cast<int> (g_0 /
static_cast<GO
> (N_L + 1));
266 else if (g_0 >= static_cast<GO> (N_R)) {
268 const GO g_R = g_0 -
static_cast<GO
> (N_R);
269 nodeIDs[k] =
static_cast<int> (R + g_R / N_L);
271 #ifdef HAVE_TPETRA_DEBUG
273 TEUCHOS_TEST_FOR_EXCEPTION(
true, std::logic_error,
274 "Tpetra::ContiguousUniformDirectory::getEntriesImpl: "
275 "should never get here. "
276 "Please report this bug to the Tpetra developers.");
278 #endif // HAVE_TPETRA_DEBUG
284 template<
class LO,
class GO,
class NT>
289 using Teuchos::gatherAll;
292 RCP<const Teuchos::Comm<int> > comm = map.getComm ();
294 TEUCHOS_TEST_FOR_EXCEPTION(! map.isDistributed (), std::invalid_argument,
295 Teuchos::typeName (*
this) <<
" constructor: Map is not distributed.");
296 TEUCHOS_TEST_FOR_EXCEPTION(! map.isContiguous (), std::invalid_argument,
297 Teuchos::typeName (*
this) <<
" constructor: Map is not contiguous.");
299 const int numProcs = comm->getSize ();
303 allMinGIDs_ = arcp<GO> (numProcs + 1);
305 GO minMyGID = map.getMinGlobalIndex ();
316 #ifdef HAVE_TPETRACORE_MPI
317 MPI_Datatype rawMpiType = MPI_INT;
318 bool useRawMpi =
true;
319 if (
typeid (GO) ==
typeid (
int)) {
320 rawMpiType = MPI_INT;
321 }
else if (
typeid (GO) ==
typeid (
long)) {
322 rawMpiType = MPI_LONG;
327 using Teuchos::rcp_dynamic_cast;
328 using Teuchos::MpiComm;
329 RCP<const MpiComm<int> > mpiComm =
330 rcp_dynamic_cast<
const MpiComm<int> > (comm);
333 if (! comm.is_null ()) {
334 MPI_Comm rawMpiComm = * (mpiComm->getRawMpiComm ());
336 MPI_Allgather (&minMyGID, 1, rawMpiType,
337 allMinGIDs_.getRawPtr (), 1, rawMpiType,
339 TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
340 "Tpetra::DistributedContiguousDirectory: MPI_Allgather failed");
342 gatherAll<int, GO> (*comm, 1, &minMyGID, numProcs, allMinGIDs_.getRawPtr ());
345 gatherAll<int, GO> (*comm, 1, &minMyGID, numProcs, allMinGIDs_.getRawPtr ());
347 #else // NOT HAVE_TPETRACORE_MPI
348 gatherAll<int, GO> (*comm, 1, &minMyGID, numProcs, allMinGIDs_.getRawPtr ());
349 #endif // HAVE_TPETRACORE_MPI
356 allMinGIDs_[numProcs] = map.getMaxAllGlobalIndex ()
357 + Teuchos::OrdinalTraits<GO>::one ();
360 template<
class LO,
class GO,
class NT>
364 std::ostringstream os;
365 os <<
"DistributedContiguousDirectory"
366 <<
"<" << Teuchos::TypeNameTraits<LO>::name ()
367 <<
", " << Teuchos::TypeNameTraits<GO>::name ()
368 <<
", " << Teuchos::TypeNameTraits<NT>::name () <<
">";
372 template<
class LO,
class GO,
class NT>
376 const Teuchos::ArrayView<const GO> &globalIDs,
377 const Teuchos::ArrayView<int> &nodeIDs,
378 const Teuchos::ArrayView<LO> &localIDs,
379 const bool computeLIDs)
const
381 using Teuchos::Array;
382 using Teuchos::ArrayRCP;
383 using Teuchos::ArrayView;
389 RCP<const Teuchos::Comm<int> > comm = map.
getComm ();
390 const int myRank = comm->getRank ();
393 typename ArrayView<int>::iterator procIter = nodeIDs.begin();
394 typename ArrayView<LO>::iterator lidIter = localIDs.begin();
395 typename ArrayView<const GO>::iterator gidIter;
396 for (gidIter = globalIDs.begin(); gidIter != globalIDs.end(); ++gidIter) {
398 *procIter++ = myRank;
415 template<
class LO,
class GO,
class NT>
419 const Teuchos::ArrayView<const GO> &globalIDs,
420 const Teuchos::ArrayView<int> &nodeIDs,
421 const Teuchos::ArrayView<LO> &localIDs,
422 const bool computeLIDs)
const
424 using Teuchos::Array;
425 using Teuchos::ArrayRCP;
426 using Teuchos::ArrayView;
431 RCP<const Teuchos::Comm<int> > comm = map.
getComm ();
432 const int numProcs = comm->getSize ();
433 const LO LINVALID = Teuchos::OrdinalTraits<LO>::invalid();
434 const GO noGIDsOnProc = std::numeric_limits<GO>::max();
438 int firstProcWithGIDs;
439 for (firstProcWithGIDs = 0; firstProcWithGIDs < numProcs;
440 firstProcWithGIDs++) {
441 if (allMinGIDs_[firstProcWithGIDs] != noGIDsOnProc)
break;
447 if (firstProcWithGIDs == numProcs) {
450 std::fill(nodeIDs.begin(), nodeIDs.end(), -1);
452 std::fill(localIDs.begin(), localIDs.end(), LINVALID);
456 const GO one = as<GO> (1);
458 / as<global_size_t>(numProcs - firstProcWithGIDs));
461 typename ArrayView<int>::iterator procIter = nodeIDs.begin();
462 typename ArrayView<LO>::iterator lidIter = localIDs.begin();
463 typename ArrayView<const GO>::iterator gidIter;
464 for (gidIter = globalIDs.begin(); gidIter != globalIDs.end(); ++gidIter) {
473 const GO firstGuess = firstProcWithGIDs + GID / std::max(nOverP, one);
474 curRank = as<int>(std::min(firstGuess, as<GO>(numProcs - 1)));
478 while (allMinGIDs_[curRank] == noGIDsOnProc) curRank++;
481 while (curRank >= firstProcWithGIDs && GID < allMinGIDs_[curRank]) {
483 while (curRank >= firstProcWithGIDs &&
484 allMinGIDs_[curRank] == noGIDsOnProc) curRank--;
486 if (curRank < firstProcWithGIDs) {
490 else if (curRank == numProcs) {
496 int above = curRank + 1;
497 while (allMinGIDs_[above] == noGIDsOnProc) above++;
499 while (GID >= allMinGIDs_[above]) {
501 if (curRank == numProcs) {
507 while (allMinGIDs_[above] == noGIDsOnProc) above++;
513 LID = as<LO> (GID - allMinGIDs_[image]);
526 template<
class LO,
class GO,
class NT>
529 oneToOneResult_ (ONE_TO_ONE_NOT_CALLED_YET),
530 locallyOneToOne_ (true),
531 useHashTables_ (false)
536 template<
class LO,
class GO,
class NT>
537 DistributedNoncontiguousDirectory<LO, GO, NT>::
538 DistributedNoncontiguousDirectory (
const map_type& map,
539 const tie_break_type& tie_break) :
540 oneToOneResult_ (ONE_TO_ONE_NOT_CALLED_YET),
541 locallyOneToOne_ (true),
542 useHashTables_ (false)
544 initialize (map, Teuchos::ptrFromRef (tie_break));
547 template<
class LO,
class GO,
class NT>
549 DistributedNoncontiguousDirectory<LO, GO, NT>::
550 initialize (
const map_type& map,
551 Teuchos::Ptr<const tie_break_type> tie_break)
554 using Teuchos::Array;
555 using Teuchos::ArrayRCP;
556 using Teuchos::ArrayView;
560 using Teuchos::typeName;
561 using Teuchos::TypeNameTraits;
564 typedef Array<int>::size_type size_type;
574 TEUCHOS_TEST_FOR_EXCEPTION(
sizeof(
global_size_t) <
sizeof(GO),
575 std::logic_error, typeName (*
this) <<
": sizeof(Tpetra::"
576 "global_size_t) = " <<
sizeof(
global_size_t) <<
" < sizeof(Global"
577 "Ordinal = " << TypeNameTraits<LO>::name () <<
") = " <<
sizeof(GO)
579 TEUCHOS_TEST_FOR_EXCEPTION(
sizeof(
global_size_t) <
sizeof(
int),
580 std::logic_error, typeName (*
this) <<
": sizeof(Tpetra::"
581 "global_size_t) = " <<
sizeof(
global_size_t) <<
" < sizeof(int) = "
582 <<
sizeof(
int) <<
".");
583 TEUCHOS_TEST_FOR_EXCEPTION(
sizeof(
global_size_t) <
sizeof(LO),
584 std::logic_error, typeName (*
this) <<
": sizeof(Tpetra::"
585 "global_size_t) = " <<
sizeof(
global_size_t) <<
" < sizeof(Local"
586 "Ordinal = " << TypeNameTraits<LO>::name () <<
") = " <<
sizeof(LO)
589 RCP<const Teuchos::Comm<int> > comm = map.getComm ();
590 const LO LINVALID = Teuchos::OrdinalTraits<LO>::invalid ();
591 const GO minAllGID = map.getMinAllGlobalIndex ();
592 const GO maxAllGID = map.getMaxAllGlobalIndex ();
599 const global_size_t numGlobalEntries = maxAllGID - minAllGID + 1;
609 directoryMap_ = rcp (
new map_type (numGlobalEntries, minAllGID, comm,
610 GloballyDistributed));
612 const size_t dir_numMyEntries = directoryMap_->getLocalNumElements ();
636 const size_t inverseSparsityThreshold = 10;
638 (dir_numMyEntries >= inverseSparsityThreshold * map.getLocalNumElements());
643 const int myRank = comm->getRank ();
644 const size_t numMyEntries = map.getLocalNumElements ();
645 Array<int> sendImageIDs (numMyEntries);
646 ArrayView<const GO> myGlobalEntries = map.getLocalElementList ();
651 directoryMap_->getRemoteIndexList (myGlobalEntries, sendImageIDs);
652 TEUCHOS_TEST_FOR_EXCEPTION(
653 lookupStatus ==
IDNotPresent, std::logic_error, Teuchos::typeName(*
this)
654 <<
" constructor: the Directory Map could not find out where one or "
655 "more of my Map's indices should go. The input to getRemoteIndexList "
656 "is " << Teuchos::toString (myGlobalEntries) <<
", and the output is "
657 << Teuchos::toString (sendImageIDs ()) <<
". The input Map itself has "
658 "the following entries on the calling process " <<
659 map.getComm ()->getRank () <<
": " <<
660 Teuchos::toString (map.getLocalElementList ()) <<
", and has "
661 << map.getGlobalNumElements () <<
" total global indices in ["
662 << map.getMinAllGlobalIndex () <<
"," << map.getMaxAllGlobalIndex ()
663 <<
"]. The Directory Map has "
664 << directoryMap_->getGlobalNumElements () <<
" total global indices in "
665 "[" << directoryMap_->getMinAllGlobalIndex () <<
"," <<
666 directoryMap_->getMaxAllGlobalIndex () <<
"], and the calling process "
667 "has GIDs [" << directoryMap_->getMinGlobalIndex () <<
"," <<
668 directoryMap_->getMaxGlobalIndex () <<
"]. "
669 "This probably means there is a bug in Map or Directory. "
670 "Please report this bug to the Tpetra developers.");
677 Distributor distor (comm);
678 const size_t numReceives = distor.createFromSends (sendImageIDs);
692 const int packetSize = 3;
693 Kokkos::View<GO*, Kokkos::HostSpace> exportEntries(
"exportEntries", packetSize * numMyEntries);
695 size_t exportIndex = 0;
696 for (
size_t i = 0; i < numMyEntries; ++i) {
697 exportEntries[exportIndex++] = myGlobalEntries[i];
698 exportEntries[exportIndex++] = as<GO> (myRank);
699 exportEntries[exportIndex++] = as<GO> (i);
705 Kokkos::View<GO*, Kokkos::HostSpace> importElements(
"importElements", packetSize * distor.getTotalReceiveLength());
708 distor.doPostsAndWaits(exportEntries, packetSize, importElements);
715 if (useHashTables_) {
730 LO* tableKeysRaw = NULL;
731 LO* tableLidsRaw = NULL;
732 int* tablePidsRaw = NULL;
734 tableKeysRaw =
new LO [numReceives];
735 tableLidsRaw =
new LO [numReceives];
736 tablePidsRaw =
new int [numReceives];
738 if (tableKeysRaw != NULL) {
739 delete [] tableKeysRaw;
741 if (tableLidsRaw != NULL) {
742 delete [] tableLidsRaw;
744 if (tablePidsRaw != NULL) {
745 delete [] tablePidsRaw;
749 ArrayRCP<LO> tableKeys (tableKeysRaw, 0, numReceives,
true);
750 ArrayRCP<LO> tableLids (tableLidsRaw, 0, numReceives,
true);
751 ArrayRCP<int> tablePids (tablePidsRaw, 0, numReceives,
true);
753 if (tie_break.is_null ()) {
755 size_type importIndex = 0;
756 for (size_type i = 0; i < static_cast<size_type> (numReceives); ++i) {
757 const GO curGID = importElements[importIndex++];
758 const LO curLID = directoryMap_->getLocalElement (curGID);
759 TEUCHOS_TEST_FOR_EXCEPTION(
760 curLID == LINVALID, std::logic_error,
761 Teuchos::typeName(*
this) <<
" constructor: Incoming global index "
762 << curGID <<
" does not have a corresponding local index in the "
763 "Directory Map. Please report this bug to the Tpetra developers.");
764 tableKeys[i] = curLID;
765 tablePids[i] = importElements[importIndex++];
766 tableLids[i] = importElements[importIndex++];
772 rcp (
new lidToPidTable_type (tableKeys (), tablePids ()));
773 locallyOneToOne_ = ! (lidToPidTable_->hasDuplicateKeys ());
775 rcp (
new lidToLidTable_type (tableKeys (), tableLids ()));
784 typedef std::map<LO, std::vector<std::pair<int, LO> > > pair_table_type;
785 pair_table_type ownedPidLidPairs;
789 size_type importIndex = 0;
790 for (size_type i = 0; i < static_cast<size_type> (numReceives); ++i) {
791 const GO curGID = importElements[importIndex++];
792 const LO dirMapLid = directoryMap_->getLocalElement (curGID);
793 TEUCHOS_TEST_FOR_EXCEPTION(
794 dirMapLid == LINVALID, std::logic_error,
795 Teuchos::typeName(*
this) <<
" constructor: Incoming global index "
796 << curGID <<
" does not have a corresponding local index in the "
797 "Directory Map. Please report this bug to the Tpetra developers.");
798 tableKeys[i] = dirMapLid;
799 const int PID = importElements[importIndex++];
800 const int LID = importElements[importIndex++];
810 ownedPidLidPairs[dirMapLid].push_back (std::make_pair (PID, LID));
823 for (size_type i = 0; i < static_cast<size_type> (numReceives); ++i) {
824 const LO dirMapLid = tableKeys[i];
825 const std::vector<std::pair<int, LO> >& pidLidList =
826 ownedPidLidPairs[dirMapLid];
827 const size_t listLen = pidLidList.size();
828 if (listLen == 0)
continue;
829 const GO dirMapGid = directoryMap_->getGlobalElement (dirMapLid);
831 locallyOneToOne_ =
false;
840 const size_type index =
841 static_cast<size_type
> (tie_break->selectedIndex (dirMapGid,
843 tablePids[i] = pidLidList[index].first;
844 tableLids[i] = pidLidList[index].second;
849 rcp (
new lidToPidTable_type (tableKeys (), tablePids ()));
851 rcp (
new lidToLidTable_type (tableKeys (), tableLids ()));
855 if (tie_break.is_null ()) {
860 PIDs_ = arcp<int> (dir_numMyEntries);
861 std::fill (PIDs_.begin (), PIDs_.end (), -1);
862 LIDs_ = arcp<LO> (dir_numMyEntries);
863 std::fill (LIDs_.begin (), LIDs_.end (), LINVALID);
865 size_type importIndex = 0;
866 for (size_type i = 0; i < static_cast<size_type> (numReceives); ++i) {
867 const GO curGID = importElements[importIndex++];
868 const LO curLID = directoryMap_->getLocalElement (curGID);
869 TEUCHOS_TEST_FOR_EXCEPTION(curLID == LINVALID, std::logic_error,
870 Teuchos::typeName(*
this) <<
" constructor: Incoming global index "
871 << curGID <<
" does not have a corresponding local index in the "
872 "Directory Map. Please report this bug to the Tpetra developers.");
877 if (PIDs_[curLID] != -1) {
878 locallyOneToOne_ =
false;
880 PIDs_[curLID] = importElements[importIndex++];
881 LIDs_[curLID] = importElements[importIndex++];
885 PIDs_ = arcp<int> (dir_numMyEntries);
886 LIDs_ = arcp<LO> (dir_numMyEntries);
887 std::fill (PIDs_.begin (), PIDs_.end (), -1);
896 Array<std::vector<std::pair<int, LO> > > ownedPidLidPairs (dir_numMyEntries);
897 size_t importIndex = 0;
898 for (
size_t i = 0; i < numReceives; ++i) {
899 const GO GID = importElements[importIndex++];
900 const int PID = importElements[importIndex++];
901 const LO LID = importElements[importIndex++];
903 const LO dirMapLid = directoryMap_->getLocalElement (GID);
904 TEUCHOS_TEST_FOR_EXCEPTION(
905 dirMapLid == LINVALID, std::logic_error,
906 Teuchos::typeName(*
this) <<
" constructor: Incoming global index "
907 << GID <<
" does not have a corresponding local index in the "
908 "Directory Map. Please report this bug to the Tpetra developers.");
909 ownedPidLidPairs[dirMapLid].push_back (std::make_pair (PID, LID));
921 for (
size_t i = 0; i < dir_numMyEntries; ++i) {
922 const std::vector<std::pair<int, LO> >& pidLidList =
924 const size_t listLen = pidLidList.size();
925 if (listLen == 0)
continue;
927 const LO dirMapLid =
static_cast<LO
> (i);
928 const GO dirMapGid = directoryMap_->getGlobalElement (dirMapLid);
930 locallyOneToOne_ =
false;
939 const size_type index =
940 static_cast<size_type
> (tie_break->selectedIndex (dirMapGid,
942 PIDs_[i] = pidLidList[index].first;
943 LIDs_[i] = pidLidList[index].second;
949 template<
class LO,
class GO,
class NT>
953 std::ostringstream os;
954 os <<
"DistributedNoncontiguousDirectory"
955 <<
"<" << Teuchos::TypeNameTraits<LO>::name ()
956 <<
", " << Teuchos::TypeNameTraits<GO>::name ()
957 <<
", " << Teuchos::TypeNameTraits<NT>::name () <<
">";
961 template<
class LO,
class GO,
class NT>
965 const Teuchos::ArrayView<const GO> &globalIDs,
966 const Teuchos::ArrayView<int> &nodeIDs,
967 const Teuchos::ArrayView<LO> &localIDs,
968 const bool computeLIDs)
const
970 using Teuchos::Array;
971 using Teuchos::ArrayRCP;
972 using Teuchos::ArrayView;
975 using Teuchos::toString;
980 using size_type =
typename Array<GO>::size_type;
981 const char funcPrefix[] =
"Tpetra::"
982 "DistributedNoncontiguousDirectory::getEntriesImpl: ";
983 const char errSuffix[] =
984 " Please report this bug to the Tpetra developers.";
986 RCP<const Teuchos::Comm<int> > comm = map.getComm ();
989 const size_t maxNumToPrint = verbose ?
992 std::unique_ptr<std::string> procPrefix;
994 std::ostringstream os;
996 if (map.getComm ().is_null ()) {
1000 os << map.getComm ()->getRank ();
1003 procPrefix = std::unique_ptr<std::string>(
1004 new std::string(os.str()));
1005 os << funcPrefix <<
"{";
1011 os <<
", computeLIDs: "
1012 << (computeLIDs ?
"true" :
"false") <<
"}" << endl;
1016 const size_t numEntries = globalIDs.size ();
1017 const LO LINVALID = Teuchos::OrdinalTraits<LO>::invalid();
1026 const int packetSize = computeLIDs ? 3 : 2;
1032 Array<int> dirImages (numEntries);
1034 std::ostringstream os;
1035 os << *procPrefix <<
"Call directoryMap_->getRemoteIndexList"
1039 res = directoryMap_->getRemoteIndexList (globalIDs, dirImages ());
1041 std::ostringstream os;
1042 os << *procPrefix <<
"Director Map getRemoteIndexList out ";
1049 size_t numMissing = 0;
1051 for (
size_t i=0; i < numEntries; ++i) {
1052 if (dirImages[i] == -1) {
1055 localIDs[i] = LINVALID;
1063 Array<int> sendImages;
1065 std::ostringstream os;
1066 os << *procPrefix <<
"Call Distributor::createFromRecvs"
1070 distor.
createFromRecvs (globalIDs, dirImages (), sendGIDs, sendImages);
1072 std::ostringstream os;
1073 os << *procPrefix <<
"Distributor::createFromRecvs result: "
1081 const size_type numSends = sendGIDs.size ();
1114 Kokkos::View<global_size_t*, Kokkos::HostSpace> exports(
"exports", packetSize * numSends);
1127 size_t exportsIndex = 0;
1129 if (useHashTables_) {
1131 std::ostringstream os;
1132 os << *procPrefix <<
"Pack exports (useHashTables_ true)"
1136 for (size_type gidIndex = 0; gidIndex < numSends; ++gidIndex) {
1137 const GO curGID = sendGIDs[gidIndex];
1139 exports[exportsIndex++] =
static_cast<global_size_t> (curGID);
1140 const LO curLID = directoryMap_->getLocalElement (curGID);
1141 TEUCHOS_TEST_FOR_EXCEPTION
1142 (curLID == LINVALID, std::logic_error, funcPrefix <<
1143 "Directory Map's global index " << curGID <<
" lacks "
1144 "a corresponding local index." << errSuffix);
1146 exports[exportsIndex++] =
1150 exports[exportsIndex++] =
1157 std::ostringstream os;
1158 os << *procPrefix <<
"Pack exports (useHashTables_ false)"
1162 for (size_type gidIndex = 0; gidIndex < numSends; ++gidIndex) {
1163 const GO curGID = sendGIDs[gidIndex];
1165 exports[exportsIndex++] =
static_cast<global_size_t> (curGID);
1166 const LO curLID = directoryMap_->getLocalElement (curGID);
1167 TEUCHOS_TEST_FOR_EXCEPTION
1168 (curLID == LINVALID, std::logic_error, funcPrefix <<
1169 "Directory Map's global index " << curGID <<
" lacks "
1170 "a corresponding local index." << errSuffix);
1172 exports[exportsIndex++] =
1176 exports[exportsIndex++] =
1182 TEUCHOS_TEST_FOR_EXCEPTION
1183 (exportsIndex > exports.size(), std::logic_error,
1184 funcPrefix <<
"On Process " << comm->getRank () <<
", "
1185 "exportsIndex = " << exportsIndex <<
" > exports.size() = "
1186 << exports.size () <<
"." << errSuffix);
1189 TEUCHOS_TEST_FOR_EXCEPTION
1190 (numEntries < numMissing, std::logic_error, funcPrefix <<
1191 "On Process " << comm->getRank () <<
", numEntries = " <<
1192 numEntries <<
" < numMissing = " << numMissing <<
"." <<
1199 const size_t numRecv = numEntries - numMissing;
1203 const size_t requiredImportLen = numRecv * packetSize;
1204 const int myRank = comm->getRank ();
1205 TEUCHOS_TEST_FOR_EXCEPTION(
1206 importLen < requiredImportLen, std::logic_error,
1207 "Tpetra::Details::DistributedNoncontiguousDirectory::getEntriesImpl: "
1208 "On Process " << myRank <<
": The 'imports' array must have length "
1209 "at least " << requiredImportLen <<
", but its actual length is " <<
1210 importLen <<
". numRecv: " << numRecv <<
", packetSize: " <<
1211 packetSize <<
", numEntries (# GIDs): " << numEntries <<
1212 ", numMissing: " << numMissing <<
": distor.getTotalReceiveLength(): "
1214 "Distributor description: " << distor.
description () <<
". "
1216 "Please report this bug to the Tpetra developers.");
1219 Kokkos::View<global_size_t*, Kokkos::HostSpace> imports(
"imports", packetSize * distor.
getTotalReceiveLength());
1224 std::ostringstream os;
1225 os << *procPrefix <<
"Call doPostsAndWaits: {"
1226 <<
"packetSize: " << packetSize <<
", ";
1233 std::ostringstream os;
1234 os << *procPrefix <<
"doPostsAndWaits result: ";
1240 Array<GO> sortedIDs (globalIDs);
1241 Array<GO> offset (numEntries);
1242 for (GO ii = 0; ii < static_cast<GO> (numEntries); ++ii) {
1245 sort2 (sortedIDs.begin(), sortedIDs.begin() + numEntries, offset.begin());
1247 std::ostringstream os;
1256 size_t importsIndex = 0;
1261 for (
size_t i = 0; i < numRecv; ++i) {
1262 const GO curGID =
static_cast<GO
> (imports[importsIndex++]);
1263 auto p1 = std::equal_range (sortedIDs.begin(),
1264 sortedIDs.end(), curGID);
1265 if (p1.first != p1.second) {
1266 const size_t j = p1.first - sortedIDs.begin();
1267 nodeIDs[offset[j]] =
1268 static_cast<int> (imports[importsIndex++]);
1270 localIDs[offset[j]] =
1271 static_cast<LO
> (imports[importsIndex++]);
1273 if (nodeIDs[offset[j]] == -1) {
1279 TEUCHOS_TEST_FOR_EXCEPTION
1280 (
size_t (importsIndex) >
size_t (imports.size ()),
1281 std::logic_error, funcPrefix <<
"On Process " <<
1282 comm->getRank () <<
": importsIndex = " << importsIndex
1283 <<
" > imports.size() = " << imports.size () <<
". "
1284 "numRecv: " << numRecv
1285 <<
", packetSize: " << packetSize <<
", "
1286 "numEntries (# GIDs): " << numEntries
1287 <<
", numMissing: " << numMissing
1288 <<
": distor.getTotalReceiveLength(): "
1291 std::ostringstream os;
1292 os << *procPrefix << funcPrefix <<
"Done!" << endl;
1299 template<
class LO,
class GO,
class NT>
1304 if (oneToOneResult_ == ONE_TO_ONE_NOT_CALLED_YET) {
1305 const int lcl121 = isLocallyOneToOne () ? 1 : 0;
1307 Teuchos::reduceAll<int, int> (comm, Teuchos::REDUCE_MIN, lcl121,
1308 Teuchos::outArg (gbl121));
1309 oneToOneResult_ = (gbl121 == 1) ? ONE_TO_ONE_TRUE : ONE_TO_ONE_FALSE;
1311 return (oneToOneResult_ == ONE_TO_ONE_TRUE);
1321 #define TPETRA_DIRECTORYIMPL_INSTANT(LO,GO,NODE) \
1322 namespace Details { \
1323 template class Directory< LO , GO , NODE >; \
1324 template class ReplicatedDirectory< LO , GO , NODE >; \
1325 template class ContiguousUniformDirectory< LO, GO, NODE >; \
1326 template class DistributedContiguousDirectory< LO , GO , NODE >; \
1327 template class DistributedNoncontiguousDirectory< LO , GO , NODE >; \
1330 #endif // TPETRA_DIRECTORYIMPL_DEF_HPP
void initialize(int *argc, char ***argv)
Initialize Tpetra.
std::string description() const override
A one-line human-readable description of this object.
Interface for breaking ties in ownership.
std::string description() const
Return a one-line description of this object.
LookupStatus
Return status of Map remote index lookup (getRemoteIndexList()).
Implementation of Directory for a distributed noncontiguous Map.
Teuchos::RCP< const Teuchos::Comm< int > > getComm() const
Accessors for the Teuchos::Comm and Kokkos Node objects.
bool isOneToOne(const Teuchos::Comm< int > &comm) const override
Whether the Directory's input Map is (globally) one to one.
Implementation of Directory for a distributed contiguous Map.
void verbosePrintArray(std::ostream &out, const ArrayType &x, const char name[], const size_t maxNumToPrint)
Print min(x.size(), maxNumToPrint) entries of x.
LookupStatus getEntriesImpl(const map_type &map, const Teuchos::ArrayView< const GlobalOrdinal > &globalIDs, const Teuchos::ArrayView< int > &nodeIDs, const Teuchos::ArrayView< LocalOrdinal > &localIDs, const bool computeLIDs) const override
Find process IDs and (optionally) local IDs for the given global IDs.
bool isOneToOne(const Teuchos::Comm< int > &comm) const override
Whether the Directory's input Map is (globally) one to one.
size_t global_size_t
Global size_t object.
LookupStatus getEntriesImpl(const map_type &map, const Teuchos::ArrayView< const GlobalOrdinal > &globalIDs, const Teuchos::ArrayView< int > &nodeIDs, const Teuchos::ArrayView< LocalOrdinal > &localIDs, const bool computeLIDs) const override
Find process IDs and (optionally) local IDs for the given global IDs.
LookupStatus getEntries(const map_type &map, const Teuchos::ArrayView< const GlobalOrdinal > &globalIDs, const Teuchos::ArrayView< int > &nodeIDs, const Teuchos::ArrayView< LocalOrdinal > &localIDs, const bool computeLIDs) const
bool isNodeGlobalElement(global_ordinal_type globalIndex) const
Whether the given global index is owned by this Map on the calling process.
Sets up and executes a communication plan for a Tpetra DistObject.
static bool verbose()
Whether Tpetra is in verbose mode.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
std::string description() const override
A one-line human-readable description of this object.
void sort2(const IT1 &first1, const IT1 &last1, const IT2 &first2)
Sort the first array, and apply the resulting permutation to the second array.
A parallel distribution of indices over processes.
ReplicatedDirectory()=default
Constructor (that takes no arguments).
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the (forward) communication plan.
local_ordinal_type getLocalElement(global_ordinal_type globalIndex) const
The local index corresponding to the given global index.
Stand-alone utility functions and macros.
static size_t verbosePrintCountThreshold()
Number of entries below which arrays, lists, etc. will be printed in debug mode.
LookupStatus getEntriesImpl(const map_type &map, const Teuchos::ArrayView< const GlobalOrdinal > &globalIDs, const Teuchos::ArrayView< int > &nodeIDs, const Teuchos::ArrayView< LocalOrdinal > &localIDs, const bool computeLIDs) const override
Find process IDs and (optionally) local IDs for the given global IDs.
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
std::string description() const override
A one-line human-readable description of this object.
global_ordinal_type getMinAllGlobalIndex() const
The minimum global index over all processes in the communicator.
global_size_t getGlobalNumElements() const
The number of elements in this Map.
Description of Tpetra's behavior.