42 #ifndef __Tpetra_DirectoryImpl_def_hpp
43 #define __Tpetra_DirectoryImpl_def_hpp
48 #include "Tpetra_Distributor.hpp"
49 #include "Tpetra_Map.hpp"
51 #include "Tpetra_Details_FixedHashTable.hpp"
52 #include "Teuchos_Comm.hpp"
57 #ifdef HAVE_TPETRACORE_MPI
59 #endif // HAVE_TPETRACORE_MPI
64 template<
class LO,
class GO,
class NT>
68 const Teuchos::ArrayView<const GO> &globalIDs,
69 const Teuchos::ArrayView<int> &nodeIDs,
70 const Teuchos::ArrayView<LO> &localIDs,
71 const bool computeLIDs)
const
75 TEUCHOS_TEST_FOR_EXCEPTION(nodeIDs.size() != globalIDs.size(),
76 std::invalid_argument, Teuchos::typeName(*
this) <<
"::getEntries(): "
77 "Output arrays do not have the right sizes. nodeIDs.size() = "
78 << nodeIDs.size() <<
" != globalIDs.size() = " << globalIDs.size()
80 TEUCHOS_TEST_FOR_EXCEPTION(
81 computeLIDs && localIDs.size() != globalIDs.size(),
82 std::invalid_argument, Teuchos::typeName(*
this) <<
"::getEntries(): "
83 "Output array do not have the right sizes. localIDs.size() = "
84 << localIDs.size() <<
" != globalIDs.size() = " << globalIDs.size()
92 std::fill (nodeIDs.begin(), nodeIDs.end(), -1);
94 std::fill (localIDs.begin(), localIDs.end(),
95 Teuchos::OrdinalTraits<LO>::invalid ());
98 return this->getEntriesImpl (map, globalIDs, nodeIDs, localIDs, computeLIDs);
102 template<
class LO,
class GO,
class NT>
105 numProcs_ (map.getComm ()->getSize ())
109 template<
class LO,
class GO,
class NT>
117 return (numProcs_ == 1);
121 template<
class LO,
class GO,
class NT>
125 std::ostringstream os;
126 os <<
"ReplicatedDirectory"
127 <<
"<" << Teuchos::TypeNameTraits<LO>::name ()
128 <<
", " << Teuchos::TypeNameTraits<GO>::name ()
129 <<
", " << Teuchos::TypeNameTraits<NT>::name () <<
">";
134 template<
class LO,
class GO,
class NT>
138 TEUCHOS_TEST_FOR_EXCEPTION(! map.isContiguous (), std::invalid_argument,
139 Teuchos::typeName (*
this) <<
" constructor: Map is not contiguous.");
140 TEUCHOS_TEST_FOR_EXCEPTION(! map.isUniform (), std::invalid_argument,
141 Teuchos::typeName (*
this) <<
" constructor: Map is not uniform.");
145 template<
class LO,
class GO,
class NT>
149 std::ostringstream os;
150 os <<
"ContiguousUniformDirectory"
151 <<
"<" << Teuchos::TypeNameTraits<LO>::name ()
152 <<
", " << Teuchos::TypeNameTraits<GO>::name ()
153 <<
", " << Teuchos::TypeNameTraits<NT>::name () <<
">";
158 template<
class LO,
class GO,
class NT>
162 const Teuchos::ArrayView<const GO> &globalIDs,
163 const Teuchos::ArrayView<int> &nodeIDs,
164 const Teuchos::ArrayView<LO> &localIDs,
165 const bool computeLIDs)
const
169 typedef typename Teuchos::ArrayView<const GO>::size_type size_type;
170 const LO invalidLid = Teuchos::OrdinalTraits<LO>::invalid ();
173 RCP<const Comm<int> > comm = map.
getComm ();
197 const size_type N_G =
199 const size_type P =
static_cast<size_type
> (comm->getSize ());
200 const size_type N_L = N_G / P;
201 const size_type R = N_G - N_L * P;
202 const size_type N_R = R * (N_L +
static_cast<size_type
> (1));
204 #ifdef HAVE_TPETRA_DEBUG
205 TEUCHOS_TEST_FOR_EXCEPTION(
206 N_G != P*N_L + R, std::logic_error,
207 "Tpetra::ContiguousUniformDirectory::getEntriesImpl: "
208 "N_G = " << N_G <<
" != P*N_L + R = " << P <<
"*" << N_L <<
" + " << R
209 <<
" = " << P*N_L + R <<
". "
210 "Please report this bug to the Tpetra developers.");
211 #endif // HAVE_TPETRA_DEBUG
213 const size_type numGids = globalIDs.size ();
216 const GO ONE =
static_cast<GO
> (1);
219 for (size_type k = 0; k < numGids; ++k) {
220 const GO g_0 = globalIDs[k] - g_min;
226 if (g_0 + ONE < ONE || g_0 >= static_cast<GO> (N_G)) {
228 localIDs[k] = invalidLid;
231 else if (g_0 < static_cast<GO> (N_R)) {
233 nodeIDs[k] =
static_cast<int> (g_0 /
static_cast<GO
> (N_L + 1));
234 localIDs[k] =
static_cast<LO
> (g_0 %
static_cast<GO
> (N_L + 1));
236 else if (g_0 >= static_cast<GO> (N_R)) {
238 const GO g_R = g_0 -
static_cast<GO
> (N_R);
239 nodeIDs[k] =
static_cast<int> (R + g_R / N_L);
240 localIDs[k] =
static_cast<int> (g_R % N_L);
242 #ifdef HAVE_TPETRA_DEBUG
244 TEUCHOS_TEST_FOR_EXCEPTION(
true, std::logic_error,
245 "Tpetra::ContiguousUniformDirectory::getEntriesImpl: "
246 "should never get here. "
247 "Please report this bug to the Tpetra developers.");
249 #endif // HAVE_TPETRA_DEBUG
253 for (size_type k = 0; k < numGids; ++k) {
254 const GO g_0 = globalIDs[k] - g_min;
259 if (g_0 + ONE < ONE || g_0 >= static_cast<GO> (N_G)) {
263 else if (g_0 < static_cast<GO> (N_R)) {
265 nodeIDs[k] =
static_cast<int> (g_0 /
static_cast<GO
> (N_L + 1));
267 else if (g_0 >= static_cast<GO> (N_R)) {
269 const GO g_R = g_0 -
static_cast<GO
> (N_R);
270 nodeIDs[k] =
static_cast<int> (R + g_R / N_L);
272 #ifdef HAVE_TPETRA_DEBUG
274 TEUCHOS_TEST_FOR_EXCEPTION(
true, std::logic_error,
275 "Tpetra::ContiguousUniformDirectory::getEntriesImpl: "
276 "should never get here. "
277 "Please report this bug to the Tpetra developers.");
279 #endif // HAVE_TPETRA_DEBUG
285 template<
class LO,
class GO,
class NT>
290 using Teuchos::gatherAll;
293 RCP<const Teuchos::Comm<int> > comm = map.getComm ();
295 TEUCHOS_TEST_FOR_EXCEPTION(! map.isDistributed (), std::invalid_argument,
296 Teuchos::typeName (*
this) <<
" constructor: Map is not distributed.");
297 TEUCHOS_TEST_FOR_EXCEPTION(! map.isContiguous (), std::invalid_argument,
298 Teuchos::typeName (*
this) <<
" constructor: Map is not contiguous.");
300 const int numProcs = comm->getSize ();
304 allMinGIDs_ = arcp<GO> (numProcs + 1);
306 GO minMyGID = map.getMinGlobalIndex ();
317 #ifdef HAVE_TPETRACORE_MPI
318 MPI_Datatype rawMpiType = MPI_INT;
319 bool useRawMpi =
true;
320 if (
typeid (GO) ==
typeid (
int)) {
321 rawMpiType = MPI_INT;
322 }
else if (
typeid (GO) ==
typeid (
long)) {
323 rawMpiType = MPI_LONG;
328 using Teuchos::rcp_dynamic_cast;
329 using Teuchos::MpiComm;
330 RCP<const MpiComm<int> > mpiComm =
331 rcp_dynamic_cast<
const MpiComm<int> > (comm);
334 if (! comm.is_null ()) {
335 MPI_Comm rawMpiComm = * (mpiComm->getRawMpiComm ());
337 MPI_Allgather (&minMyGID, 1, rawMpiType,
338 allMinGIDs_.getRawPtr (), 1, rawMpiType,
340 TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
341 "Tpetra::DistributedContiguousDirectory: MPI_Allgather failed");
343 gatherAll<int, GO> (*comm, 1, &minMyGID, numProcs, allMinGIDs_.getRawPtr ());
346 gatherAll<int, GO> (*comm, 1, &minMyGID, numProcs, allMinGIDs_.getRawPtr ());
348 #else // NOT HAVE_TPETRACORE_MPI
349 gatherAll<int, GO> (*comm, 1, &minMyGID, numProcs, allMinGIDs_.getRawPtr ());
350 #endif // HAVE_TPETRACORE_MPI
357 allMinGIDs_[numProcs] = map.getMaxAllGlobalIndex ()
358 + Teuchos::OrdinalTraits<GO>::one ();
361 template<
class LO,
class GO,
class NT>
365 std::ostringstream os;
366 os <<
"DistributedContiguousDirectory"
367 <<
"<" << Teuchos::TypeNameTraits<LO>::name ()
368 <<
", " << Teuchos::TypeNameTraits<GO>::name ()
369 <<
", " << Teuchos::TypeNameTraits<NT>::name () <<
">";
373 template<
class LO,
class GO,
class NT>
377 const Teuchos::ArrayView<const GO> &globalIDs,
378 const Teuchos::ArrayView<int> &nodeIDs,
379 const Teuchos::ArrayView<LO> &localIDs,
380 const bool computeLIDs)
const
382 using Teuchos::Array;
383 using Teuchos::ArrayRCP;
384 using Teuchos::ArrayView;
390 RCP<const Teuchos::Comm<int> > comm = map.
getComm ();
391 const int myRank = comm->getRank ();
394 typename ArrayView<int>::iterator procIter = nodeIDs.begin();
395 typename ArrayView<LO>::iterator lidIter = localIDs.begin();
396 typename ArrayView<const GO>::iterator gidIter;
397 for (gidIter = globalIDs.begin(); gidIter != globalIDs.end(); ++gidIter) {
399 *procIter++ = myRank;
416 template<
class LO,
class GO,
class NT>
420 const Teuchos::ArrayView<const GO> &globalIDs,
421 const Teuchos::ArrayView<int> &nodeIDs,
422 const Teuchos::ArrayView<LO> &localIDs,
423 const bool computeLIDs)
const
425 using Teuchos::Array;
426 using Teuchos::ArrayRCP;
427 using Teuchos::ArrayView;
432 RCP<const Teuchos::Comm<int> > comm = map.
getComm ();
433 const int numProcs = comm->getSize ();
435 const LO LINVALID = Teuchos::OrdinalTraits<LO>::invalid();
439 typename ArrayView<int>::iterator procIter = nodeIDs.begin();
440 typename ArrayView<LO>::iterator lidIter = localIDs.begin();
441 typename ArrayView<const GO>::iterator gidIter;
442 for (gidIter = globalIDs.begin(); gidIter != globalIDs.end(); ++gidIter) {
452 const GO one = as<GO> (1);
453 const GO two = as<GO> (2);
454 const GO nOverP_GID = as<GO> (nOverP);
455 const GO lowerBound = GID / std::max(nOverP_GID, one) + two;
456 curRank = as<int>(std::min(lowerBound, as<GO>(numProcs - 1)));
459 while (curRank >= 0 && curRank < numProcs) {
460 if (allMinGIDs_[curRank] <= GID) {
461 if (GID < allMinGIDs_[curRank + 1]) {
475 LID = as<LO> (GID - allMinGIDs_[image]);
488 template<
class LO,
class GO,
class NT>
491 oneToOneResult_ (ONE_TO_ONE_NOT_CALLED_YET),
492 locallyOneToOne_ (true),
493 useHashTables_ (false)
498 template<
class LO,
class GO,
class NT>
499 DistributedNoncontiguousDirectory<LO, GO, NT>::
500 DistributedNoncontiguousDirectory (
const map_type& map,
501 const tie_break_type& tie_break) :
502 oneToOneResult_ (ONE_TO_ONE_NOT_CALLED_YET),
503 locallyOneToOne_ (true),
504 useHashTables_ (false)
506 initialize (map, Teuchos::ptrFromRef (tie_break));
509 template<
class LO,
class GO,
class NT>
511 DistributedNoncontiguousDirectory<LO, GO, NT>::
512 initialize (
const map_type& map,
513 Teuchos::Ptr<const tie_break_type> tie_break)
516 using Teuchos::Array;
517 using Teuchos::ArrayRCP;
518 using Teuchos::ArrayView;
522 using Teuchos::typeName;
523 using Teuchos::TypeNameTraits;
526 typedef Array<int>::size_type size_type;
536 TEUCHOS_TEST_FOR_EXCEPTION(
sizeof(
global_size_t) <
sizeof(GO),
537 std::logic_error, typeName (*
this) <<
": sizeof(Tpetra::"
538 "global_size_t) = " <<
sizeof(
global_size_t) <<
" < sizeof(Global"
539 "Ordinal = " << TypeNameTraits<LO>::name () <<
") = " <<
sizeof(GO)
541 TEUCHOS_TEST_FOR_EXCEPTION(
sizeof(
global_size_t) <
sizeof(
int),
542 std::logic_error, typeName (*
this) <<
": sizeof(Tpetra::"
543 "global_size_t) = " <<
sizeof(
global_size_t) <<
" < sizeof(int) = "
544 <<
sizeof(
int) <<
".");
545 TEUCHOS_TEST_FOR_EXCEPTION(
sizeof(
global_size_t) <
sizeof(LO),
546 std::logic_error, typeName (*
this) <<
": sizeof(Tpetra::"
547 "global_size_t) = " <<
sizeof(
global_size_t) <<
" < sizeof(Local"
548 "Ordinal = " << TypeNameTraits<LO>::name () <<
") = " <<
sizeof(LO)
551 RCP<const Teuchos::Comm<int> > comm = map.getComm ();
552 const LO LINVALID = Teuchos::OrdinalTraits<LO>::invalid ();
553 const GO minAllGID = map.getMinAllGlobalIndex ();
554 const GO maxAllGID = map.getMaxAllGlobalIndex ();
561 const global_size_t numGlobalEntries = maxAllGID - minAllGID + 1;
571 directoryMap_ = rcp (
new map_type (numGlobalEntries, minAllGID, comm,
572 GloballyDistributed));
574 const size_t dir_numMyEntries = directoryMap_->getNodeNumElements ();
598 const size_t inverseSparsityThreshold = 10;
600 (dir_numMyEntries >= inverseSparsityThreshold * map.getNodeNumElements());
605 const int myRank = comm->getRank ();
606 const size_t numMyEntries = map.getNodeNumElements ();
607 Array<int> sendImageIDs (numMyEntries);
608 ArrayView<const GO> myGlobalEntries = map.getNodeElementList ();
613 directoryMap_->getRemoteIndexList (myGlobalEntries, sendImageIDs);
614 TEUCHOS_TEST_FOR_EXCEPTION(
615 lookupStatus ==
IDNotPresent, std::logic_error, Teuchos::typeName(*
this)
616 <<
" constructor: the Directory Map could not find out where one or "
617 "more of my Map's indices should go. The input to getRemoteIndexList "
618 "is " << Teuchos::toString (myGlobalEntries) <<
", and the output is "
619 << Teuchos::toString (sendImageIDs ()) <<
". The input Map itself has "
620 "the following entries on the calling process " <<
621 map.getComm ()->getRank () <<
": " <<
622 Teuchos::toString (map.getNodeElementList ()) <<
", and has "
623 << map.getGlobalNumElements () <<
" total global indices in ["
624 << map.getMinAllGlobalIndex () <<
"," << map.getMaxAllGlobalIndex ()
625 <<
"]. The Directory Map has "
626 << directoryMap_->getGlobalNumElements () <<
" total global indices in "
627 "[" << directoryMap_->getMinAllGlobalIndex () <<
"," <<
628 directoryMap_->getMaxAllGlobalIndex () <<
"], and the calling process "
629 "has GIDs [" << directoryMap_->getMinGlobalIndex () <<
"," <<
630 directoryMap_->getMaxGlobalIndex () <<
"]. "
631 "This probably means there is a bug in Map or Directory. "
632 "Please report this bug to the Tpetra developers.");
639 Distributor distor (comm);
640 const size_t numReceives = distor.createFromSends (sendImageIDs);
654 const int packetSize = 3;
655 Array<GO> exportEntries (packetSize * numMyEntries);
657 size_type exportIndex = 0;
658 for (size_type i = 0; i < static_cast<size_type> (numMyEntries); ++i) {
659 exportEntries[exportIndex++] = myGlobalEntries[i];
660 exportEntries[exportIndex++] = as<GO> (myRank);
661 exportEntries[exportIndex++] = as<GO> (i);
667 Array<GO> importElements (packetSize * distor.getTotalReceiveLength ());
670 distor.doPostsAndWaits (exportEntries ().getConst (), packetSize, importElements ());
677 if (useHashTables_) {
692 LO* tableKeysRaw = NULL;
693 LO* tableLidsRaw = NULL;
694 int* tablePidsRaw = NULL;
696 tableKeysRaw =
new LO [numReceives];
697 tableLidsRaw =
new LO [numReceives];
698 tablePidsRaw =
new int [numReceives];
700 if (tableKeysRaw != NULL) {
701 delete [] tableKeysRaw;
703 if (tableLidsRaw != NULL) {
704 delete [] tableLidsRaw;
706 if (tablePidsRaw != NULL) {
707 delete [] tablePidsRaw;
711 ArrayRCP<LO> tableKeys (tableKeysRaw, 0, numReceives,
true);
712 ArrayRCP<LO> tableLids (tableLidsRaw, 0, numReceives,
true);
713 ArrayRCP<int> tablePids (tablePidsRaw, 0, numReceives,
true);
715 if (tie_break.is_null ()) {
717 size_type importIndex = 0;
718 for (size_type i = 0; i < static_cast<size_type> (numReceives); ++i) {
719 const GO curGID = importElements[importIndex++];
720 const LO curLID = directoryMap_->getLocalElement (curGID);
721 TEUCHOS_TEST_FOR_EXCEPTION(
722 curLID == LINVALID, std::logic_error,
723 Teuchos::typeName(*
this) <<
" constructor: Incoming global index "
724 << curGID <<
" does not have a corresponding local index in the "
725 "Directory Map. Please report this bug to the Tpetra developers.");
726 tableKeys[i] = curLID;
727 tablePids[i] = importElements[importIndex++];
728 tableLids[i] = importElements[importIndex++];
733 typedef Kokkos::Device<
typename NT::execution_space,
734 typename NT::memory_space> DT;
736 rcp (
new Details::FixedHashTable<LO, int, DT> (tableKeys (),
738 locallyOneToOne_ = ! (lidToPidTable_->hasDuplicateKeys ());
740 rcp (
new Details::FixedHashTable<LO, LO, DT> (tableKeys (),
750 typedef std::map<LO, std::vector<std::pair<int, LO> > > pair_table_type;
751 pair_table_type ownedPidLidPairs;
755 size_type importIndex = 0;
756 for (size_type i = 0; i < static_cast<size_type> (numReceives); ++i) {
757 const GO curGID = importElements[importIndex++];
758 const LO dirMapLid = directoryMap_->getLocalElement (curGID);
759 TEUCHOS_TEST_FOR_EXCEPTION(
760 dirMapLid == LINVALID, std::logic_error,
761 Teuchos::typeName(*
this) <<
" constructor: Incoming global index "
762 << curGID <<
" does not have a corresponding local index in the "
763 "Directory Map. Please report this bug to the Tpetra developers.");
764 tableKeys[i] = dirMapLid;
765 const int PID = importElements[importIndex++];
766 const int LID = importElements[importIndex++];
776 ownedPidLidPairs[dirMapLid].push_back (std::make_pair (PID, LID));
789 for (size_type i = 0; i < static_cast<size_type> (numReceives); ++i) {
790 const LO dirMapLid = tableKeys[i];
791 const std::vector<std::pair<int, LO> >& pidLidList =
792 ownedPidLidPairs[dirMapLid];
793 const size_t listLen = pidLidList.size();
794 if (listLen == 0)
continue;
795 const GO dirMapGid = directoryMap_->getGlobalElement (dirMapLid);
797 locallyOneToOne_ =
false;
806 const size_type index =
807 static_cast<size_type
> (tie_break->selectedIndex (dirMapGid,
809 tablePids[i] = pidLidList[index].first;
810 tableLids[i] = pidLidList[index].second;
814 typedef Kokkos::Device<
typename NT::execution_space,
815 typename NT::memory_space> DT;
817 rcp (
new Details::FixedHashTable<LO, int, DT> (tableKeys (),
820 rcp (
new Details::FixedHashTable<LO, LO, DT> (tableKeys (),
825 if (tie_break.is_null ()) {
830 PIDs_ = arcp<int> (dir_numMyEntries);
831 std::fill (PIDs_.begin (), PIDs_.end (), -1);
832 LIDs_ = arcp<LO> (dir_numMyEntries);
833 std::fill (LIDs_.begin (), LIDs_.end (), LINVALID);
835 size_type importIndex = 0;
836 for (size_type i = 0; i < static_cast<size_type> (numReceives); ++i) {
837 const GO curGID = importElements[importIndex++];
838 const LO curLID = directoryMap_->getLocalElement (curGID);
839 TEUCHOS_TEST_FOR_EXCEPTION(curLID == LINVALID, std::logic_error,
840 Teuchos::typeName(*
this) <<
" constructor: Incoming global index "
841 << curGID <<
" does not have a corresponding local index in the "
842 "Directory Map. Please report this bug to the Tpetra developers.");
847 if (PIDs_[curLID] != -1) {
848 locallyOneToOne_ =
false;
850 PIDs_[curLID] = importElements[importIndex++];
851 LIDs_[curLID] = importElements[importIndex++];
855 PIDs_ = arcp<int> (dir_numMyEntries);
856 LIDs_ = arcp<LO> (dir_numMyEntries);
857 std::fill (PIDs_.begin (), PIDs_.end (), -1);
866 Array<std::vector<std::pair<int, LO> > > ownedPidLidPairs (dir_numMyEntries);
867 size_type importIndex = 0;
868 for (size_type i = 0; i < static_cast<size_type> (numReceives); ++i) {
869 const GO GID = importElements[importIndex++];
870 const int PID = importElements[importIndex++];
871 const LO LID = importElements[importIndex++];
873 const LO dirMapLid = directoryMap_->getLocalElement (GID);
874 TEUCHOS_TEST_FOR_EXCEPTION(
875 dirMapLid == LINVALID, std::logic_error,
876 Teuchos::typeName(*
this) <<
" constructor: Incoming global index "
877 << GID <<
" does not have a corresponding local index in the "
878 "Directory Map. Please report this bug to the Tpetra developers.");
879 ownedPidLidPairs[dirMapLid].push_back (std::make_pair (PID, LID));
891 for (
size_t i = 0; i < dir_numMyEntries; ++i) {
892 const std::vector<std::pair<int, LO> >& pidLidList =
894 const size_t listLen = pidLidList.size();
895 if (listLen == 0)
continue;
897 const LO dirMapLid =
static_cast<LO
> (i);
898 const GO dirMapGid = directoryMap_->getGlobalElement (dirMapLid);
900 locallyOneToOne_ =
false;
909 const size_type index =
910 static_cast<size_type
> (tie_break->selectedIndex (dirMapGid,
912 PIDs_[i] = pidLidList[index].first;
913 LIDs_[i] = pidLidList[index].second;
919 template<
class LO,
class GO,
class NT>
923 std::ostringstream os;
924 os <<
"DistributedNoncontiguousDirectory"
925 <<
"<" << Teuchos::TypeNameTraits<LO>::name ()
926 <<
", " << Teuchos::TypeNameTraits<GO>::name ()
927 <<
", " << Teuchos::TypeNameTraits<NT>::name () <<
">";
931 template<
class LO,
class GO,
class NT>
935 const Teuchos::ArrayView<const GO> &globalIDs,
936 const Teuchos::ArrayView<int> &nodeIDs,
937 const Teuchos::ArrayView<LO> &localIDs,
938 const bool computeLIDs)
const
940 using Teuchos::Array;
941 using Teuchos::ArrayRCP;
942 using Teuchos::ArrayView;
945 using Teuchos::toString;
946 using ::Tpetra::Details::Behavior;
949 using size_type =
typename Array<GO>::size_type;
950 const char funcPrefix[] =
"Tpetra::"
951 "DistributedNoncontiguousDirectory::getEntriesImpl: ";
952 const char errSuffix[] =
953 " Please report this bug to the Tpetra developers.";
955 RCP<const Teuchos::Comm<int> > comm = map.getComm ();
958 std::unique_ptr<std::string> procPrefix;
960 std::ostringstream os;
962 if (map.getComm ().is_null ()) {
966 os << map.getComm ()->getRank ();
969 procPrefix = std::unique_ptr<std::string> (
new std::string (os.str ()));
970 os << funcPrefix <<
"{GIDs: " << toString (globalIDs)
971 <<
", PIDs: " << toString (nodeIDs)
972 <<
", LIDs: " << toString (localIDs)
973 <<
", computeLIDs: " << (computeLIDs ?
"true" :
"false")
978 const size_t numEntries = globalIDs.size ();
979 const LO LINVALID = Teuchos::OrdinalTraits<LO>::invalid();
988 const int packetSize = computeLIDs ? 3 : 2;
994 Array<int> dirImages (numEntries);
996 std::ostringstream os;
997 os << *procPrefix <<
"Call directoryMap_->getRemoteIndexList"
1001 res = directoryMap_->getRemoteIndexList (globalIDs, dirImages ());
1003 std::ostringstream os;
1004 os << *procPrefix <<
"directoryMap_->getRemoteIndexList "
1005 "PIDs result: " << toString (dirImages) << endl;
1010 size_t numMissing = 0;
1012 for (
size_t i=0; i < numEntries; ++i) {
1013 if (dirImages[i] == -1) {
1016 localIDs[i] = LINVALID;
1024 Array<int> sendImages;
1026 std::ostringstream os;
1027 os << *procPrefix <<
"Call Distributor::createFromRecvs"
1031 distor.
createFromRecvs (globalIDs, dirImages (), sendGIDs, sendImages);
1033 std::ostringstream os;
1034 os << *procPrefix <<
"Distributor::createFromRecvs result: "
1035 <<
"{sendGIDs: " << toString (sendGIDs)
1036 <<
", sendPIDs: " << toString (sendImages)
1040 const size_type numSends = sendGIDs.size ();
1073 Array<global_size_t> exports (packetSize * numSends);
1086 size_type exportsIndex = 0;
1088 if (useHashTables_) {
1090 std::ostringstream os;
1091 os << *procPrefix <<
"Pack exports (useHashTables_ true)"
1095 for (size_type gidIndex = 0; gidIndex < numSends; ++gidIndex) {
1096 const GO curGID = sendGIDs[gidIndex];
1098 exports[exportsIndex++] =
static_cast<global_size_t> (curGID);
1099 const LO curLID = directoryMap_->getLocalElement (curGID);
1100 TEUCHOS_TEST_FOR_EXCEPTION
1101 (curLID == LINVALID, std::logic_error, funcPrefix <<
1102 "Directory Map's global index " << curGID <<
" lacks "
1103 "a corresponding local index." << errSuffix);
1105 exports[exportsIndex++] =
1109 exports[exportsIndex++] =
1116 std::ostringstream os;
1117 os << *procPrefix <<
"Pack exports (useHashTables_ false)"
1121 for (size_type gidIndex = 0; gidIndex < numSends; ++gidIndex) {
1122 const GO curGID = sendGIDs[gidIndex];
1124 exports[exportsIndex++] =
static_cast<global_size_t> (curGID);
1125 const LO curLID = directoryMap_->getLocalElement (curGID);
1126 TEUCHOS_TEST_FOR_EXCEPTION
1127 (curLID == LINVALID, std::logic_error, funcPrefix <<
1128 "Directory Map's global index " << curGID <<
" lacks "
1129 "a corresponding local index." << errSuffix);
1131 exports[exportsIndex++] =
1135 exports[exportsIndex++] =
1141 TEUCHOS_TEST_FOR_EXCEPTION
1142 (exportsIndex > exports.size (), std::logic_error,
1143 funcPrefix <<
"On Process " << comm->getRank () <<
", "
1144 "exportsIndex = " << exportsIndex <<
" > exports.size() = "
1145 << exports.size () <<
"." << errSuffix);
1148 TEUCHOS_TEST_FOR_EXCEPTION
1149 (numEntries < numMissing, std::logic_error, funcPrefix <<
1150 "On Process " << comm->getRank () <<
", numEntries = " <<
1151 numEntries <<
" < numMissing = " << numMissing <<
"." <<
1158 const size_t numRecv = numEntries - numMissing;
1162 const size_t requiredImportLen = numRecv * packetSize;
1163 const int myRank = comm->getRank ();
1164 TEUCHOS_TEST_FOR_EXCEPTION(
1165 importLen < requiredImportLen, std::logic_error,
1166 "Tpetra::Details::DistributedNoncontiguousDirectory::getEntriesImpl: "
1167 "On Process " << myRank <<
": The 'imports' array must have length "
1168 "at least " << requiredImportLen <<
", but its actual length is " <<
1169 importLen <<
". numRecv: " << numRecv <<
", packetSize: " <<
1170 packetSize <<
", numEntries (# GIDs): " << numEntries <<
1171 ", numMissing: " << numMissing <<
": distor.getTotalReceiveLength(): "
1173 "Distributor description: " << distor.
description () <<
". "
1175 "Please report this bug to the Tpetra developers.");
1183 std::ostringstream os;
1184 os << *procPrefix <<
"Call doPostsAndWaits: {packetSize: "
1185 << packetSize <<
", exports: " << toString (exports) <<
"}"
1189 distor.
doPostsAndWaits (exports ().getConst (), packetSize, imports ());
1191 std::ostringstream os;
1192 os << *procPrefix <<
"doPostsAndWaits result: "
1193 << toString (imports) << endl;
1197 Array<GO> sortedIDs (globalIDs);
1198 Array<GO> offset (numEntries);
1199 for (GO ii = 0; ii < static_cast<GO> (numEntries); ++ii) {
1202 sort2 (sortedIDs.begin(), sortedIDs.begin() + numEntries, offset.begin());
1204 std::ostringstream os;
1205 os << *procPrefix <<
"sortedIDs: " << toString (sortedIDs)
1206 <<
", offset: " << toString (offset) << endl;
1210 size_t importsIndex = 0;
1215 for (
size_t i = 0; i < numRecv; ++i) {
1216 const GO curGID =
static_cast<GO
> (imports[importsIndex++]);
1217 auto p1 = std::equal_range (sortedIDs.begin(),
1218 sortedIDs.end(), curGID);
1219 if (p1.first != p1.second) {
1220 const size_t j = p1.first - sortedIDs.begin();
1221 nodeIDs[offset[j]] =
1222 static_cast<int> (imports[importsIndex++]);
1224 localIDs[offset[j]] =
1225 static_cast<LO
> (imports[importsIndex++]);
1227 if (nodeIDs[offset[j]] == -1) {
1233 TEUCHOS_TEST_FOR_EXCEPTION
1234 (
size_t (importsIndex) >
size_t (imports.size ()),
1235 std::logic_error, funcPrefix <<
"On Process " <<
1236 comm->getRank () <<
": importsIndex = " << importsIndex
1237 <<
" > imports.size() = " << imports.size () <<
". "
1238 "numRecv: " << numRecv
1239 <<
", packetSize: " << packetSize <<
", "
1240 "numEntries (# GIDs): " << numEntries
1241 <<
", numMissing: " << numMissing
1242 <<
": distor.getTotalReceiveLength(): "
1245 std::ostringstream os;
1246 os << *procPrefix << funcPrefix <<
"Done!" << endl;
1253 template<
class LO,
class GO,
class NT>
1258 if (oneToOneResult_ == ONE_TO_ONE_NOT_CALLED_YET) {
1259 const int lcl121 = isLocallyOneToOne () ? 1 : 0;
1261 Teuchos::reduceAll<int, int> (comm, Teuchos::REDUCE_MIN, lcl121,
1262 Teuchos::outArg (gbl121));
1263 oneToOneResult_ = (gbl121 == 1) ? ONE_TO_ONE_TRUE : ONE_TO_ONE_FALSE;
1265 return (oneToOneResult_ == ONE_TO_ONE_TRUE);
1275 #define TPETRA_DIRECTORYIMPL_INSTANT(LO,GO,NODE) \
1276 namespace Details { \
1277 template class Directory< LO , GO , NODE >; \
1278 template class ReplicatedDirectory< LO , GO , NODE >; \
1279 template class ContiguousUniformDirectory< LO, GO, NODE >; \
1280 template class DistributedContiguousDirectory< LO , GO , NODE >; \
1281 template class DistributedNoncontiguousDirectory< LO , GO , NODE >; \
1284 #endif // __Tpetra_DirectoryImpl_def_hpp
void doPostsAndWaits(const Teuchos::ArrayView< const Packet > &exports, size_t numPackets, const Teuchos::ArrayView< Packet > &imports)
Execute the (forward) communication plan.
void initialize(int *argc, char ***argv)
Initialize Tpetra.
Interface for breaking ties in ownership.
std::string description() const
A one-line human-readable description of this object.
std::string description() const
Return a one-line description of this object.
bool isNodeGlobalElement(GlobalOrdinal globalIndex) const
Whether the given global index is owned by this Map on the calling process.
LookupStatus
Return status of Map remote index lookup (getRemoteIndexList()).
Implementation of Directory for a distributed noncontiguous Map.
Teuchos::RCP< const Teuchos::Comm< int > > getComm() const
Accessors for the Teuchos::Comm and Kokkos Node objects.
bool isOneToOne(const Teuchos::Comm< int > &comm) const override
Whether the Directory's input Map is (globally) one to one.
Implementation of Directory for a distributed contiguous Map.
GlobalOrdinal getMinAllGlobalIndex() const
The minimum global index over all processes in the communicator.
LookupStatus getEntriesImpl(const map_type &map, const Teuchos::ArrayView< const GlobalOrdinal > &globalIDs, const Teuchos::ArrayView< int > &nodeIDs, const Teuchos::ArrayView< LocalOrdinal > &localIDs, const bool computeLIDs) const override
Find process IDs and (optionally) local IDs for the given global IDs.
bool isOneToOne(const Teuchos::Comm< int > &comm) const override
Whether the Directory's input Map is (globally) one to one.
size_t global_size_t
Global size_t object.
LookupStatus getEntriesImpl(const map_type &map, const Teuchos::ArrayView< const GlobalOrdinal > &globalIDs, const Teuchos::ArrayView< int > &nodeIDs, const Teuchos::ArrayView< LocalOrdinal > &localIDs, const bool computeLIDs) const override
Find process IDs and (optionally) local IDs for the given global IDs.
LookupStatus getEntries(const map_type &map, const Teuchos::ArrayView< const GlobalOrdinal > &globalIDs, const Teuchos::ArrayView< int > &nodeIDs, const Teuchos::ArrayView< LocalOrdinal > &localIDs, const bool computeLIDs) const
Sets up and executes a communication plan for a Tpetra DistObject.
static bool verbose()
Whether Tpetra is in verbose mode.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
std::string description() const
A one-line human-readable description of this object.
LocalOrdinal getLocalElement(GlobalOrdinal globalIndex) const
The local index corresponding to the given global index.
void sort2(const IT1 &first1, const IT1 &last1, const IT2 &first2)
Sort the first array, and apply the resulting permutation to the second array.
A parallel distribution of indices over processes.
ReplicatedDirectory()=default
Constructor (that takes no arguments).
std::string description() const
A one-line human-readable description of this object.
LookupStatus getEntriesImpl(const map_type &map, const Teuchos::ArrayView< const GlobalOrdinal > &globalIDs, const Teuchos::ArrayView< int > &nodeIDs, const Teuchos::ArrayView< LocalOrdinal > &localIDs, const bool computeLIDs) const override
Find process IDs and (optionally) local IDs for the given global IDs.
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
global_size_t getGlobalNumElements() const
The number of elements in this Map.