40 #ifndef TPETRA_IMPORT_DEF_HPP
41 #define TPETRA_IMPORT_DEF_HPP
43 #include "Tpetra_Distributor.hpp"
44 #include "Tpetra_Map.hpp"
45 #include "Tpetra_ImportExportData.hpp"
48 #include "Tpetra_Export.hpp"
50 #include "Tpetra_Details_DualViewUtil.hpp"
51 #include "Tpetra_Details_gathervPrint.hpp"
53 #include "Teuchos_as.hpp"
54 #ifdef HAVE_TPETRA_MMM_TIMINGS
55 #include "Teuchos_TimeMonitor.hpp"
62 std::string toString (
const std::vector<T>& x)
64 std::ostringstream os;
66 const std::size_t N = x.size ();
67 for (std::size_t k = 0; k < N; ++k) {
69 if (k + std::size_t (1) < N) {
77 template<
class ElementType,
class DeviceType>
78 std::string toString (
const Kokkos::View<const ElementType*, DeviceType>& x)
80 std::ostringstream os;
82 const std::size_t N = std::size_t (x.extent (0));
83 for (std::size_t k = 0; k < N; ++k) {
85 if (k + std::size_t (1) < N) {
98 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
100 Import<LocalOrdinal,GlobalOrdinal,Node>::
101 init (
const Teuchos::RCP<const map_type>& source,
102 const Teuchos::RCP<const map_type>& ,
104 Teuchos::Array<int> & remotePIDs,
105 const Teuchos::RCP<Teuchos::ParameterList>& plist)
107 using ::Tpetra::Details::ProfilingRegion;
108 using Teuchos::Array;
113 ProfilingRegion regionImportInit (
"Tpetra::Import::init");
115 std::unique_ptr<std::string> verbPrefix;
116 if (this->verbose ()) {
117 std::ostringstream os;
118 const int myRank = source->getComm ()->getRank ();
119 os <<
"Proc " << myRank <<
": Tpetra::Import::init: ";
120 verbPrefix = std::unique_ptr<std::string> (
new std::string (os.str ()));
122 this->verboseOutputStream () << os.str ();
125 Array<GlobalOrdinal> remoteGIDs;
127 #ifdef HAVE_TPETRA_MMM_TIMINGS
128 using Teuchos::TimeMonitor;
131 label = plist->get(
"Timer Label",label);
132 std::string prefix = std::string(
"Tpetra ")+ label + std::string(
":iport_ctor:preIData: ");
137 #ifdef HAVE_TPETRA_MMM_TIMINGS
138 auto MM(*TimeMonitor::getNewTimer(prefix));
140 if (this->verbose ()) {
141 std::ostringstream os;
142 os << *verbPrefix <<
"Call setupSamePermuteRemote" << endl;
143 this->verboseOutputStream () << os.str ();
145 setupSamePermuteRemote (remoteGIDs);
148 #ifdef HAVE_TPETRA_MMM_TIMINGS
149 prefix = std::string(
"Tpetra ")+ label + std::string(
":iport_ctor:preSetupExport: ");
150 auto MM2(*TimeMonitor::getNewTimer(prefix));
152 if (source->isDistributed ()) {
153 if (this->verbose ()) {
154 std::ostringstream os;
155 os << *verbPrefix <<
"Call setupExport" << endl;
156 this->verboseOutputStream () << os.str ();
158 setupExport (remoteGIDs,useRemotePIDs,remotePIDs);
160 else if (this->verbose ()) {
161 std::ostringstream os;
162 os << *verbPrefix <<
"Source Map not distributed; skip setupExport"
164 this->verboseOutputStream () << os.str ();
168 TEUCHOS_ASSERT( ! this->TransferData_->permuteFromLIDs_.need_sync_device () );
169 TEUCHOS_ASSERT( ! this->TransferData_->permuteFromLIDs_.need_sync_host () );
170 TEUCHOS_ASSERT( ! this->TransferData_->permuteToLIDs_.need_sync_device () );
171 TEUCHOS_ASSERT( ! this->TransferData_->permuteToLIDs_.need_sync_host () );
172 TEUCHOS_ASSERT( ! this->TransferData_->remoteLIDs_.need_sync_device () );
173 TEUCHOS_ASSERT( ! this->TransferData_->remoteLIDs_.need_sync_host () );
174 TEUCHOS_ASSERT( ! this->TransferData_->exportLIDs_.need_sync_device () );
175 TEUCHOS_ASSERT( ! this->TransferData_->exportLIDs_.need_sync_host () );
177 if (this->verbose ()) {
178 std::ostringstream os;
179 os << *verbPrefix <<
"Done!" << endl;
180 this->verboseOutputStream () << os.str ();
184 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
186 Import (
const Teuchos::RCP<const map_type >& source,
187 const Teuchos::RCP<const map_type >& target) :
188 base_type (source, target, Teuchos::null, Teuchos::null,
"Import")
190 Teuchos::Array<int> dummy;
191 #ifdef HAVE_TPETRA_MMM_TIMINGS
192 Teuchos::RCP<Teuchos::ParameterList> mypars = rcp(
new Teuchos::ParameterList);
193 mypars->set(
"Timer Label",
"Naive_tAFC");
194 init(source, target,
false, dummy, mypars);
196 init (source, target,
false, dummy, Teuchos::null);
200 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
202 Import (
const Teuchos::RCP<const map_type>& source,
203 const Teuchos::RCP<const map_type>& target,
204 const Teuchos::RCP<Teuchos::FancyOStream>& out) :
205 base_type (source, target, out, Teuchos::null,
"Import")
207 Teuchos::Array<int> dummy;
208 init (source, target,
false, dummy, Teuchos::null);
211 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
213 Import (
const Teuchos::RCP<const map_type>& source,
214 const Teuchos::RCP<const map_type>& target,
215 const Teuchos::RCP<Teuchos::ParameterList>& plist) :
216 base_type (source, target, Teuchos::null, plist,
"Import")
218 Teuchos::Array<int> dummy;
219 init (source, target,
false, dummy, plist);
222 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
224 Import (
const Teuchos::RCP<const map_type>& source,
225 const Teuchos::RCP<const map_type>& target,
226 const Teuchos::RCP<Teuchos::FancyOStream>& out,
227 const Teuchos::RCP<Teuchos::ParameterList>& plist) :
228 base_type (source, target, out, plist,
"Import")
230 Teuchos::Array<int> dummy;
231 init (source, target,
false, dummy, plist);
234 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
236 Import (
const Teuchos::RCP<const map_type>& source,
237 const Teuchos::RCP<const map_type>& target,
238 Teuchos::Array<int>& remotePIDs,
239 const Teuchos::RCP<Teuchos::ParameterList>& plist) :
240 base_type (source, target, Teuchos::null, plist,
"Import")
242 init (source, target,
true, remotePIDs, plist);
245 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
251 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
260 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
264 const Teuchos::ArrayView<int> & userRemotePIDs,
265 const Teuchos::ArrayView<const LocalOrdinal> & userExportLIDs,
266 const Teuchos::ArrayView<const int> & userExportPIDs,
267 const Teuchos::RCP<Teuchos::ParameterList>& plist,
268 const Teuchos::RCP<Teuchos::FancyOStream>& out) :
269 base_type (source, target, out, plist,
"Import")
271 using ::Tpetra::Details::makeDualViewFromArrayView;
273 using Teuchos::Array;
274 using Teuchos::ArrayRCP;
275 using Teuchos::ArrayView;
280 using LO = LocalOrdinal;
281 using GO = GlobalOrdinal;
282 using size_type = Teuchos::Array<int>::size_type;
284 std::unique_ptr<std::string> prefix;
286 auto comm = source.is_null () ? Teuchos::null : source->getComm ();
287 const int myRank = comm.is_null () ? -1 : comm->getRank ();
288 std::ostringstream os;
289 os <<
"Proc " << myRank <<
": Tpetra::Import createExpert ctor: ";
290 prefix = std::unique_ptr<std::string> (
new std::string (os.str ()));
291 os <<
"Start" << endl;
295 ArrayView<const GO> sourceGIDs = source->getNodeElementList ();
296 ArrayView<const GO> targetGIDs = target->getNodeElementList ();
298 Array<GO> tRemoteGIDs;
300 std::ostringstream os;
301 os << *prefix <<
"Call setupSamePermuteRemote" << endl;
304 setupSamePermuteRemote (tRemoteGIDs);
307 std::ostringstream os;
308 os << *prefix <<
"Sort & filter IDs" << endl;
312 auto tRemoteLIDs = this->
TransferData_->remoteLIDs_.view_host ();
314 Teuchos::Array<int> tRemotePIDs (userRemotePIDs);
317 std::ostringstream os;
318 os << *prefix <<
"Target Map has remote LIDs but source Map is not "
319 "distributed. Importing to a submap of the target Map." << endl;
327 "::constructExpert: Target Map has remote LIDs but source Map "
328 "is not distributed. Importing to a submap of the target Map.");
329 TEUCHOS_TEST_FOR_EXCEPTION
330 (tRemotePIDs.size () != tRemoteGIDs.size () ||
331 size_t (tRemoteGIDs.size ()) !=
size_t (tRemoteLIDs.extent (0)),
332 std::runtime_error,
"Import::Import createExpert version: "
333 "Size mismatch on userRemotePIDs, remoteGIDs, and remoteLIDs "
334 "Array's to sort3.");
336 sort3 (tRemotePIDs.begin (),
338 tRemoteGIDs.begin (),
339 tRemoteLIDs.data ());
343 size_type indexIntoRemotePIDs = tRemotePIDs.size ();
344 for (size_type i = 0; i < indexIntoRemotePIDs; ++i) {
345 if (tRemotePIDs[i] == -1) {
354 if (indexIntoRemotePIDs - cnt > 0) {
355 Array<GO> newRemoteGIDs(indexIntoRemotePIDs-cnt);
356 Array<LO> newRemoteLIDs(indexIntoRemotePIDs-cnt);
357 Array<int> newRemotePIDs(indexIntoRemotePIDs-cnt);
359 for (size_type j = 0; j < indexIntoRemotePIDs; ++j)
360 if(tRemotePIDs[j] != -1) {
361 newRemoteGIDs[cnt] = tRemoteGIDs[j];
362 newRemotePIDs[cnt] = tRemotePIDs[j];
363 newRemoteLIDs[cnt] = target->getLocalElement(tRemoteGIDs[j]);
366 indexIntoRemotePIDs = cnt;
367 tRemoteGIDs = newRemoteGIDs;
368 tRemotePIDs = newRemotePIDs;
370 newRemoteLIDs ().getConst (),
374 indexIntoRemotePIDs = 0;
381 this->
TransferData_->exportPIDs_ = Teuchos::Array<int> (userExportPIDs);
383 userExportLIDs,
"exportLIDs");
385 bool locallyComplete =
true;
386 for (size_type i = 0; i < userExportPIDs.size () && locallyComplete; ++i) {
387 if (userExportPIDs[i] == -1) {
388 locallyComplete =
false;
394 std::ostringstream os;
395 os << *prefix <<
"locallyComplete: "
396 << (locallyComplete ?
"true" :
"false")
397 <<
"; call createFromSendsAndRecvs" << endl;
401 #ifdef HAVE_TPETRA_MMM_TIMINGS
402 std::string mmm_prefix =
403 std::string(
"Tpetra ") + std::string(
":iport_ctor:cFSAR ");
405 auto MM3(*Teuchos::TimeMonitor::getNewTimer(mmm_prefix));
411 TEUCHOS_ASSERT( ! this->
TransferData_->permuteFromLIDs_.need_sync_device () );
412 TEUCHOS_ASSERT( ! this->
TransferData_->permuteFromLIDs_.need_sync_host () );
413 TEUCHOS_ASSERT( ! this->
TransferData_->permuteToLIDs_.need_sync_device () );
414 TEUCHOS_ASSERT( ! this->
TransferData_->permuteToLIDs_.need_sync_host () );
415 TEUCHOS_ASSERT( ! this->
TransferData_->remoteLIDs_.need_sync_device () );
416 TEUCHOS_ASSERT( ! this->
TransferData_->remoteLIDs_.need_sync_host () );
417 TEUCHOS_ASSERT( ! this->
TransferData_->exportLIDs_.need_sync_device () );
418 TEUCHOS_ASSERT( ! this->
TransferData_->exportLIDs_.need_sync_host () );
421 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
423 Import (
const Teuchos::RCP<const map_type>& source,
424 const Teuchos::RCP<const map_type>& target,
425 const size_t numSameIDs,
426 Teuchos::Array<LocalOrdinal>& permuteToLIDs,
427 Teuchos::Array<LocalOrdinal>& permuteFromLIDs,
428 Teuchos::Array<LocalOrdinal>& remoteLIDs,
429 Teuchos::Array<LocalOrdinal>& exportLIDs,
430 Teuchos::Array<int>& exportPIDs,
432 const Teuchos::RCP<Teuchos::FancyOStream>& out,
433 const Teuchos::RCP<Teuchos::ParameterList>& plist) :
434 base_type (source, target, out, plist,
"Import")
436 using ::Tpetra::Details::makeDualViewFromArrayView;
439 std::unique_ptr<std::string> prefix;
441 auto comm = source.is_null () ? Teuchos::null : source->getComm ();
442 const int myRank = comm.is_null () ? -1 : comm->getRank ();
443 std::ostringstream os;
444 os <<
"Proc " << myRank <<
": Tpetra::Import export ctor: ";
445 prefix = std::unique_ptr<std::string> (
new std::string (os.str ()));
446 os <<
"Start" << endl;
450 bool locallyComplete =
true;
451 for (Teuchos::Array<int>::size_type i = 0; i < exportPIDs.size (); ++i) {
452 if (exportPIDs[i] == -1) {
453 locallyComplete =
false;
457 std::ostringstream os;
458 os << *prefix <<
"numSameIDs: " << numSameIDs <<
", locallyComplete: "
459 << (locallyComplete ?
"true" :
"false") << endl;
466 makeDualViewFromArrayView (this->
TransferData_->permuteToLIDs_,
467 permuteToLIDs ().getConst (),
469 TEUCHOS_ASSERT(
size_t (this->
TransferData_->permuteToLIDs_.extent (0)) ==
470 size_t (permuteToLIDs.size ()) );
471 makeDualViewFromArrayView (this->
TransferData_->permuteFromLIDs_,
472 permuteFromLIDs ().getConst (),
474 TEUCHOS_ASSERT(
size_t (this->
TransferData_->permuteFromLIDs_.extent (0)) ==
475 size_t (permuteFromLIDs.size ()) );
477 remoteLIDs ().getConst (),
479 TEUCHOS_ASSERT(
size_t (this->
TransferData_->remoteLIDs_.extent (0)) ==
480 size_t (remoteLIDs.size ()) );
482 exportLIDs ().getConst (),
484 TEUCHOS_ASSERT(
size_t (this->
TransferData_->exportLIDs_.extent (0)) ==
485 size_t (exportLIDs.size ()) );
489 TEUCHOS_ASSERT( ! this->
TransferData_->permuteFromLIDs_.need_sync_device () );
490 TEUCHOS_ASSERT( ! this->
TransferData_->permuteFromLIDs_.need_sync_host () );
491 TEUCHOS_ASSERT( ! this->
TransferData_->permuteToLIDs_.need_sync_device () );
492 TEUCHOS_ASSERT( ! this->
TransferData_->permuteToLIDs_.need_sync_host () );
493 TEUCHOS_ASSERT( ! this->
TransferData_->remoteLIDs_.need_sync_device () );
494 TEUCHOS_ASSERT( ! this->
TransferData_->remoteLIDs_.need_sync_host () );
495 TEUCHOS_ASSERT( ! this->
TransferData_->exportLIDs_.need_sync_device () );
496 TEUCHOS_ASSERT( ! this->
TransferData_->exportLIDs_.need_sync_host () );
501 template <
class LO,
class GO,
class NT>
502 struct ImportLocalSetupResult
504 Teuchos::RCP<const ::Tpetra::Map<LO, GO, NT> > targetMap;
508 std::vector<GO> remoteGIDs;
509 std::vector<LO> remoteLIDs;
510 std::vector<int> remotePIDs;
515 void printArray (std::ostream& out,
const T x[],
const std::size_t N)
518 for (std::size_t k = 0; k < N; ++k) {
527 template<
class LO,
class GO,
class NT>
528 ImportLocalSetupResult<LO, GO, NT>
529 setupSamePermuteRemoteFromUserGlobalIndexList (const ::Tpetra::Map<LO, GO, NT>& sourceMap,
530 const GO targetMapRemoteOrPermuteGlobalIndices[],
531 const int targetMapRemoteOrPermuteProcessRanks[],
532 const LO numTargetMapRemoteOrPermuteGlobalIndices,
533 const bool mayReorderTargetMapIndicesLocally,
534 Teuchos::FancyOStream* out,
535 const std::string* verboseHeader,
540 const int myRank = sourceMap.getComm ()->getRank ();
541 ImportLocalSetupResult<LO, GO, NT> result;
544 std::ostringstream os;
545 os << *verboseHeader <<
"- Import::setupSPR w/ remote GIDs & PIDs: " << endl
546 << *verboseHeader <<
" Input GIDs: ";
547 printArray (os, targetMapRemoteOrPermuteGlobalIndices, numTargetMapRemoteOrPermuteGlobalIndices);
548 os << endl <<
" Input PIDs: ";
549 printArray (os, targetMapRemoteOrPermuteProcessRanks, numTargetMapRemoteOrPermuteGlobalIndices);
559 std::vector<GO> badGIDs;
560 std::vector<int> badPIDs;
561 const Teuchos::Comm<int>& comm = * (sourceMap.getComm ());
562 const int numProcs = comm.getSize ();
564 for (LO k = 0; k < numTargetMapRemoteOrPermuteGlobalIndices; ++k) {
565 const GO tgtGID = targetMapRemoteOrPermuteGlobalIndices[k];
566 if (sourceMap.isNodeGlobalElement (tgtGID)) {
567 badGIDs.push_back (tgtGID);
569 const int tgtPID = targetMapRemoteOrPermuteProcessRanks[k];
570 if (tgtPID < 0 || tgtPID >= numProcs) {
571 badPIDs.push_back (tgtPID);
575 std::array<int, 2> lclStatus {{
576 badGIDs.size () == 0 ? 1 : 0,
577 badPIDs.size () == 0 ? 1 : 0
579 std::array<int, 2> gblStatus {{0, 0}};
580 Teuchos::reduceAll<int, int> (comm, Teuchos::REDUCE_MIN, 2,
581 lclStatus.data (), gblStatus.data ());
582 const bool good = gblStatus[0] == 1 && gblStatus[1] == 1;
585 if (verbose && gblStatus[0] != 1) {
586 std::ostringstream os;
587 os << *verboseHeader <<
"- Some input GIDs are already in the source Map: ";
588 printArray (os, badGIDs.data (), badGIDs.size ());
590 ::Tpetra::Details::gathervPrint (*out, os.str (), comm);
592 if (verbose && gblStatus[0] != 1) {
593 std::ostringstream os;
594 os << *verboseHeader <<
"- Some input PIDs are invalid: ";
595 printArray (os, badPIDs.data (), badPIDs.size ());
597 ::Tpetra::Details::gathervPrint (*out, os.str (), comm);
601 std::ostringstream os;
602 os <<
"Tpetra::Import constructor that takes remote GIDs and PIDs: ";
603 if (gblStatus[0] != 1) {
604 os <<
"Some input GIDs (global indices) are already in the source Map! ";
606 if (gblStatus[1] != 1) {
607 os <<
"Some input PIDs (process ranks) are invalid! ";
609 os <<
"Rerun with the environment variable TPETRA_VERBOSE=Tpetra::Import "
610 "to see what GIDs and/or PIDs are bad.";
611 TEUCHOS_TEST_FOR_EXCEPTION(
true, std::invalid_argument, os.str ());
618 const LO numLclSrcIDs =
static_cast<LO
> (sourceMap.getNodeNumElements ());
619 const LO numLclTgtIDs = numLclSrcIDs + numTargetMapRemoteOrPermuteGlobalIndices;
621 std::ostringstream os;
622 os << *verboseHeader <<
"- Copy source Map GIDs into target Map GID list: "
623 "numLclSrcIDs=" << numLclSrcIDs
624 <<
", numTargetMapRemoteOrPermuteGlobalIndices="
625 << numTargetMapRemoteOrPermuteGlobalIndices << endl;
628 std::vector<GO> tgtGIDs (numLclTgtIDs);
629 if (sourceMap.isContiguous ()) {
630 GO curTgtGID = sourceMap.getMinGlobalIndex ();
631 for (LO k = 0; k < numLclSrcIDs; ++k, ++curTgtGID) {
632 tgtGIDs[k] = curTgtGID;
636 auto srcGIDs = sourceMap.getNodeElementList ();
637 for (LO k = 0; k < numLclSrcIDs; ++k) {
638 tgtGIDs[k] = srcGIDs[k];
641 std::copy (targetMapRemoteOrPermuteGlobalIndices,
642 targetMapRemoteOrPermuteGlobalIndices + numTargetMapRemoteOrPermuteGlobalIndices,
643 tgtGIDs.begin () + numLclSrcIDs);
656 std::ostringstream os;
657 os << *verboseHeader <<
"- Sort by PID? "
658 << (mayReorderTargetMapIndicesLocally ?
"true" :
"false") << endl;
661 std::vector<int> tgtPIDs (targetMapRemoteOrPermuteProcessRanks,
662 targetMapRemoteOrPermuteProcessRanks + numTargetMapRemoteOrPermuteGlobalIndices);
663 result.numPermutes = 0;
664 if (mayReorderTargetMapIndicesLocally) {
665 Tpetra::sort2 (tgtPIDs.begin (), tgtPIDs.end (), tgtGIDs.begin () + numLclSrcIDs);
666 auto range = std::equal_range (tgtPIDs.begin (), tgtPIDs.end (), myRank);
667 if (range.second > range.first) {
668 result.numPermutes =
static_cast<LO
> (range.second - range.first);
672 result.numPermutes =
static_cast<LO
> (std::count (tgtPIDs.begin (), tgtPIDs.end (), myRank));
675 const LO numRemotes = numTargetMapRemoteOrPermuteGlobalIndices - result.numPermutes;
676 result.numSameIDs =
static_cast<LO
> (sourceMap.getNodeNumElements ());
679 std::ostringstream os;
680 os << *verboseHeader <<
"- numSame=" << result.numSameIDs
681 <<
", numPermutes=" << result.numPermutes
682 <<
", numRemotes=" << numRemotes << endl;
686 if (result.numPermutes == 0) {
688 std::ostringstream os;
689 os << *verboseHeader <<
"- No permutes" << endl;
692 result.remoteGIDs = std::vector<GO> (tgtGIDs.begin () + numLclSrcIDs, tgtGIDs.end ());
693 result.remotePIDs.swap (tgtPIDs);
694 result.remoteLIDs.resize (numRemotes);
695 for (LO k = 0; k < numRemotes; ++k) {
696 const LO tgtLid = result.numSameIDs + k;
697 result.remoteLIDs[k] = tgtLid;
700 std::ostringstream os;
701 os << *verboseHeader <<
"- Remote GIDs: "
702 << Teuchos::toString (result.remoteGIDs) << endl;
703 os << *verboseHeader <<
"- Remote PIDs: "
704 << Teuchos::toString (result.remotePIDs) << endl;
705 os << *verboseHeader <<
"- Remote LIDs: "
706 << Teuchos::toString (result.remoteLIDs) << endl;
714 result.remoteGIDs.reserve (numRemotes);
715 result.remoteLIDs.reserve (numRemotes);
716 result.remotePIDs.reserve (numRemotes);
717 for (LO k = 0; k < numTargetMapRemoteOrPermuteGlobalIndices; ++k) {
718 const LO tgtLid = result.numSameIDs + k;
719 const GO tgtGid = tgtGIDs[numLclSrcIDs + k];
720 const int tgtPid = tgtPIDs[k];
722 if (tgtPid != myRank) {
723 result.remoteGIDs.push_back (tgtGid);
724 result.remoteLIDs.push_back (tgtLid);
725 result.remotePIDs.push_back (tgtPid);
729 std::ostringstream os;
730 os << *verboseHeader <<
"- Some permutes" << endl;
735 if (sourceMap.isDistributed ()) {
737 std::ostringstream os;
738 os << *verboseHeader <<
"- Sort remotes by PID, as Import always does"
740 << *verboseHeader <<
"-- remotePIDs before: "
741 << Teuchos::toString (result.remotePIDs) << endl
742 << *verboseHeader <<
"-- remoteGIDs before: "
743 << Teuchos::toString (result.remoteGIDs) << endl
744 << *verboseHeader <<
"-- remoteLIDs before: "
745 << Teuchos::toString (result.remoteLIDs) << endl;
749 sort3 (result.remotePIDs.begin (),
750 result.remotePIDs.end (),
751 result.remoteGIDs.begin (),
752 result.remoteLIDs.begin ());
754 std::ostringstream os;
755 os << *verboseHeader <<
"-- remotePIDs after: "
756 << Teuchos::toString (result.remotePIDs) << endl
757 << *verboseHeader <<
"-- remoteGIDs after: "
758 << Teuchos::toString (result.remoteGIDs) << endl
759 << *verboseHeader <<
"-- remoteLIDs after: "
760 << Teuchos::toString (result.remoteLIDs) << endl;
761 std::cerr << os.str ();
766 std::ostringstream os;
767 os << *verboseHeader <<
"- Make target Map" << endl;
770 using ::Teuchos::rcp;
771 typedef ::Tpetra::Map<LO, GO, NT> map_type;
772 typedef ::Tpetra::global_size_t GST;
773 const GST MAP_COMPUTES_GLOBAL_COUNT = ::Teuchos::OrdinalTraits<GST>::invalid ();
774 result.targetMap = rcp (
new map_type (MAP_COMPUTES_GLOBAL_COUNT,
777 sourceMap.getIndexBase (),
778 sourceMap.getComm ()));
780 std::ostringstream os;
781 os << *verboseHeader <<
"- Done with sameSPR..." << endl;
788 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
791 const GlobalOrdinal targetMapRemoteOrPermuteGlobalIndices[],
792 const int targetMapRemoteOrPermuteProcessRanks[],
793 const LocalOrdinal numTargetMapRemoteOrPermuteGlobalIndices,
794 const bool mayReorderTargetMapIndicesLocally,
795 const Teuchos::RCP<Teuchos::ParameterList>& plist,
796 const Teuchos::RCP<Teuchos::FancyOStream>& debugOutput) :
800 base_type (sourceMap, Teuchos::null, debugOutput, plist,
"Import")
802 using ::Tpetra::Details::Behavior;
803 using ::Tpetra::Details::makeDualViewFromOwningHostView;
804 using ::Tpetra::Details::makeDualViewFromVector;
805 using ::Tpetra::Details::printDualView;
806 using ::Tpetra::Details::view_alloc_no_init;
807 using Teuchos::ArrayView;
808 using Teuchos::getFancyOStream;
811 using Teuchos::rcpFromRef;
813 typedef LocalOrdinal LO;
814 typedef GlobalOrdinal GO;
817 const bool debug = Behavior::debug (
"Import") ||
818 Behavior::debug (
"Tpetra::Import");
820 std::unique_ptr<std::string> verbPfx;
821 if (this->verbose ()) {
822 std::ostringstream os;
823 const int myRank = sourceMap->getComm ()->getRank ();
824 os <<
"Proc " << myRank <<
": Tpetra::Import ctor from remotes: ";
825 verbPfx = std::unique_ptr<std::string> (
new std::string (os.str ()));
826 os <<
"mayReorder=" << (mayReorderTargetMapIndicesLocally ?
"true" :
"false")
832 ImportLocalSetupResult<LO, GO, NT> localSetupResult =
833 setupSamePermuteRemoteFromUserGlobalIndexList<LO, GO, NT>
835 targetMapRemoteOrPermuteGlobalIndices,
836 targetMapRemoteOrPermuteProcessRanks,
837 numTargetMapRemoteOrPermuteGlobalIndices,
838 mayReorderTargetMapIndicesLocally,
850 localSetupResult.targetMap,
851 this->TransferData_->out_,
853 this->TransferData_->numSameIDs_ = localSetupResult.numSameIDs;
856 makeDualViewFromVector (this->TransferData_->remoteLIDs_,
857 localSetupResult.remoteLIDs,
866 this->TransferData_->isLocallyComplete_ =
true;
868 Teuchos::Array<GO> exportGIDs;
869 if (sourceMap->isDistributed ()) {
870 if (this->verbose ()) {
871 std::ostringstream os;
872 os << *verbPfx <<
"Make Distributor (createFromRecvs)" << endl;
875 ArrayView<const GO> remoteGIDs (localSetupResult.remoteGIDs.data (),
876 localSetupResult.remoteGIDs.size ());
877 ArrayView<const int> remotePIDs (localSetupResult.remotePIDs.data (),
878 localSetupResult.remotePIDs.size ());
883 Distributor& distributor = this->TransferData_->distributor_;
887 this->TransferData_->exportPIDs_);
896 if (this->verbose ()) {
897 std::ostringstream os;
898 os << *verbPfx <<
"Compute exportLIDs" << endl;
901 using size_type =
typename Teuchos::Array<GO>::size_type;
902 const size_type numExportIDs = exportGIDs.size ();
904 typename decltype (this->TransferData_->exportLIDs_)::t_host
905 exportLIDs (view_alloc_no_init (
"exportLIDs"), numExportIDs);
906 for (size_type k = 0; k < numExportIDs; ++k) {
907 exportLIDs[k] = sourceMap->getLocalElement (exportGIDs[k]);
909 makeDualViewFromOwningHostView (this->TransferData_->exportLIDs_, exportLIDs);
912 if (this->verbose ()) {
913 std::ostringstream os;
915 printDualView (os, this->TransferData_->remoteLIDs_,
916 "ImportExportData::remoteLIDs_");
920 if (this->verbose ()) {
921 std::ostringstream os;
922 os << *verbPfx <<
"Done!" << endl;
927 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
931 const Teuchos::EVerbosityLevel verbLevel)
const
934 this->describeImpl (out,
"Tpetra::Import", verbLevel);
937 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
941 auto out = Teuchos::getFancyOStream (Teuchos::rcpFromRef (os));
943 this->describe (*out, Teuchos::VERB_EXTREME);
946 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
951 using ::Tpetra::Details::makeDualViewFromOwningHostView;
952 using ::Tpetra::Details::ProfilingRegion;
953 using ::Tpetra::Details::view_alloc_no_init;
955 using Teuchos::Array;
956 using Teuchos::ArrayRCP;
957 using Teuchos::ArrayView;
960 typedef LocalOrdinal LO;
961 typedef GlobalOrdinal GO;
962 typedef typename ArrayView<const GO>::size_type size_type;
963 ProfilingRegion regionExport (
"Tpetra::Import::setupSamePermuteRemote");
965 const map_type& source = * (this->getSourceMap ());
966 const map_type& target = * (this->getTargetMap ());
967 ArrayView<const GO> sourceGIDs = source.getNodeElementList ();
968 ArrayView<const GO> targetGIDs = target.getNodeElementList ();
970 #ifdef HAVE_TPETRA_DEBUG
971 ArrayView<const GO> rawSrcGids = sourceGIDs;
972 ArrayView<const GO> rawTgtGids = targetGIDs;
974 const GO*
const rawSrcGids = sourceGIDs.getRawPtr ();
975 const GO*
const rawTgtGids = targetGIDs.getRawPtr ();
976 #endif // HAVE_TPETRA_DEBUG
977 const size_type numSrcGids = sourceGIDs.size ();
978 const size_type numTgtGids = targetGIDs.size ();
979 const size_type numGids = std::min (numSrcGids, numTgtGids);
987 size_type numSameGids = 0;
988 for ( ; numSameGids < numGids && rawSrcGids[numSameGids] == rawTgtGids[numSameGids]; ++numSameGids)
990 this->TransferData_->numSameIDs_ = numSameGids;
1004 const LO LINVALID = Teuchos::OrdinalTraits<LO>::invalid ();
1005 const LO numTgtLids = as<LO> (numTgtGids);
1007 for (LO tgtLid = numSameGids; tgtLid < numTgtLids; ++tgtLid) {
1008 const GO curTargetGid = rawTgtGids[tgtLid];
1012 const LO srcLid = source.getLocalElement (curTargetGid);
1013 if (srcLid != LINVALID) {
1017 const LO numRemotes = (numTgtLids - numSameGids) - numPermutes;
1019 using host_perm_type =
1020 typename decltype (this->TransferData_->permuteToLIDs_)::t_host;
1021 host_perm_type permuteToLIDs
1022 (view_alloc_no_init (
"permuteToLIDs"), numPermutes);
1023 host_perm_type permuteFromLIDs
1024 (view_alloc_no_init (
"permuteFromLIDs"), numPermutes);
1025 typename decltype (this->TransferData_->remoteLIDs_)::t_host remoteLIDs
1029 LO numPermutes2 = 0;
1031 for (LO tgtLid = numSameGids; tgtLid < numTgtLids; ++tgtLid) {
1032 const GO curTargetGid = rawTgtGids[tgtLid];
1033 const LO srcLid = source.getLocalElement (curTargetGid);
1034 if (srcLid != LINVALID) {
1035 permuteToLIDs[numPermutes2] = tgtLid;
1036 permuteFromLIDs[numPermutes2] = srcLid;
1040 remoteGIDs.push_back (curTargetGid);
1041 remoteLIDs[numRemotes2] = tgtLid;
1045 TEUCHOS_ASSERT( numPermutes == numPermutes2 );
1046 TEUCHOS_ASSERT( numRemotes == numRemotes2 );
1047 TEUCHOS_ASSERT(
size_t (numPermutes) + remoteGIDs.size () == size_t (numTgtLids - numSameGids) );
1053 if (remoteLIDs.extent (0) != 0 && ! source.isDistributed ()) {
1059 this->TransferData_->isLocallyComplete_ =
false;
1063 (
true, std::runtime_error,
"::setupSamePermuteRemote(): Target has "
1064 "remote LIDs but Source is not distributed globally. Importing to a "
1065 "submap of the target map.");
1070 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1071 void Import<LocalOrdinal,GlobalOrdinal,Node>::
1072 setupExport (Teuchos::Array<GlobalOrdinal>& remoteGIDs,
1074 Teuchos::Array<int>& userRemotePIDs,
1075 const Teuchos::RCP<Teuchos::ParameterList>& plist)
1077 using ::Tpetra::Details::makeDualViewFromOwningHostView;
1078 using ::Tpetra::Details::view_alloc_no_init;
1079 using Teuchos::Array;
1080 using Teuchos::ArrayView;
1082 using GO = GlobalOrdinal;
1083 typedef typename Array<int>::difference_type size_type;
1084 const char tfecfFuncName[] =
"setupExport: ";
1085 const char suffix[] =
" Please report this bug to the Tpetra developers.";
1087 std::unique_ptr<std::string> prefix;
1088 if (this->verbose ()) {
1089 auto srcMap = this->getSourceMap ();
1090 auto comm = srcMap.is_null () ? Teuchos::null : srcMap->getComm ();
1091 const int myRank = comm.is_null () ? -1 : comm->getRank ();
1092 std::ostringstream os;
1093 os <<
"Proc " << myRank <<
": Tpetra::Import::setupExport: ";
1094 prefix = std::unique_ptr<std::string> (
new std::string (os.str ()));
1095 os <<
"Start" << std::endl;
1096 this->verboseOutputStream () << os.str ();
1099 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1100 (this->getSourceMap ().is_null (), std::logic_error,
1101 "Source Map is null. " << suffix);
1102 const map_type& source = * (this->getSourceMap ());
1104 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1105 (! useRemotePIDs && (userRemotePIDs.size() > 0), std::invalid_argument,
1106 "remotePIDs are non-empty but their use has not been requested.");
1107 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1108 (userRemotePIDs.size () > 0 && remoteGIDs.size () != userRemotePIDs.size (),
1109 std::invalid_argument,
"remotePIDs must either be of size zero or match "
1110 "the size of remoteGIDs.");
1114 ArrayView<GO> remoteGIDsView = remoteGIDs ();
1115 ArrayView<int> remoteProcIDsView;
1135 Array<int> newRemotePIDs;
1138 if (! useRemotePIDs) {
1139 newRemotePIDs.resize (remoteGIDsView.size ());
1140 if (this->verbose ()) {
1141 std::ostringstream os;
1142 os << *prefix <<
"Call sourceMap.getRemoteIndexList" << endl;
1143 this->verboseOutputStream () << os.str ();
1145 lookup = source.getRemoteIndexList (remoteGIDsView, newRemotePIDs ());
1147 Array<int>& remoteProcIDs = useRemotePIDs ? userRemotePIDs : newRemotePIDs;
1153 this->TransferData_->isLocallyComplete_ =
false;
1158 (
true, std::runtime_error,
"::setupExport(): the source Map wasn't "
1159 "able to figure out which process owns one or more of the GIDs in the "
1160 "list of remote GIDs. This probably means that there is at least one "
1161 "GID owned by some process in the target Map which is not owned by any"
1162 " process in the source Map. (That is, the source and target Maps do "
1163 "not contain the same set of GIDs globally.)");
1169 const size_type numInvalidRemote =
1170 std::count_if (remoteProcIDs.begin (), remoteProcIDs.end (),
1171 std::bind1st (std::equal_to<int> (), -1));
1172 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1173 (numInvalidRemote == 0, std::logic_error,
"Calling getRemoteIndexList "
1174 "on the source Map returned IDNotPresent, but none of the returned "
1175 "\"remote\" process ranks are -1. Please report this bug to the "
1176 "Tpetra developers.");
1178 #ifdef HAVE_TPETRA_MMM_TIMINGS
1179 using Teuchos::TimeMonitor;
1181 if(!plist.is_null())
1182 label = plist->get(
"Timer Label",label);
1183 std::string prefix = std::string(
"Tpetra ")+ label + std::string(
":iport_ctor:setupExport:1 ");
1184 auto MM = rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix)));
1190 const size_type totalNumRemote = this->getNumRemoteIDs ();
1191 if (numInvalidRemote == totalNumRemote) {
1193 remoteProcIDs.clear ();
1194 remoteGIDs.clear ();
1195 this->TransferData_->remoteLIDs_ =
1196 decltype (this->TransferData_->remoteLIDs_) ();
1201 size_type numValidRemote = 0;
1202 #ifdef HAVE_TPETRA_DEBUG
1203 ArrayView<GO> remoteGIDsPtr = remoteGIDsView;
1205 GO*
const remoteGIDsPtr = remoteGIDsView.getRawPtr ();
1206 #endif // HAVE_TPETRA_DEBUG
1209 auto remoteLIDs = this->TransferData_->remoteLIDs_.view_host ();
1211 for (size_type r = 0; r < totalNumRemote; ++r) {
1213 if (remoteProcIDs[r] != -1) {
1214 remoteProcIDs[numValidRemote] = remoteProcIDs[r];
1215 remoteGIDsPtr[numValidRemote] = remoteGIDsPtr[r];
1216 remoteLIDs[numValidRemote] = remoteLIDs[r];
1220 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1221 (numValidRemote != totalNumRemote - numInvalidRemote,
1222 std::logic_error,
"After removing invalid remote GIDs and packing "
1223 "the valid remote GIDs, numValidRemote = " << numValidRemote
1224 <<
" != totalNumRemote - numInvalidRemote = "
1225 << totalNumRemote - numInvalidRemote
1226 <<
". Please report this bug to the Tpetra developers.");
1228 remoteProcIDs.resize (numValidRemote);
1229 remoteGIDs.resize (numValidRemote);
1231 Kokkos::resize (remoteLIDs, numValidRemote);
1232 this->TransferData_->remoteLIDs_ = decltype (this->TransferData_->remoteLIDs_) ();
1236 remoteGIDsView = remoteGIDs ();
1244 this->TransferData_->remoteLIDs_.modify_host ();
1245 auto remoteLIDs = this->TransferData_->remoteLIDs_.view_host ();
1246 sort3 (remoteProcIDs.begin (),
1247 remoteProcIDs.end (),
1248 remoteGIDsView.getRawPtr (),
1249 remoteLIDs.data ());
1250 this->TransferData_->remoteLIDs_.sync_device ();
1258 Array<GO> exportGIDs;
1260 #ifdef HAVE_TPETRA_MMM_TIMINGS
1261 using Teuchos::TimeMonitor;
1263 if(!plist.is_null())
1264 label = plist->get(
"Timer Label",label);
1265 std::string prefix2 = std::string(
"Tpetra ")+ label + std::string(
":iport_ctor:setupExport:3 ");
1266 auto MM = rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix2)));
1269 if (this->verbose ()) {
1270 std::ostringstream os;
1271 os << *prefix <<
"Call createFromRecvs" << endl;
1272 this->verboseOutputStream () << endl;
1274 this->TransferData_->distributor_.createFromRecvs (remoteGIDsView ().getConst (),
1275 remoteProcIDs, exportGIDs,
1276 this->TransferData_->exportPIDs_);
1282 #ifdef HAVE_TPETRA_MMM_TIMINGS
1283 prefix2 = std::string(
"Tpetra ")+ label + std::string(
":iport_ctor:setupExport:4 ");
1285 MM = rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix2)));
1291 const size_type numExportIDs = exportGIDs.size ();
1292 if (numExportIDs > 0) {
1293 typename decltype (this->TransferData_->exportLIDs_)::t_host
1295 ArrayView<const GO> expGIDs = exportGIDs ();
1296 for (size_type k = 0; k < numExportIDs; ++k) {
1297 exportLIDs[k] = source.getLocalElement (expGIDs[k]);
1302 if (this->verbose ()) {
1303 std::ostringstream os;
1304 os << *prefix <<
"Done!" << endl;
1305 this->verboseOutputStream () << os.str ();
1309 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1313 Teuchos::Array<std::pair<int,GlobalOrdinal>>& remotePGIDs,
1314 typename Teuchos::Array<GlobalOrdinal>::size_type& numSameGIDs,
1315 typename Teuchos::Array<GlobalOrdinal>::size_type& numPermuteGIDs,
1316 typename Teuchos::Array<GlobalOrdinal>::size_type& numRemoteGIDs,
1317 const Teuchos::ArrayView<const GlobalOrdinal>& sameGIDs1,
1318 const Teuchos::ArrayView<const GlobalOrdinal>& sameGIDs2,
1319 Teuchos::Array<GlobalOrdinal>& permuteGIDs1,
1320 Teuchos::Array<GlobalOrdinal>& permuteGIDs2,
1321 Teuchos::Array<GlobalOrdinal>& remoteGIDs1,
1322 Teuchos::Array<GlobalOrdinal>& remoteGIDs2,
1323 Teuchos::Array<int>& remotePIDs1,
1324 Teuchos::Array<int>& remotePIDs2)
const
1327 typedef GlobalOrdinal GO;
1328 typedef typename Teuchos::Array<GO>::size_type size_type;
1330 const size_type numSameGIDs1 = sameGIDs1.size();
1331 const size_type numSameGIDs2 = sameGIDs2.size();
1334 std::sort(permuteGIDs1.begin(), permuteGIDs1.end());
1335 std::sort(permuteGIDs2.begin(), permuteGIDs2.end());
1340 unionTgtGIDs.reserve(numSameGIDs1 + numSameGIDs2 +
1341 permuteGIDs1.size() + permuteGIDs2.size() +
1342 remoteGIDs1.size() + remoteGIDs2.size());
1346 typename Teuchos::Array<GO>::iterator permuteGIDs1_end;
1347 typename Teuchos::Array<GO>::iterator permuteGIDs2_end;
1348 if (numSameGIDs2 > numSameGIDs1) {
1350 numSameGIDs = numSameGIDs2;
1351 permuteGIDs2_end = permuteGIDs2.end();
1354 std::copy(sameGIDs2.begin(), sameGIDs2.end(), std::back_inserter(unionTgtGIDs));
1358 permuteGIDs1_end = std::set_difference(permuteGIDs1.begin(), permuteGIDs1.end(),
1359 unionTgtGIDs.begin()+numSameGIDs1, unionTgtGIDs.end(),
1360 permuteGIDs1.begin());
1364 numSameGIDs = numSameGIDs1;
1365 permuteGIDs1_end = permuteGIDs1.end();
1368 std::copy(sameGIDs1.begin(), sameGIDs1.end(), std::back_inserter(unionTgtGIDs));
1372 permuteGIDs2_end = std::set_difference(permuteGIDs2.begin(), permuteGIDs2.end(),
1373 unionTgtGIDs.begin()+numSameGIDs2, unionTgtGIDs.end(),
1374 permuteGIDs2.begin());
1379 std::set_union(permuteGIDs1.begin(), permuteGIDs1_end,
1380 permuteGIDs2.begin(), permuteGIDs2_end,
1381 std::back_inserter(unionTgtGIDs));
1384 Teuchos::Array<std::pair<int,GO>> remotePGIDs1(remoteGIDs1.size());
1385 for (size_type k=0; k<remoteGIDs1.size(); k++)
1386 remotePGIDs1[k] = std::make_pair(remotePIDs1[k], remoteGIDs1[k]);
1387 std::sort(remotePGIDs1.begin(), remotePGIDs1.end());
1389 Teuchos::Array<std::pair<int,GO>> remotePGIDs2(remoteGIDs2.size());
1390 for (size_type k=0; k<remoteGIDs2.size(); k++)
1391 remotePGIDs2[k] = std::make_pair(remotePIDs2[k], remoteGIDs2[k]);
1392 std::sort(remotePGIDs2.begin(), remotePGIDs2.end());
1394 remotePGIDs.reserve(remotePGIDs1.size()+remotePGIDs2.size());
1395 std::merge(remotePGIDs1.begin(), remotePGIDs1.end(),
1396 remotePGIDs2.begin(), remotePGIDs2.end(),
1397 std::back_inserter(remotePGIDs));
1398 auto it = std::unique(remotePGIDs.begin(), remotePGIDs.end());
1399 remotePGIDs.resize(std::distance(remotePGIDs.begin(), it));
1402 const size_type oldSize = unionTgtGIDs.size();
1403 unionTgtGIDs.resize(oldSize+remotePGIDs.size());
1404 for (size_type start=oldSize, k=0; k<remotePGIDs.size(); k++)
1405 unionTgtGIDs[start+k] = remotePGIDs[k].second;
1408 numRemoteGIDs = remotePGIDs.size();
1409 numPermuteGIDs = unionTgtGIDs.size() - numSameGIDs - numRemoteGIDs;
1414 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1415 Teuchos::RCP<const Import<LocalOrdinal, GlobalOrdinal, Node> >
1419 using ::Tpetra::Details::Behavior;
1420 using Teuchos::Array;
1421 using Teuchos::ArrayView;
1423 using Teuchos::Comm;
1426 using Teuchos::outArg;
1427 using Teuchos::REDUCE_MIN;
1428 using Teuchos::reduceAll;
1430 using LO = LocalOrdinal;
1431 using GO = GlobalOrdinal;
1433 using size_type =
typename Array<GO>::size_type;
1435 #ifdef HAVE_TPETRA_MMM_TIMINGS
1436 using Teuchos::TimeMonitor;
1437 std::string label = std::string(
"Tpetra::Import::setUnion");
1438 TimeMonitor MM(*TimeMonitor::getNewTimer(label));
1441 RCP<const map_type> srcMap = this->getSourceMap ();
1442 RCP<const map_type> tgtMap1 = this->getTargetMap ();
1444 RCP<const Comm<int> > comm = srcMap->getComm ();
1446 const bool debug = Behavior::debug (
"Import::setUnion") ||
1447 Behavior::debug (
"Tpetra::Import::setUnion");
1450 TEUCHOS_TEST_FOR_EXCEPTION
1451 (! srcMap->isSameAs (* (rhs.
getSourceMap ())), std::invalid_argument,
1452 "Tpetra::Import::setUnion: The source Map of the input Import must be the "
1453 "same as (in the sense of Map::isSameAs) the source Map of this Import.");
1454 const Comm<int>& comm1 = * (tgtMap1->getComm ());
1455 const Comm<int>& comm2 = * (tgtMap2->getComm ());
1456 TEUCHOS_TEST_FOR_EXCEPTION
1458 std::invalid_argument,
"Tpetra::Import::setUnion: "
1459 "The target Maps must have congruent communicators.");
1465 if (tgtMap1->isSameAs (*tgtMap2)) {
1466 return rcp (
new import_type (*
this));
1474 const size_type numSameGIDs1 = this->getNumSameIDs();
1475 ArrayView<const GO> sameGIDs1 = (tgtMap1->getNodeElementList())(0,numSameGIDs1);
1478 ArrayView<const GO> sameGIDs2 = (tgtMap2->getNodeElementList())(0,numSameGIDs2);
1481 ArrayView<const LO> permuteToLIDs1 = this->getPermuteToLIDs();
1482 Array<GO> permuteGIDs1(permuteToLIDs1.size());
1483 for (size_type k=0; k<permuteGIDs1.size(); k++)
1484 permuteGIDs1[k] = tgtMap1->getGlobalElement(permuteToLIDs1[k]);
1487 Array<GO> permuteGIDs2(permuteToLIDs2.size());
1488 for (size_type k=0; k<permuteGIDs2.size(); k++)
1489 permuteGIDs2[k] = tgtMap2->getGlobalElement(permuteToLIDs2[k]);
1492 ArrayView<const LO> remoteLIDs1 = this->getRemoteLIDs();
1493 Array<GO> remoteGIDs1(remoteLIDs1.size());
1494 for (size_type k=0; k<remoteLIDs1.size(); k++)
1495 remoteGIDs1[k] = this->getTargetMap()->getGlobalElement(remoteLIDs1[k]);
1498 Array<GO> remoteGIDs2(remoteLIDs2.size());
1499 for (size_type k=0; k<remoteLIDs2.size(); k++)
1500 remoteGIDs2[k] = rhs.
getTargetMap()->getGlobalElement(remoteLIDs2[k]);
1503 Array<int> remotePIDs1;
1504 Tpetra::Import_Util::getRemotePIDs(*
this, remotePIDs1);
1506 Array<int> remotePIDs2;
1507 Tpetra::Import_Util::getRemotePIDs(rhs, remotePIDs2);
1510 Array<GO> unionTgtGIDs;
1511 Array<std::pair<int,GO>> remotePGIDs;
1512 size_type numSameIDsUnion, numPermuteIDsUnion, numRemoteIDsUnion;
1514 findUnionTargetGIDs(unionTgtGIDs, remotePGIDs,
1515 numSameIDsUnion, numPermuteIDsUnion, numRemoteIDsUnion,
1516 sameGIDs1, sameGIDs2, permuteGIDs1, permuteGIDs2,
1517 remoteGIDs1, remoteGIDs2, remotePIDs1, remotePIDs2);
1520 Array<LO> remoteLIDsUnion(numRemoteIDsUnion);
1521 Array<GO> remoteGIDsUnion(numRemoteIDsUnion);
1522 Array<int> remotePIDsUnion(numRemoteIDsUnion);
1523 const size_type unionRemoteIDsStart = numSameIDsUnion + numPermuteIDsUnion;
1524 for (size_type k = 0; k < numRemoteIDsUnion; ++k) {
1525 remoteLIDsUnion[k] = unionRemoteIDsStart + k;
1526 remotePIDsUnion[k] = remotePGIDs[k].first;
1527 remoteGIDsUnion[k] = remotePGIDs[k].second;
1532 Array<LO> permuteToLIDsUnion(numPermuteIDsUnion);
1533 Array<LO> permuteFromLIDsUnion(numPermuteIDsUnion);
1534 for (size_type k = 0; k < numPermuteIDsUnion; ++k) {
1535 size_type idx = numSameIDsUnion + k;
1536 permuteToLIDsUnion[k] =
static_cast<LO
>(idx);
1537 permuteFromLIDsUnion[k] = srcMap->getLocalElement(unionTgtGIDs[idx]);
1540 #ifdef HAVE_TPETRA_MMM_TIMINGS
1541 MM.disableTimer(label);
1542 label =
"Tpetra::Import::setUnion : Construct Target Map";
1543 TimeMonitor MM2(*TimeMonitor::getNewTimer(label));
1547 const GST INVALID = Teuchos::OrdinalTraits<GST>::invalid ();
1548 const GO indexBaseUnion = std::min(tgtMap1->getIndexBase(), tgtMap2->getIndexBase());
1549 RCP<const map_type> unionTgtMap =
1550 rcp(
new map_type(INVALID, unionTgtGIDs(), indexBaseUnion, comm));
1552 #ifdef HAVE_TPETRA_MMM_TIMINGS
1553 MM2.disableTimer(label);
1554 label =
"Tpetra::Import::setUnion : Export GIDs";
1555 TimeMonitor MM3(*TimeMonitor::getNewTimer(label));
1566 Array<GO> exportGIDsUnion;
1567 Array<LO> exportLIDsUnion;
1568 Array<int> exportPIDsUnion;
1569 Distributor distributor (comm, this->TransferData_->out_);
1571 #ifdef TPETRA_IMPORT_SETUNION_USE_CREATE_FROM_SENDS
1579 ArrayView<const LO> exportLIDs1 = this->getExportLIDs ();
1580 ArrayView<const LO> exportPIDs1 = this->getExportPIDs ();
1593 Array<LO> exportLIDs1Copy (exportLIDs1.begin (), exportLIDs1.end ());
1594 Array<int> exportPIDs1Copy (exportLIDs1.begin (), exportLIDs1.end ());
1595 sort2 (exportLIDs1Copy.begin (), exportLIDs1Copy.end (),
1596 exportPIDs1Copy.begin ());
1597 typename ArrayView<LO>::iterator exportLIDs1_end = exportLIDs1Copy.end ();
1598 typename ArrayView<LO>::iterator exportPIDs1_end = exportPIDs1Copy.end ();
1599 merge2 (exportLIDs1_end, exportPIDs1_end,
1600 exportLIDs1Copy.begin (), exportLIDs1_end,
1601 exportPIDs1Copy.begin (), exportPIDs1_end,
1604 Array<LO> exportLIDs2Copy (exportLIDs2.begin (), exportLIDs2.end ());
1605 Array<int> exportPIDs2Copy (exportLIDs2.begin (), exportLIDs2.end ());
1606 sort2 (exportLIDs2Copy.begin (), exportLIDs2Copy.end (),
1607 exportPIDs2Copy.begin ());
1608 typename ArrayView<LO>::iterator exportLIDs2_end = exportLIDs2Copy.end ();
1609 typename ArrayView<LO>::iterator exportPIDs2_end = exportPIDs2Copy.end ();
1610 merge2 (exportLIDs2_end, exportPIDs2_end,
1611 exportLIDs2Copy.begin (), exportLIDs2_end,
1612 exportPIDs2Copy.begin (), exportPIDs2_end,
1619 keyValueMerge (exportLIDs1Copy.begin (), exportLIDs1Copy.end (),
1620 exportPIDs1Copy.begin (), exportPIDs1Copy.end (),
1621 exportLIDs2Copy.begin (), exportLIDs2Copy.end (),
1622 exportPIDs2Copy.begin (), exportPIDs2Copy.end (),
1623 std::back_inserter (exportLIDsUnion),
1624 std::back_inserter (exportPIDsUnion),
1628 sort2 (exportPIDsUnion.begin (), exportPIDsUnion.end (),
1629 exportLIDsUnion.begin ());
1635 #else // NOT TPETRA_IMPORT_SETUNION_USE_CREATE_FROM_SENDS
1643 remotePIDsUnion().getConst(),
1644 exportGIDsUnion, exportPIDsUnion);
1647 const size_type numExportIDsUnion = exportGIDsUnion.size ();
1648 exportLIDsUnion.resize (numExportIDsUnion);
1649 for (size_type k = 0; k < numExportIDsUnion; ++k) {
1650 exportLIDsUnion[k] = srcMap->getLocalElement (exportGIDsUnion[k]);
1652 #endif // TPETRA_IMPORT_SETUNION_USE_CREATE_FROM_SENDS
1655 #ifdef HAVE_TPETRA_MMM_TIMINGS
1656 MM3.disableTimer(label);
1657 label =
"Tpetra::Import::setUnion : Construct Import";
1658 TimeMonitor MM4(*TimeMonitor::getNewTimer(label));
1660 RCP<const import_type> unionImport =
1661 rcp (
new import_type (srcMap, unionTgtMap,
1662 as<size_t> (numSameIDsUnion),
1663 permuteToLIDsUnion, permuteFromLIDsUnion,
1664 remoteLIDsUnion, exportLIDsUnion,
1665 exportPIDsUnion, distributor,
1666 this->TransferData_->out_));
1670 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1671 Teuchos::RCP<const Import<LocalOrdinal, GlobalOrdinal, Node> >
1675 using Teuchos::Array;
1676 using Teuchos::ArrayView;
1678 using Teuchos::Comm;
1681 using Teuchos::outArg;
1682 using Teuchos::REDUCE_MIN;
1683 using Teuchos::reduceAll;
1684 typedef LocalOrdinal LO;
1685 typedef GlobalOrdinal GO;
1686 Teuchos::RCP<const Import<LocalOrdinal, GlobalOrdinal, Node> > unionImport;
1687 RCP<const map_type> srcMap = this->getSourceMap ();
1688 RCP<const map_type> tgtMap = this->getTargetMap ();
1689 RCP<const Comm<int> > comm = srcMap->getComm ();
1691 ArrayView<const GO> srcGIDs = srcMap->getNodeElementList ();
1692 ArrayView<const GO> tgtGIDs = tgtMap->getNodeElementList ();
1695 size_t numSameIDsNew = srcMap->getNodeNumElements ();
1696 size_t numRemoteIDsNew = this->getNumRemoteIDs ();
1697 Array<LO> permuteToLIDsNew, permuteFromLIDsNew;
1700 ArrayView<const LO> remoteLIDsOld = this->getRemoteLIDs ();
1701 ArrayView<const LO> exportLIDsOld = this->getExportLIDs ();
1704 Array<GO> GIDs(numSameIDsNew + numRemoteIDsNew);
1705 for(
size_t i=0; i<numSameIDsNew; i++)
1706 GIDs[i] = srcGIDs[i];
1709 Array<LO> remoteLIDsNew(numRemoteIDsNew);
1710 for(
size_t i=0; i<numRemoteIDsNew; i++) {
1711 GIDs[numSameIDsNew + i] = tgtGIDs[remoteLIDsOld[i]];
1712 remoteLIDsNew[i] = numSameIDsNew+i;
1716 GO GO_INVALID = Teuchos::OrdinalTraits<GO>::invalid();
1717 RCP<const map_type> targetMapNew =
1718 rcp (
new map_type (GO_INVALID, GIDs, tgtMap->getIndexBase (),
1719 tgtMap->getComm ()));
1722 Array<int> exportPIDsnew (this->getExportPIDs ());
1723 Array<LO> exportLIDsnew (this->getExportLIDs ());
1741 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1742 Teuchos::RCP<const Import<LocalOrdinal, GlobalOrdinal, Node> >
1746 using ::Tpetra::Details::Behavior;
1747 using ::Tpetra::Details::gathervPrint;
1748 using Teuchos::outArg;
1750 using Teuchos::REDUCE_MIN;
1751 using Teuchos::reduceAll;
1753 using LO = LocalOrdinal;
1754 using GO = GlobalOrdinal;
1757 const char funcPrefix[] =
"Tpetra::createRemoteOnlyImport: ";
1760 const bool debug = Behavior::debug ();
1762 const size_t NumRemotes = this->getNumRemoteIDs ();
1763 std::unique_ptr<std::string> procPrefix;
1764 Teuchos::RCP<const Teuchos::Comm<int>> comm;
1766 comm = remoteTarget.is_null () ? Teuchos::null :
1767 remoteTarget->getComm ();
1768 std::ostringstream os;
1770 if (comm.is_null ()) {
1774 os << comm->getRank ();
1777 procPrefix = std::unique_ptr<std::string> (
new std::string (os.str ()));
1781 std::ostringstream lclErr;
1782 if (remoteTarget.is_null ()) {
1785 else if (NumRemotes != remoteTarget->getNodeNumElements ()) {
1787 lclErr << *procPrefix <<
"getNumRemoteIDs() = " << NumRemotes
1788 <<
" != remoteTarget->getNodeNumElements() = "
1789 << remoteTarget->getNodeNumElements () <<
"." << endl;
1792 if (comm.is_null ()) {
1793 lclSuccess = gblSuccess;
1796 reduceAll (*comm, REDUCE_MIN, lclSuccess, outArg (gblSuccess));
1798 TEUCHOS_TEST_FOR_EXCEPTION
1799 (gblSuccess == -1, std::invalid_argument, funcPrefix
1800 <<
"Input target Map is null on at least one process.");
1802 if (gblSuccess != 1) {
1803 if (comm.is_null ()) {
1804 TEUCHOS_TEST_FOR_EXCEPTION
1805 (
true, std::runtime_error, lclErr.str ());
1808 std::ostringstream gblErr;
1809 gblErr << funcPrefix << endl;
1810 gathervPrint (gblErr, lclErr.str (), *comm);
1811 TEUCHOS_TEST_FOR_EXCEPTION
1812 (
true, std::runtime_error, gblErr.str ());
1818 Teuchos::ArrayView<const LO> oldRemoteLIDs = this->getRemoteLIDs ();
1819 Teuchos::Array<LO> newRemoteLIDs (NumRemotes);
1820 const map_type& tgtMap = * (this->getTargetMap ());
1821 size_t badCount = 0;
1823 std::unique_ptr<std::vector<size_t>> badIndices;
1825 badIndices = std::unique_ptr<std::vector<size_t>> (
new std::vector<size_t>);
1827 for (
size_t i = 0; i < NumRemotes; ++i) {
1828 const LO oldLclInd = oldRemoteLIDs[i];
1829 if (oldLclInd == Teuchos::OrdinalTraits<LO>::invalid ()) {
1831 if (debug) { badIndices->push_back (i); }
1835 if (gblInd == Teuchos::OrdinalTraits<GO>::invalid ()) {
1837 if (debug) { badIndices->push_back (i); }
1840 const LO newLclInd = remoteTarget->getLocalElement (gblInd);
1841 if (newLclInd == Teuchos::OrdinalTraits<LO>::invalid ()) {
1843 if (debug) { badIndices->push_back (i); }
1846 newRemoteLIDs[i] = newLclInd;
1848 if (i > 0 && newRemoteLIDs[i] < newRemoteLIDs[i-1]) {
1850 if (debug) { badIndices->push_back (i); }
1854 if (badCount != 0) {
1859 if (comm.is_null ()) {
1860 lclSuccess = gblSuccess;
1863 reduceAll (*comm, REDUCE_MIN, lclSuccess, outArg (gblSuccess));
1865 std::ostringstream lclErr;
1866 if (lclSuccess != 1) {
1867 lclErr << *procPrefix <<
"Count of bad indices: " << badCount
1868 <<
", bad indices: [";
1871 for (
size_t k = 0; k < badCount; ++k) {
1872 const size_t badIndex = (*badIndices)[k];
1873 lclErr <<
"(" << badIndex <<
","
1874 << oldRemoteLIDs[badIndex] <<
")";
1875 if (k +
size_t (1) < badCount) {
1879 lclErr <<
"]" << endl;
1882 if (gblSuccess != 1) {
1883 std::ostringstream gblErr;
1884 gblErr << funcPrefix <<
"this->getRemoteLIDs() has \"bad\" "
1885 "indices on one or more processes. \"Bad\" means that the "
1886 "indices are invalid, they don't exist in the target Map, "
1887 "they don't exist in remoteTarget, or they are not in "
1888 "sorted order. In what follows, I will show the \"bad\" "
1889 "indices as (k, LID) pairs, where k is the zero-based "
1890 "index of the LID in this->getRemoteLIDs()." << endl;
1891 if (comm.is_null ()) {
1892 gblErr << lclErr.str ();
1895 gathervPrint (gblErr, lclErr.str (), *comm);
1897 TEUCHOS_TEST_FOR_EXCEPTION
1898 (
true, std::runtime_error, gblErr.str ());
1902 TEUCHOS_TEST_FOR_EXCEPTION
1903 (lclSuccess == 0, std::runtime_error, funcPrefix
1904 <<
"this->getRemoteLIDs() has " << badCount
1905 <<
"ind" << (badCount == 1 ?
"ex" :
"ices")
1906 <<
" \"bad\" indices on this process." << endl);
1914 Teuchos::Array<int> newExportPIDs (this->getExportPIDs ());
1915 Teuchos::Array<LO> newExportLIDs (this->getExportLIDs ());
1916 Teuchos::Array<LO> dummy;
1919 return rcp (
new import_type (this->getSourceMap (), remoteTarget,
1920 static_cast<size_t> (0), dummy, dummy,
1921 newRemoteLIDs, newExportLIDs,
1922 newExportPIDs, newDistor));
1927 #define TPETRA_IMPORT_CLASS_INSTANT(LO, GO, NODE) \
1928 template class Import< LO , GO , NODE >;
1937 #define TPETRA_IMPORT_INSTANT(LO, GO, NODE) \
1938 TPETRA_IMPORT_CLASS_INSTANT(LO, GO, NODE)
1940 #endif // TPETRA_IMPORT_DEF_HPP
bool congruent(const Teuchos::Comm< int > &comm1, const Teuchos::Comm< int > &comm2)
Whether the two communicators are congruent.
Communication plan for data redistribution from a uniquely-owned to a (possibly) multiply-owned distr...
Teuchos::FancyOStream & verboseOutputStream() const
Valid (nonnull) output stream for verbose output.
Teuchos::RCP< const Import< LocalOrdinal, GlobalOrdinal, Node > > createRemoteOnlyImport(const Teuchos::RCP< const map_type > &remoteTarget) const
Returns an importer that contains only the remote entries of this.
void sort3(const IT1 &first1, const IT1 &last1, const IT2 &first2, const IT3 &first3)
Sort the first array, and apply the same permutation to the second and third arrays.
Declaration of Tpetra::Details::Profiling, a scope guard for Kokkos Profiling.
LookupStatus
Return status of Map remote index lookup (getRemoteIndexList()).
Teuchos::ArrayView< const int > getExportPIDs() const
List of processes to which entries will be sent.
Teuchos::ArrayView< const LocalOrdinal > getPermuteToLIDs() const
List of local IDs in the target Map that are permuted.
virtual void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
Teuchos::RCP< const map_type > getTargetMap() const
The target Map used to construct this Export or Import.
size_t getNumSameIDs() const
Number of initial identical IDs.
size_t getNumRemoteIDs() const
Number of entries not on the calling process.
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
Teuchos::RCP< const Import< LocalOrdinal, GlobalOrdinal, Node > > setUnion() const
Return the union of this Import this->getSourceMap()
size_t global_size_t
Global size_t object.
Import(const Teuchos::RCP< const map_type > &source, const Teuchos::RCP< const map_type > &target)
Construct an Import from the source and target Maps.
Teuchos::ArrayView< const LocalOrdinal > getRemoteLIDs() const
List of entries in the target Map to receive from other processes.
void keyValueMerge(KeyInputIterType keyBeg1, KeyInputIterType keyEnd1, ValueInputIterType valBeg1, ValueInputIterType valEnd1, KeyInputIterType keyBeg2, KeyInputIterType keyEnd2, ValueInputIterType valBeg2, ValueInputIterType valEnd2, KeyOutputIterType keyOut, ValueOutputIterType valOut, BinaryFunction f)
Merge two sorted (by keys) sequences of unique (key,value) pairs by combining pairs with equal keys...
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
auto view_alloc_no_init(const std::string &label) ->
Use in place of the string label as the first argument of Kokkos::View's constructor, in case you want to allocate without initializing.
Communication plan for data redistribution from a (possibly) multiply-owned to a uniquely-owned distr...
#define TPETRA_ABUSE_WARNING(throw_exception_test, Exception, msg)
Handle an abuse warning, according to HAVE_TPETRA_THROW_ABUSE_WARNINGS and HAVE_TPETRA_PRINT_ABUSE_WA...
Implementation detail of Import and Export.
Sets up and executes a communication plan for a Tpetra DistObject.
GlobalOrdinal getGlobalElement(LocalOrdinal localIndex) const
The global index corresponding to the given local index.
Teuchos::RCP< ImportExportData< LocalOrdinal, GlobalOrdinal, Node > > TransferData_
All the data needed for executing the Export communication plan.
Teuchos::RCP< const map_type > getSourceMap() const
The source Map used to construct this Export or Import.
void findUnionTargetGIDs(Teuchos::Array< GlobalOrdinal > &unionTgtGIDs, Teuchos::Array< std::pair< int, GlobalOrdinal >> &remotePGIDs, typename Teuchos::Array< GlobalOrdinal >::size_type &numSameGIDs, typename Teuchos::Array< GlobalOrdinal >::size_type &numPermuteGIDs, typename Teuchos::Array< GlobalOrdinal >::size_type &numRemoteGIDs, const Teuchos::ArrayView< const GlobalOrdinal > &sameGIDs1, const Teuchos::ArrayView< const GlobalOrdinal > &sameGIDs2, Teuchos::Array< GlobalOrdinal > &permuteGIDs1, Teuchos::Array< GlobalOrdinal > &permuteGIDs2, Teuchos::Array< GlobalOrdinal > &remoteGIDs1, Teuchos::Array< GlobalOrdinal > &remoteGIDs2, Teuchos::Array< int > &remotePIDs1, Teuchos::Array< int > &remotePIDs2) const
Find the union of the target IDs from two Import objects.
Teuchos::ArrayView< const LocalOrdinal > getExportLIDs() const
List of entries in the source Map that will be sent to other processes.
void sort2(const IT1 &first1, const IT1 &last1, const IT2 &first2)
Sort the first array, and apply the resulting permutation to the second array.
A parallel distribution of indices over processes.
Internal functions and macros designed for use with Tpetra::Import and Tpetra::Export objects...
Stand-alone utility functions and macros.
void makeDualViewFromOwningHostView(Kokkos::DualView< ElementType *, DeviceType > &dv, const typename Kokkos::DualView< ElementType *, DeviceType >::t_host &hostView)
Initialize dv such that its host View is hostView.
KOKKOS_INLINE_FUNCTION void merge2(IT1 &indResultOut, IT2 &valResultOut, IT1 indBeg, IT1 indEnd, IT2 valBeg, IT2)
Merge values in place, additively, with the same index.
bool verbose() const
Whether to print verbose debugging output.
virtual void print(std::ostream &os) const
Print the Import's data to the given output stream.
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
Binary function that returns its first argument.
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.