Tpetra parallel linear algebra  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Tpetra_Import_def.hpp
1 // @HEADER
2 // *****************************************************************************
3 // Tpetra: Templated Linear Algebra Services Package
4 //
5 // Copyright 2008 NTESS and the Tpetra contributors.
6 // SPDX-License-Identifier: BSD-3-Clause
7 // *****************************************************************************
8 // @HEADER
9 
10 #ifndef TPETRA_IMPORT_DEF_HPP
11 #define TPETRA_IMPORT_DEF_HPP
12 
13 #include "Tpetra_Distributor.hpp"
14 #include "Tpetra_Map.hpp"
15 #include "Tpetra_ImportExportData.hpp"
16 #include "Tpetra_Util.hpp"
17 #include "Tpetra_Import_Util.hpp"
18 #include "Tpetra_Export.hpp"
19 #include "Tpetra_Details_Behavior.hpp"
20 #include "Tpetra_Details_DualViewUtil.hpp"
21 #include "Tpetra_Details_gathervPrint.hpp"
22 #include "Tpetra_Details_Profiling.hpp"
23 #include "Teuchos_as.hpp"
24 #ifdef HAVE_TPETRA_MMM_TIMINGS
25 #include "Teuchos_TimeMonitor.hpp"
26 #endif
27 #include <array>
28 #include <memory>
29 
30 namespace Teuchos {
31  template<class T>
32  std::string toString (const std::vector<T>& x)
33  {
34  std::ostringstream os;
35  os << "[";
36  const std::size_t N = x.size ();
37  for (std::size_t k = 0; k < N; ++k) {
38  os << x[k];
39  if (k + std::size_t (1) < N) {
40  os << ",";
41  }
42  }
43  os << "]";
44  return os.str ();
45  }
46 
47  template<class ElementType, class DeviceType>
48  std::string toString (const Kokkos::View<const ElementType*, DeviceType>& x)
49  {
50  std::ostringstream os;
51  os << "[";
52  const std::size_t N = std::size_t (x.extent (0));
53  for (std::size_t k = 0; k < N; ++k) {
54  os << x[k];
55  if (k + std::size_t (1) < N) {
56  os << ",";
57  }
58  }
59  os << "]";
60  return os.str ();
61  }
62 } // namespace Teuchos
63 
64 namespace Tpetra {
65 
66  // head: init(source, target, true, remotePIDs, Teuchos::null);
67 
68  template <class LocalOrdinal, class GlobalOrdinal, class Node>
69  void
70  Import<LocalOrdinal,GlobalOrdinal,Node>::
71  init (const Teuchos::RCP<const map_type>& source,
72  const Teuchos::RCP<const map_type>& /* target */,
73  bool useRemotePIDs,
74  Teuchos::Array<int> & remotePIDs,
75  const Teuchos::RCP<Teuchos::ParameterList>& plist)
76  {
77  using ::Tpetra::Details::ProfilingRegion;
78  using Teuchos::Array;
79  using Teuchos::null;
80  using Teuchos::Ptr;
81  using Teuchos::rcp;
82  using std::endl;
83  ProfilingRegion regionImportInit ("Tpetra::Import::init");
84 
85  std::unique_ptr<std::string> verbPrefix;
86  if (this->verbose ()) {
87  std::ostringstream os;
88  const int myRank = source->getComm ()->getRank ();
89  os << "Proc " << myRank << ": Tpetra::Import::init: ";
90  verbPrefix = std::unique_ptr<std::string> (new std::string (os.str ()));
91  os << endl;
92  this->verboseOutputStream () << os.str ();
93  }
94 
95  Array<GlobalOrdinal> remoteGIDs;
96 
97 #ifdef HAVE_TPETRA_MMM_TIMINGS
98  using Teuchos::TimeMonitor;
99  std::string label;
100  if(!plist.is_null())
101  label = plist->get("Timer Label",label);
102  std::string prefix = std::string("Tpetra ")+ label + std::string(":iport_ctor:preIData: ");
103 #else
104  (void)plist;
105 #endif
106  {
107 #ifdef HAVE_TPETRA_MMM_TIMINGS
108  auto MM(*TimeMonitor::getNewTimer(prefix));
109 #endif
110  if (this->verbose ()) {
111  std::ostringstream os;
112  os << *verbPrefix << "Call setupSamePermuteRemote" << endl;
113  this->verboseOutputStream () << os.str ();
114  }
115  setupSamePermuteRemote (remoteGIDs);
116  }
117  {
118 #ifdef HAVE_TPETRA_MMM_TIMINGS
119  prefix = std::string("Tpetra ")+ label + std::string(":iport_ctor:preSetupExport: ");
120  auto MM2(*TimeMonitor::getNewTimer(prefix));
121 #endif
122  if (source->isDistributed ()) {
123  if (this->verbose ()) {
124  std::ostringstream os;
125  os << *verbPrefix << "Call setupExport" << endl;
126  this->verboseOutputStream () << os.str ();
127  }
128  setupExport (remoteGIDs,useRemotePIDs,remotePIDs);
129  }
130  else if (this->verbose ()) {
131  std::ostringstream os;
132  os << *verbPrefix << "Source Map not distributed; skip setupExport"
133  << endl;
134  this->verboseOutputStream () << os.str ();
135  }
136  }
137 
138  TEUCHOS_ASSERT( ! this->TransferData_->permuteFromLIDs_.need_sync_device () );
139  TEUCHOS_ASSERT( ! this->TransferData_->permuteFromLIDs_.need_sync_host () );
140  TEUCHOS_ASSERT( ! this->TransferData_->permuteToLIDs_.need_sync_device () );
141  TEUCHOS_ASSERT( ! this->TransferData_->permuteToLIDs_.need_sync_host () );
142  TEUCHOS_ASSERT( ! this->TransferData_->remoteLIDs_.need_sync_device () );
143  TEUCHOS_ASSERT( ! this->TransferData_->remoteLIDs_.need_sync_host () );
144  TEUCHOS_ASSERT( ! this->TransferData_->exportLIDs_.need_sync_device () );
145  TEUCHOS_ASSERT( ! this->TransferData_->exportLIDs_.need_sync_host () );
146 
147  this->detectRemoteExportLIDsContiguous();
148 
149  if (this->verbose ()) {
150  std::ostringstream os;
151  os << *verbPrefix << "Done!" << endl;
152  this->verboseOutputStream () << os.str ();
153  }
154  }
155 
156  template <class LocalOrdinal, class GlobalOrdinal, class Node>
157  Import<LocalOrdinal,GlobalOrdinal,Node>::
158  Import (const Teuchos::RCP<const map_type >& source,
159  const Teuchos::RCP<const map_type >& target) :
160  base_type (source, target, Teuchos::null, Teuchos::null, "Import")
161  {
162  Teuchos::Array<int> dummy;
163 #ifdef HAVE_TPETRA_MMM_TIMINGS
164  Teuchos::RCP<Teuchos::ParameterList> mypars = rcp(new Teuchos::ParameterList);
165  mypars->set("Timer Label","Naive_tAFC");
166  init(source, target, false, dummy, mypars);
167 #else
168  init (source, target, false, dummy, Teuchos::null);
169 #endif
170  }
171 
172  template <class LocalOrdinal, class GlobalOrdinal, class Node>
173  Import<LocalOrdinal,GlobalOrdinal,Node>::
174  Import (const Teuchos::RCP<const map_type>& source,
175  const Teuchos::RCP<const map_type>& target,
176  const Teuchos::RCP<Teuchos::FancyOStream>& out) :
177  base_type (source, target, out, Teuchos::null, "Import")
178  {
179  Teuchos::Array<int> dummy;
180  init (source, target, false, dummy, Teuchos::null);
181  }
182 
183  template <class LocalOrdinal, class GlobalOrdinal, class Node>
184  Import<LocalOrdinal,GlobalOrdinal,Node>::
185  Import (const Teuchos::RCP<const map_type>& source,
186  const Teuchos::RCP<const map_type>& target,
187  const Teuchos::RCP<Teuchos::ParameterList>& plist) :
188  base_type (source, target, Teuchos::null, plist, "Import")
189  {
190  Teuchos::Array<int> dummy;
191  init (source, target, false, dummy, plist);
192  }
193 
194  template <class LocalOrdinal, class GlobalOrdinal, class Node>
195  Import<LocalOrdinal,GlobalOrdinal,Node>::
196  Import (const Teuchos::RCP<const map_type>& source,
197  const Teuchos::RCP<const map_type>& target,
198  const Teuchos::RCP<Teuchos::FancyOStream>& out,
199  const Teuchos::RCP<Teuchos::ParameterList>& plist) :
200  base_type (source, target, out, plist, "Import")
201  {
202  Teuchos::Array<int> dummy;
203  init (source, target, false, dummy, plist);
204  }
205 
206  template <class LocalOrdinal, class GlobalOrdinal, class Node>
207  Import<LocalOrdinal,GlobalOrdinal,Node>::
208  Import (const Teuchos::RCP<const map_type>& source,
209  const Teuchos::RCP<const map_type>& target,
210  Teuchos::Array<int>& remotePIDs,
211  const Teuchos::RCP<Teuchos::ParameterList>& plist) :
212  base_type (source, target, Teuchos::null, plist, "Import")
213  {
214  init (source, target, true, remotePIDs, plist);
215  }
216 
217  template <class LocalOrdinal, class GlobalOrdinal, class Node>
218  Import<LocalOrdinal,GlobalOrdinal,Node>::
219  Import (const Import<LocalOrdinal,GlobalOrdinal,Node>& rhs) :
220  base_type (rhs)
221  {}
222 
223  template <class LocalOrdinal, class GlobalOrdinal, class Node>
224  Import<LocalOrdinal,GlobalOrdinal,Node>::
225  Import (const Export<LocalOrdinal,GlobalOrdinal,Node>& exporter) :
226  base_type (exporter, typename base_type::reverse_tag ())
227  {}
228 
229  // cblcbl
230  // This is the "createExpert" version of the constructor to be used with pid/gid pairs obtained from
231  // reverse communication
232  template <class LocalOrdinal, class GlobalOrdinal, class Node>
233  Import<LocalOrdinal,GlobalOrdinal,Node>::
234  Import (const Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> >& source,
235  const Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> >& target,
236  const Teuchos::ArrayView<int> & userRemotePIDs,
237  const Teuchos::ArrayView<const LocalOrdinal> & userExportLIDs,
238  const Teuchos::ArrayView<const int> & userExportPIDs,
239  const Teuchos::RCP<Teuchos::ParameterList>& plist,
240  const Teuchos::RCP<Teuchos::FancyOStream>& out) :
241  base_type (source, target, out, plist, "Import")
242  {
243  using ::Tpetra::Details::makeDualViewFromArrayView;
244  using Teuchos::arcp;
245  using Teuchos::Array;
246  using Teuchos::ArrayRCP;
247  using Teuchos::ArrayView;
248  using Teuchos::as;
249  using Teuchos::null;
250  using Teuchos::rcp;
251  using std::endl;
252  using LO = LocalOrdinal;
253  using GO = GlobalOrdinal;
254  using size_type = Teuchos::Array<int>::size_type;
255 
256  std::unique_ptr<std::string> prefix;
257  if (this->verbose ()) {
258  auto comm = source.is_null () ? Teuchos::null : source->getComm ();
259  const int myRank = comm.is_null () ? -1 : comm->getRank ();
260  std::ostringstream os;
261  os << "Proc " << myRank << ": Tpetra::Import createExpert ctor: ";
262  prefix = std::unique_ptr<std::string> (new std::string (os.str ()));
263  os << "Start" << endl;
264  this->verboseOutputStream () << os.str ();
265  }
266 
267  ArrayView<const GO> sourceGIDs = source->getLocalElementList ();
268  ArrayView<const GO> targetGIDs = target->getLocalElementList ();
269 
270  Array<GO> tRemoteGIDs;
271  if (this->verbose ()) {
272  std::ostringstream os;
273  os << *prefix << "Call setupSamePermuteRemote" << endl;
274  this->verboseOutputStream () << os.str ();
275  }
276  setupSamePermuteRemote (tRemoteGIDs);
277 
278  if (this->verbose ()) {
279  std::ostringstream os;
280  os << *prefix << "Sort & filter IDs" << endl;
281  this->verboseOutputStream () << os.str ();
282  }
283 
284  auto tRemoteLIDs = this->TransferData_->remoteLIDs_.view_host ();
285  this->TransferData_->remoteLIDs_.modify_host ();
286  Teuchos::Array<int> tRemotePIDs (userRemotePIDs);
287 
288  if (this->verbose () && this->getNumRemoteIDs () > 0 && ! source->isDistributed ()) {
289  std::ostringstream os;
290  os << *prefix << "Target Map has remote LIDs but source Map is not "
291  "distributed. Importing to a submap of the target Map." << endl;
292  this->verboseOutputStream () << os.str ();
293  }
294  // FIXME (mfh 03 Feb 2019) I don't see this as "abuse"; it's
295  // perfectly valid Petra Object Model behavior.
296  TPETRA_ABUSE_WARNING
297  (getNumRemoteIDs () > 0 && ! source->isDistributed (),
298  std::runtime_error,
299  "::constructExpert: Target Map has remote LIDs but source Map "
300  "is not distributed. Importing to a submap of the target Map.");
301  TEUCHOS_TEST_FOR_EXCEPTION
302  (tRemotePIDs.size () != tRemoteGIDs.size () ||
303  size_t (tRemoteGIDs.size ()) != size_t (tRemoteLIDs.extent (0)),
304  std::runtime_error, "Import::Import createExpert version: "
305  "Size mismatch on userRemotePIDs, remoteGIDs, and remoteLIDs "
306  "Array's to sort3.");
307 
308  sort3 (tRemotePIDs.begin (),
309  tRemotePIDs.end (),
310  tRemoteGIDs.begin (),
311  tRemoteLIDs.data ());
312 
313  //Get rid of IDs that don't exist in SourceMap
314  size_type cnt = 0;
315  size_type indexIntoRemotePIDs = tRemotePIDs.size ();
316  for (size_type i = 0; i < indexIntoRemotePIDs; ++i) {
317  if (tRemotePIDs[i] == -1) {
318  ++cnt;
319  }
320  }
321 
322  if (cnt == 0) { // done modifying remoteLIDs_
323  this->TransferData_->remoteLIDs_.sync_device ();
324  }
325  else {
326  if (indexIntoRemotePIDs - cnt > 0) {
327  Array<GO> newRemoteGIDs(indexIntoRemotePIDs-cnt);
328  Array<LO> newRemoteLIDs(indexIntoRemotePIDs-cnt);
329  Array<int> newRemotePIDs(indexIntoRemotePIDs-cnt);
330  cnt = 0;
331 
332  for (size_type j = 0; j < indexIntoRemotePIDs; ++j)
333  if(tRemotePIDs[j] != -1) {
334  newRemoteGIDs[cnt] = tRemoteGIDs[j];
335  newRemotePIDs[cnt] = tRemotePIDs[j];
336  newRemoteLIDs[cnt] = target->getLocalElement(tRemoteGIDs[j]);
337  ++cnt;
338  }
339  indexIntoRemotePIDs = cnt;
340  tRemoteGIDs = newRemoteGIDs;
341  tRemotePIDs = newRemotePIDs;
342  makeDualViewFromArrayView (this->TransferData_->remoteLIDs_,
343  newRemoteLIDs ().getConst (),
344  "remoteLIDs");
345  }
346  else { //valid RemoteIDs empty
347  indexIntoRemotePIDs = 0;
348  tRemoteGIDs.clear();
349  tRemotePIDs.clear();
350  this->TransferData_->remoteLIDs_ = decltype (this->TransferData_->remoteLIDs_) ();
351  }
352  }
353 
354  this->TransferData_->exportPIDs_ = Teuchos::Array<int> (userExportPIDs);
355  makeDualViewFromArrayView (this->TransferData_->exportLIDs_,
356  userExportLIDs, "exportLIDs");
357 
358  bool locallyComplete = true;
359  for (size_type i = 0; i < userExportPIDs.size () && locallyComplete; ++i) {
360  if (userExportPIDs[i] == -1) {
361  locallyComplete = false;
362  }
363  }
364  this->TransferData_->isLocallyComplete_ = locallyComplete;
365 
366  if (this->verbose ()) {
367  std::ostringstream os;
368  os << *prefix << "locallyComplete: "
369  << (locallyComplete ? "true" : "false")
370  << "; call createFromSendsAndRecvs" << endl;
371  this->verboseOutputStream () << os.str ();
372  }
373  {
374 #ifdef HAVE_TPETRA_MMM_TIMINGS
375  std::string mmm_prefix =
376  std::string("Tpetra ") + std::string(":iport_ctor:cFSAR ");
377 
378  auto MM3(*Teuchos::TimeMonitor::getNewTimer(mmm_prefix));
379 #endif
380  Distributor& distributor = this->TransferData_->distributor_;
381  distributor.createFromSendsAndRecvs (this->TransferData_->exportPIDs_, tRemotePIDs);
382  }
383 
384  this->detectRemoteExportLIDsContiguous();
385 
386  TEUCHOS_ASSERT( ! this->TransferData_->permuteFromLIDs_.need_sync_device () );
387  TEUCHOS_ASSERT( ! this->TransferData_->permuteFromLIDs_.need_sync_host () );
388  TEUCHOS_ASSERT( ! this->TransferData_->permuteToLIDs_.need_sync_device () );
389  TEUCHOS_ASSERT( ! this->TransferData_->permuteToLIDs_.need_sync_host () );
390  TEUCHOS_ASSERT( ! this->TransferData_->remoteLIDs_.need_sync_device () );
391  TEUCHOS_ASSERT( ! this->TransferData_->remoteLIDs_.need_sync_host () );
392  TEUCHOS_ASSERT( ! this->TransferData_->exportLIDs_.need_sync_device () );
393  TEUCHOS_ASSERT( ! this->TransferData_->exportLIDs_.need_sync_host () );
394  }
395 
396  template <class LocalOrdinal, class GlobalOrdinal, class Node>
397  Import<LocalOrdinal,GlobalOrdinal,Node>::
398  Import (const Teuchos::RCP<const map_type>& source,
399  const Teuchos::RCP<const map_type>& target,
400  const size_t numSameIDs,
401  Teuchos::Array<LocalOrdinal>& permuteToLIDs,
402  Teuchos::Array<LocalOrdinal>& permuteFromLIDs,
403  Teuchos::Array<LocalOrdinal>& remoteLIDs,
404  Teuchos::Array<LocalOrdinal>& exportLIDs,
405  Teuchos::Array<int>& exportPIDs,
406  Distributor& distributor,
407  const Teuchos::RCP<Teuchos::FancyOStream>& out,
408  const Teuchos::RCP<Teuchos::ParameterList>& plist) :
409  base_type (source, target, out, plist, "Import")
410  {
411  using ::Tpetra::Details::makeDualViewFromArrayView;
412  using std::endl;
413 
414  std::unique_ptr<std::string> prefix;
415  if (this->verbose ()) {
416  auto comm = source.is_null () ? Teuchos::null : source->getComm ();
417  const int myRank = comm.is_null () ? -1 : comm->getRank ();
418  std::ostringstream os;
419  os << "Proc " << myRank << ": Tpetra::Import export ctor: ";
420  prefix = std::unique_ptr<std::string> (new std::string (os.str ()));
421  os << "Start" << endl;
422  this->verboseOutputStream () << os.str ();
423  }
424 
425  bool locallyComplete = true;
426  for (Teuchos::Array<int>::size_type i = 0; i < exportPIDs.size (); ++i) {
427  if (exportPIDs[i] == -1) {
428  locallyComplete = false;
429  }
430  }
431  if (this->verbose ()) {
432  std::ostringstream os;
433  os << *prefix << "numSameIDs: " << numSameIDs << ", locallyComplete: "
434  << (locallyComplete ? "true" : "false") << endl;
435  this->verboseOutputStream () << os.str ();
436  }
437 
438  this->TransferData_->isLocallyComplete_ = locallyComplete;
439  this->TransferData_->numSameIDs_ = numSameIDs;
440 
441  makeDualViewFromArrayView (this->TransferData_->permuteToLIDs_,
442  permuteToLIDs ().getConst (),
443  "permuteToLIDs");
444  TEUCHOS_ASSERT( size_t (this->TransferData_->permuteToLIDs_.extent (0)) ==
445  size_t (permuteToLIDs.size ()) );
446  makeDualViewFromArrayView (this->TransferData_->permuteFromLIDs_,
447  permuteFromLIDs ().getConst (),
448  "permuteFromLIDs");
449  TEUCHOS_ASSERT( size_t (this->TransferData_->permuteFromLIDs_.extent (0)) ==
450  size_t (permuteFromLIDs.size ()) );
451  makeDualViewFromArrayView (this->TransferData_->remoteLIDs_,
452  remoteLIDs ().getConst (),
453  "remoteLIDs");
454  TEUCHOS_ASSERT( size_t (this->TransferData_->remoteLIDs_.extent (0)) ==
455  size_t (remoteLIDs.size ()) );
456  makeDualViewFromArrayView (this->TransferData_->exportLIDs_,
457  exportLIDs ().getConst (),
458  "exportLIDs");
459  TEUCHOS_ASSERT( size_t (this->TransferData_->exportLIDs_.extent (0)) ==
460  size_t (exportLIDs.size ()) );
461  this->TransferData_->exportPIDs_.swap (exportPIDs);
462  this->TransferData_->distributor_.swap (distributor);
463 
464  this->detectRemoteExportLIDsContiguous();
465 
466  TEUCHOS_ASSERT( ! this->TransferData_->permuteFromLIDs_.need_sync_device () );
467  TEUCHOS_ASSERT( ! this->TransferData_->permuteFromLIDs_.need_sync_host () );
468  TEUCHOS_ASSERT( ! this->TransferData_->permuteToLIDs_.need_sync_device () );
469  TEUCHOS_ASSERT( ! this->TransferData_->permuteToLIDs_.need_sync_host () );
470  TEUCHOS_ASSERT( ! this->TransferData_->remoteLIDs_.need_sync_device () );
471  TEUCHOS_ASSERT( ! this->TransferData_->remoteLIDs_.need_sync_host () );
472  TEUCHOS_ASSERT( ! this->TransferData_->exportLIDs_.need_sync_device () );
473  TEUCHOS_ASSERT( ! this->TransferData_->exportLIDs_.need_sync_host () );
474  }
475 
476  namespace { // (anonymous)
477 
478  template <class LO, class GO, class NT>
479  struct ImportLocalSetupResult
480  {
481  Teuchos::RCP<const ::Tpetra::Map<LO, GO, NT> > targetMap;
482  LO numSameIDs;
483  // std::vector<LO> permuteToLIDs; // users aren't supposed to have permutes
484  // std::vector<LO> permuteFromLIDs; // users aren't suppoosed to have permutes
485  std::vector<GO> remoteGIDs;
486  std::vector<LO> remoteLIDs;
487  std::vector<int> remotePIDs;
488  LO numPermutes; // users aren't supposed to have permutes
489  };
490 
491  template<class T>
492  void printArray (std::ostream& out, const T x[], const std::size_t N)
493  {
494  out << "[";
495  for (std::size_t k = 0; k < N; ++k) {
496  out << x[k];
497  if (k + 1 < N) {
498  out << ", ";
499  }
500  }
501  out << "]";
502  }
503 
504  template<class LO, class GO, class NT>
505  ImportLocalSetupResult<LO, GO, NT>
506  setupSamePermuteRemoteFromUserGlobalIndexList (const ::Tpetra::Map<LO, GO, NT>& sourceMap,
507  const GO targetMapRemoteOrPermuteGlobalIndices[],
508  const int targetMapRemoteOrPermuteProcessRanks[],
509  const LO numTargetMapRemoteOrPermuteGlobalIndices,
510  const bool mayReorderTargetMapIndicesLocally,
511  Teuchos::FancyOStream* out, // only valid if verbose
512  const std::string* verboseHeader, // only valid if verbose
513  const bool verbose,
514  const bool debug)
515  {
516  using std::endl;
517  const int myRank = sourceMap.getComm ()->getRank ();
518  ImportLocalSetupResult<LO, GO, NT> result;
519 
520  if (verbose) {
521  std::ostringstream os;
522  os << *verboseHeader << "- Import::setupSPR w/ remote GIDs & PIDs: " << endl
523  << *verboseHeader << " Input GIDs: ";
524  printArray (os, targetMapRemoteOrPermuteGlobalIndices, numTargetMapRemoteOrPermuteGlobalIndices);
525  os << endl << " Input PIDs: ";
526  printArray (os, targetMapRemoteOrPermuteProcessRanks, numTargetMapRemoteOrPermuteGlobalIndices);
527  os << endl;
528  *out << os.str ();
529  }
530 
531  // In debug mode, check whether any of the input GIDs are
532  // actually in the source Map on the calling process. That's an
533  // error, because it means duplicate GIDs on the calling
534  // process. Also check if any of the input PIDs are invalid.
535  if (debug) {
536  std::vector<GO> badGIDs;
537  std::vector<int> badPIDs;
538  const Teuchos::Comm<int>& comm = * (sourceMap.getComm ());
539  const int numProcs = comm.getSize ();
540 
541  for (LO k = 0; k < numTargetMapRemoteOrPermuteGlobalIndices; ++k) {
542  const GO tgtGID = targetMapRemoteOrPermuteGlobalIndices[k];
543  if (sourceMap.isNodeGlobalElement (tgtGID)) {
544  badGIDs.push_back (tgtGID);
545  }
546  const int tgtPID = targetMapRemoteOrPermuteProcessRanks[k];
547  if (tgtPID < 0 || tgtPID >= numProcs) {
548  badPIDs.push_back (tgtPID);
549  }
550  }
551 
552  std::array<int, 2> lclStatus {{
553  badGIDs.size () == 0 ? 1 : 0,
554  badPIDs.size () == 0 ? 1 : 0
555  }};
556  std::array<int, 2> gblStatus {{0, 0}}; // output argument
557  Teuchos::reduceAll<int, int> (comm, Teuchos::REDUCE_MIN, 2,
558  lclStatus.data (), gblStatus.data ());
559  const bool good = gblStatus[0] == 1 && gblStatus[1] == 1;
560  // Don't actually print all the "bad" GIDs and/or PIDs unless
561  // in verbose mode, since there could be many of them.
562  if (verbose && gblStatus[0] != 1) {
563  std::ostringstream os;
564  os << *verboseHeader << "- Some input GIDs are already in the source Map: ";
565  printArray (os, badGIDs.data (), badGIDs.size ());
566  os << endl;
567  ::Tpetra::Details::gathervPrint (*out, os.str (), comm);
568  }
569  if (verbose && gblStatus[0] != 1) {
570  std::ostringstream os;
571  os << *verboseHeader << "- Some input PIDs are invalid: ";
572  printArray (os, badPIDs.data (), badPIDs.size ());
573  os << endl;
574  ::Tpetra::Details::gathervPrint (*out, os.str (), comm);
575  }
576 
577  if (! good) {
578  std::ostringstream os;
579  os << "Tpetra::Import constructor that takes remote GIDs and PIDs: ";
580  if (gblStatus[0] != 1) {
581  os << "Some input GIDs (global indices) are already in the source Map! ";
582  }
583  if (gblStatus[1] != 1) {
584  os << "Some input PIDs (process ranks) are invalid! ";
585  }
586  os << "Rerun with the environment variable TPETRA_VERBOSE=Tpetra::Import "
587  "to see what GIDs and/or PIDs are bad.";
588  TEUCHOS_TEST_FOR_EXCEPTION(true, std::invalid_argument, os.str ());
589  }
590  }
591 
592  // Create list of GIDs to go into target Map. We need to copy
593  // the GIDs into this list anyway, so once we have them, we can
594  // sort the "remotes" in place.
595  const LO numLclSrcIDs = static_cast<LO> (sourceMap.getLocalNumElements ());
596  const LO numLclTgtIDs = numLclSrcIDs + numTargetMapRemoteOrPermuteGlobalIndices;
597  if (verbose) {
598  std::ostringstream os;
599  os << *verboseHeader << "- Copy source Map GIDs into target Map GID list: "
600  "numLclSrcIDs=" << numLclSrcIDs
601  << ", numTargetMapRemoteOrPermuteGlobalIndices="
602  << numTargetMapRemoteOrPermuteGlobalIndices << endl;
603  *out << os.str ();
604  }
605  std::vector<GO> tgtGIDs (numLclTgtIDs); // will go into target Map ctor
606  if (sourceMap.isContiguous ()) {
607  GO curTgtGID = sourceMap.getMinGlobalIndex ();
608  for (LO k = 0; k < numLclSrcIDs; ++k, ++curTgtGID) {
609  tgtGIDs[k] = curTgtGID;
610  }
611  }
612  else { // avoid calling getLocalElementList on a contiguous Map
613  auto srcGIDs = sourceMap.getLocalElementList (); // Teuchos::ArrayView has a different
614  for (LO k = 0; k < numLclSrcIDs; ++k) { // iterator type, so can't std::copy
615  tgtGIDs[k] = srcGIDs[k];
616  }
617  }
618  std::copy (targetMapRemoteOrPermuteGlobalIndices,
619  targetMapRemoteOrPermuteGlobalIndices + numTargetMapRemoteOrPermuteGlobalIndices,
620  tgtGIDs.begin () + numLclSrcIDs);
621 
622  // Optionally, sort input by process rank, so that remotes
623  // coming from the same process are grouped together. Only sort
624  // remote GIDs. While doing so, detect permutes (input "remote"
625  // GIDs whose rank is the same as that of the calling process).
626  //
627  // Permutes are actually an error. We normally detect them in
628  // debug mode, but if we sort, we have a nearly free opportunity
629  // to do so. We may also safely ignore permutes as duplicates.
630  //
631  // NOTE: tgtPIDs only includes remotes, not source Map entries.
632  if (verbose) {
633  std::ostringstream os;
634  os << *verboseHeader << "- Sort by PID? "
635  << (mayReorderTargetMapIndicesLocally ? "true" : "false") << endl;
636  *out << os.str ();
637  }
638  std::vector<int> tgtPIDs (targetMapRemoteOrPermuteProcessRanks,
639  targetMapRemoteOrPermuteProcessRanks + numTargetMapRemoteOrPermuteGlobalIndices);
640  result.numPermutes = 0;
641  if (mayReorderTargetMapIndicesLocally) {
642  Tpetra::sort2 (tgtPIDs.begin (), tgtPIDs.end (), tgtGIDs.begin () + numLclSrcIDs);
643  auto range = std::equal_range (tgtPIDs.begin (), tgtPIDs.end (), myRank); // binary search
644  if (range.second > range.first) {
645  result.numPermutes = static_cast<LO> (range.second - range.first);
646  }
647  }
648  else { // don't sort; linear search to count permutes
649  result.numPermutes = static_cast<LO> (std::count (tgtPIDs.begin (), tgtPIDs.end (), myRank));
650  }
651  // The _actual_ number of remotes.
652  const LO numRemotes = numTargetMapRemoteOrPermuteGlobalIndices - result.numPermutes;
653  result.numSameIDs = static_cast<LO> (sourceMap.getLocalNumElements ());
654 
655  if (verbose) {
656  std::ostringstream os;
657  os << *verboseHeader << "- numSame=" << result.numSameIDs
658  << ", numPermutes=" << result.numPermutes
659  << ", numRemotes=" << numRemotes << endl;
660  *out << os.str ();
661  }
662 
663  if (result.numPermutes == 0) {
664  if (verbose) {
665  std::ostringstream os;
666  os << *verboseHeader << "- No permutes" << endl;
667  *out << os.str ();
668  }
669  result.remoteGIDs = std::vector<GO> (tgtGIDs.begin () + numLclSrcIDs, tgtGIDs.end ());
670  result.remotePIDs.swap (tgtPIDs);
671  result.remoteLIDs.resize (numRemotes);
672  for (LO k = 0; k < numRemotes; ++k) {
673  const LO tgtLid = result.numSameIDs + k;
674  result.remoteLIDs[k] = tgtLid;
675  }
676  if (verbose) {
677  std::ostringstream os;
678  os << *verboseHeader << "- Remote GIDs: "
679  << Teuchos::toString (result.remoteGIDs) << endl;
680  os << *verboseHeader << "- Remote PIDs: "
681  << Teuchos::toString (result.remotePIDs) << endl;
682  os << *verboseHeader << "- Remote LIDs: "
683  << Teuchos::toString (result.remoteLIDs) << endl;
684  *out << os.str ();
685  }
686  }
687  else { // separate permutes from remotes
688  // This case doesn't need to be optimal; it just needs to be
689  // correct. Users really shouldn't give permutes to this
690  // Import constructor.
691  result.remoteGIDs.reserve (numRemotes);
692  result.remoteLIDs.reserve (numRemotes);
693  result.remotePIDs.reserve (numRemotes);
694  for (LO k = 0; k < numTargetMapRemoteOrPermuteGlobalIndices; ++k) {
695  const LO tgtLid = result.numSameIDs + k;
696  const GO tgtGid = tgtGIDs[numLclSrcIDs + k];
697  const int tgtPid = tgtPIDs[k];
698 
699  if (tgtPid != myRank) { // it's a remote
700  result.remoteGIDs.push_back (tgtGid);
701  result.remoteLIDs.push_back (tgtLid);
702  result.remotePIDs.push_back (tgtPid);
703  }
704  }
705  if (verbose) {
706  std::ostringstream os;
707  os << *verboseHeader << "- Some permutes" << endl;
708  *out << os.str ();
709  }
710  }
711 
712  if (sourceMap.isDistributed ()) {
713  if (verbose) {
714  std::ostringstream os;
715  os << *verboseHeader << "- Sort remotes by PID, as Import always does"
716  << endl
717  << *verboseHeader << "-- remotePIDs before: "
718  << Teuchos::toString (result.remotePIDs) << endl
719  << *verboseHeader << "-- remoteGIDs before: "
720  << Teuchos::toString (result.remoteGIDs) << endl
721  << *verboseHeader << "-- remoteLIDs before: "
722  << Teuchos::toString (result.remoteLIDs) << endl;
723  *out << os.str ();
724  }
725  // Import always sorts these, regardless of what the user wanted.
726  sort3 (result.remotePIDs.begin (),
727  result.remotePIDs.end (),
728  result.remoteGIDs.begin (),
729  result.remoteLIDs.begin ());
730  if (verbose) {
731  std::ostringstream os;
732  os << *verboseHeader << "-- remotePIDs after: "
733  << Teuchos::toString (result.remotePIDs) << endl
734  << *verboseHeader << "-- remoteGIDs after: "
735  << Teuchos::toString (result.remoteGIDs) << endl
736  << *verboseHeader << "-- remoteLIDs after: "
737  << Teuchos::toString (result.remoteLIDs) << endl;
738  std::cerr << os.str ();
739  }
740  }
741 
742  if (verbose) {
743  std::ostringstream os;
744  os << *verboseHeader << "- Make target Map" << endl;
745  *out << os.str ();
746  }
747  using ::Teuchos::rcp;
748  typedef ::Tpetra::Map<LO, GO, NT> map_type;
749  typedef ::Tpetra::global_size_t GST;
750  const GST MAP_COMPUTES_GLOBAL_COUNT = ::Teuchos::OrdinalTraits<GST>::invalid ();
751  result.targetMap = rcp (new map_type (MAP_COMPUTES_GLOBAL_COUNT,
752  tgtGIDs.data (),
753  numLclTgtIDs,
754  sourceMap.getIndexBase (),
755  sourceMap.getComm ()));
756  if (verbose) {
757  std::ostringstream os;
758  os << *verboseHeader << "- Done with sameSPR..." << endl;
759  *out << os.str ();
760  }
761  return result;
762  }
763  } // namespace (anonymous)
764 
765  template <class LocalOrdinal, class GlobalOrdinal, class Node>
766  Import<LocalOrdinal,GlobalOrdinal,Node>::
767  Import (const Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> >& sourceMap,
768  const GlobalOrdinal targetMapRemoteOrPermuteGlobalIndices[],
769  const int targetMapRemoteOrPermuteProcessRanks[],
770  const LocalOrdinal numTargetMapRemoteOrPermuteGlobalIndices,
771  const bool mayReorderTargetMapIndicesLocally,
772  const Teuchos::RCP<Teuchos::ParameterList>& plist,
773  const Teuchos::RCP<Teuchos::FancyOStream>& debugOutput) :
774  // Special case: target Map is null on base_type construction.
775  // It's worthwhile for invariants like out_ not being null.
776  // We'll set TransferData_ again below.
777  base_type (sourceMap, Teuchos::null, debugOutput, plist, "Import")
778  {
779  using ::Tpetra::Details::Behavior;
780  using ::Tpetra::Details::makeDualViewFromOwningHostView;
781  using ::Tpetra::Details::makeDualViewFromVector;
782  using ::Tpetra::Details::printDualView;
783  using ::Tpetra::Details::view_alloc_no_init;
784  using Teuchos::ArrayView;
785  using Teuchos::getFancyOStream;
786  using Teuchos::RCP;
787  using Teuchos::rcp;
788  using Teuchos::rcpFromRef;
789  using std::endl;
790  typedef LocalOrdinal LO;
791  typedef GlobalOrdinal GO;
792  typedef Node NT;
793 
794  const bool debug = Behavior::debug ("Import") ||
795  Behavior::debug ("Tpetra::Import");
796 
797  std::unique_ptr<std::string> verbPfx;
798  if (this->verbose ()) {
799  std::ostringstream os;
800  const int myRank = sourceMap->getComm ()->getRank ();
801  os << "Proc " << myRank << ": Tpetra::Import ctor from remotes: ";
802  verbPfx = std::unique_ptr<std::string> (new std::string (os.str ()));
803  os << "mayReorder=" << (mayReorderTargetMapIndicesLocally ? "true" : "false")
804  << endl;
805  this->verboseOutputStream () << os.str ();
806  }
807 
808  TEUCHOS_ASSERT( ! this->TransferData_.is_null () );
809  ImportLocalSetupResult<LO, GO, NT> localSetupResult =
810  setupSamePermuteRemoteFromUserGlobalIndexList<LO, GO, NT>
811  (*sourceMap,
812  targetMapRemoteOrPermuteGlobalIndices,
813  targetMapRemoteOrPermuteProcessRanks,
814  numTargetMapRemoteOrPermuteGlobalIndices,
815  mayReorderTargetMapIndicesLocally,
816  this->TransferData_->out_.getRawPtr (),
817  verbPfx.get (),
818  this->verbose (),
819  debug);
820 
821  // Since we invoked the base_type constructor above, we know that
822  // out_ is nonnull, so we don't have to waste time creating it
823  // again.
824  using data_type = ImportExportData<LO, GO, NT>;
825  TEUCHOS_ASSERT( ! this->TransferData_.is_null () );
826  this->TransferData_ = rcp (new data_type (sourceMap,
827  localSetupResult.targetMap,
828  this->TransferData_->out_,
829  plist));
830  this->TransferData_->numSameIDs_ = localSetupResult.numSameIDs;
831  // Skip permutes; they are user error, because they duplicate
832  // non-remote indices.
833  makeDualViewFromVector (this->TransferData_->remoteLIDs_,
834  localSetupResult.remoteLIDs,
835  "remoteLIDs");
836  // "Is locally complete" for an Import means that all target Map
837  // indices on the calling process exist on at least one process
838  // (not necessarily this one) in the source Map. For this
839  // constructor, this is true if and only if all input target PIDs
840  // are valid PIDs in the communicator.
841  //
842  // FIXME (mfh 20 Feb 2018) For now, assume this is always true.
843  this->TransferData_->isLocallyComplete_ = true;
844 
845  Teuchos::Array<GO> exportGIDs;
846  if (sourceMap->isDistributed ()) {
847  if (this->verbose ()) {
848  std::ostringstream os;
849  os << *verbPfx << "Make Distributor (createFromRecvs)" << endl;
850  this->verboseOutputStream () << os.str ();
851  }
852  ArrayView<const GO> remoteGIDs (localSetupResult.remoteGIDs.data (),
853  localSetupResult.remoteGIDs.size ());
854  ArrayView<const int> remotePIDs (localSetupResult.remotePIDs.data (),
855  localSetupResult.remotePIDs.size ());
856  // Call Distributor::createFromRecvs to turn the remote GIDs and
857  // their owning PIDs into a send-and-receive communication plan.
858  // remoteGIDs and remotePIDs are input; exportGIDs and
859  // exportPIDs are output arrays that createFromRecvs allocates.
860  Distributor& distributor = this->TransferData_->distributor_;
861  distributor.createFromRecvs (remoteGIDs,
862  remotePIDs,
863  exportGIDs,
864  this->TransferData_->exportPIDs_);
865  // Find the LIDs corresponding to the (outgoing) GIDs in
866  // exportGIDs. For sparse matrix-vector multiply, this tells
867  // the calling process how to index into the source vector to
868  // get the elements which it needs to send.
869  //
870  // NOTE (mfh 03 Mar 2014) This is now a candidate for a
871  // thread-parallel kernel, but only if using the new thread-safe
872  // Map implementation.
873  if (this->verbose ()) {
874  std::ostringstream os;
875  os << *verbPfx << "Compute exportLIDs" << endl;
876  this->verboseOutputStream () << os.str ();
877  }
878  using size_type = typename Teuchos::Array<GO>::size_type;
879  const size_type numExportIDs = exportGIDs.size ();
880 
881  typename decltype (this->TransferData_->exportLIDs_)::t_host
882  exportLIDs (view_alloc_no_init ("exportLIDs"), numExportIDs);
883 
884  for (size_type k = 0; k < numExportIDs; ++k) {
885  exportLIDs[k] = sourceMap->getLocalElement (exportGIDs[k]);
886  }
887  makeDualViewFromOwningHostView (this->TransferData_->exportLIDs_, exportLIDs);
888  }
889 
890  if (this->verbose ()) {
891  std::ostringstream os;
892  os << *verbPfx;
893  printDualView (os, this->TransferData_->remoteLIDs_,
894  "ImportExportData::remoteLIDs_");
895  os << endl;
896  this->verboseOutputStream () << os.str ();
897  }
898  if (this->verbose ()) {
899  std::ostringstream os;
900  os << *verbPfx << "Done!" << endl;
901  this->verboseOutputStream () << os.str ();
902  }
903  }
904 
905  template <class LocalOrdinal, class GlobalOrdinal, class Node>
906  void
907  Import<LocalOrdinal,GlobalOrdinal,Node>::
908  describe (Teuchos::FancyOStream& out,
909  const Teuchos::EVerbosityLevel verbLevel) const
910  {
911  // Call the base class' method. It does all the work.
912  this->describeImpl (out, "Tpetra::Import", verbLevel);
913  }
914 
915  template <class LocalOrdinal, class GlobalOrdinal, class Node>
916  void Import<LocalOrdinal,GlobalOrdinal,Node>::
917  print (std::ostream& os) const
918  {
919  auto out = Teuchos::getFancyOStream (Teuchos::rcpFromRef (os));
920  // "Print" traditionally meant "everything."
921  this->describe (*out, Teuchos::VERB_EXTREME);
922  }
923 
924  template <class LocalOrdinal, class GlobalOrdinal, class Node>
925  void
926  Import<LocalOrdinal,GlobalOrdinal,Node>::
927  setupSamePermuteRemote (Teuchos::Array<GlobalOrdinal>& remoteGIDs)
928  {
929  using ::Tpetra::Details::makeDualViewFromOwningHostView;
930  using ::Tpetra::Details::ProfilingRegion;
931  using ::Tpetra::Details::view_alloc_no_init;
932  using Teuchos::arcp;
933  using Teuchos::Array;
934  using Teuchos::ArrayRCP;
935  using Teuchos::ArrayView;
936  using Teuchos::as;
937  using Teuchos::null;
938  typedef LocalOrdinal LO;
939  typedef GlobalOrdinal GO;
940  typedef typename ArrayView<const GO>::size_type size_type;
941  ProfilingRegion regionExport ("Tpetra::Import::setupSamePermuteRemote");
942 
943  const map_type& source = * (this->getSourceMap ());
944  const map_type& target = * (this->getTargetMap ());
945  ArrayView<const GO> sourceGIDs = source.getLocalElementList ();
946  ArrayView<const GO> targetGIDs = target.getLocalElementList ();
947 
948 #ifdef HAVE_TPETRA_DEBUG
949  ArrayView<const GO> rawSrcGids = sourceGIDs;
950  ArrayView<const GO> rawTgtGids = targetGIDs;
951 #else
952  const GO* const rawSrcGids = sourceGIDs.getRawPtr ();
953  const GO* const rawTgtGids = targetGIDs.getRawPtr ();
954 #endif // HAVE_TPETRA_DEBUG
955  const size_type numSrcGids = sourceGIDs.size ();
956  const size_type numTgtGids = targetGIDs.size ();
957  const size_type numGids = std::min (numSrcGids, numTgtGids);
958 
959  // Compute numSameIDs_: the number of initial GIDs that are the
960  // same (and occur in the same order) in both Maps. The point of
961  // numSameIDs_ is for the common case of an Import where all the
962  // overlapping GIDs are at the end of the target Map, but
963  // otherwise the source and target Maps are the same. This allows
964  // a fast contiguous copy for the initial "same IDs."
965  size_type numSameGids = 0;
966  for ( ; numSameGids < numGids && rawSrcGids[numSameGids] == rawTgtGids[numSameGids]; ++numSameGids)
967  {} // third clause of 'for' does everything
968  this->TransferData_->numSameIDs_ = numSameGids;
969 
970  // Compute permuteToLIDs_, permuteFromLIDs_, remoteGIDs, and
971  // remoteLIDs_. The first two arrays are IDs to be permuted, and
972  // the latter two arrays are IDs to be received ("imported"),
973  // called "remote" IDs.
974  //
975  // IDs to permute are in both the source and target Maps, which
976  // means we don't have to send or receive them, but we do have to
977  // rearrange (permute) them in general. IDs to receive are in the
978  // target Map, but not the source Map.
979 
980  // Iterate over the target Map's LIDs, since we only need to do
981  // GID -> LID lookups for the source Map.
982  const LO LINVALID = Teuchos::OrdinalTraits<LO>::invalid ();
983  const LO numTgtLids = as<LO> (numTgtGids);
984  LO numPermutes = 0;
985 
986  for (LO tgtLid = numSameGids; tgtLid < numTgtLids; ++tgtLid) {
987  const GO curTargetGid = rawTgtGids[tgtLid];
988  // getLocalElement() returns LINVALID if the GID isn't in the
989  // source Map. This saves us a lookup (which
990  // isNodeGlobalElement() would do).
991  const LO srcLid = source.getLocalElement (curTargetGid);
992  if (srcLid != LINVALID) { // if source.isNodeGlobalElement (curTargetGid)
993  ++numPermutes;
994  }
995  }
996  const LO numRemotes = (numTgtLids - numSameGids) - numPermutes;
997 
998  using host_perm_type =
999  typename decltype (this->TransferData_->permuteToLIDs_)::t_host;
1000  host_perm_type permuteToLIDs
1001  (view_alloc_no_init ("permuteToLIDs"), numPermutes);
1002  host_perm_type permuteFromLIDs
1003  (view_alloc_no_init ("permuteFromLIDs"), numPermutes);
1004  typename decltype (this->TransferData_->remoteLIDs_)::t_host remoteLIDs
1005  (view_alloc_no_init ("permuteFromLIDs"), numRemotes);
1006 
1007  {
1008  LO numPermutes2 = 0;
1009  LO numRemotes2 = 0;
1010  for (LO tgtLid = numSameGids; tgtLid < numTgtLids; ++tgtLid) {
1011  const GO curTargetGid = rawTgtGids[tgtLid];
1012  const LO srcLid = source.getLocalElement (curTargetGid);
1013  if (srcLid != LINVALID) {
1014  permuteToLIDs[numPermutes2] = tgtLid;
1015  permuteFromLIDs[numPermutes2] = srcLid;
1016  ++numPermutes2;
1017  }
1018  else {
1019  remoteGIDs.push_back (curTargetGid);
1020  remoteLIDs[numRemotes2] = tgtLid;
1021  ++numRemotes2;
1022  }
1023  }
1024  TEUCHOS_ASSERT( numPermutes == numPermutes2 );
1025  TEUCHOS_ASSERT( numRemotes == numRemotes2 );
1026  TEUCHOS_ASSERT( size_t (numPermutes) + remoteGIDs.size () == size_t (numTgtLids - numSameGids) );
1027  }
1028 
1029  makeDualViewFromOwningHostView (this->TransferData_->permuteToLIDs_, permuteToLIDs);
1030  makeDualViewFromOwningHostView (this->TransferData_->permuteFromLIDs_, permuteFromLIDs);
1031  makeDualViewFromOwningHostView (this->TransferData_->remoteLIDs_, remoteLIDs);
1032  if (remoteLIDs.extent (0) != 0 && ! source.isDistributed ()) {
1033  // This Import has remote LIDs, meaning that the target Map has
1034  // entries on this process that are not in the source Map on
1035  // this process. However, the source Map is not distributed
1036  // globally. This implies that this Import is not locally
1037  // complete on this process.
1038  this->TransferData_->isLocallyComplete_ = false;
1039  // mfh 12 Sep 2016: I disagree that this is "abuse"; it may be
1040  // correct behavior, depending on the circumstances.
1041  TPETRA_ABUSE_WARNING
1042  (true, std::runtime_error, "::setupSamePermuteRemote(): Target has "
1043  "remote LIDs but Source is not distributed globally. Importing to a "
1044  "submap of the target map.");
1045  }
1046  }
1047 
1048 
1049  template <class LocalOrdinal, class GlobalOrdinal, class Node>
1050  void Import<LocalOrdinal,GlobalOrdinal,Node>::
1051  setupExport (Teuchos::Array<GlobalOrdinal>& remoteGIDs,
1052  bool useRemotePIDs,
1053  Teuchos::Array<int>& userRemotePIDs,
1054  const Teuchos::RCP<Teuchos::ParameterList>& plist)
1055  {
1056  using ::Tpetra::Details::makeDualViewFromOwningHostView;
1057  using ::Tpetra::Details::view_alloc_no_init;
1058  using Teuchos::Array;
1059  using Teuchos::ArrayView;
1060  using std::endl;
1061  using GO = GlobalOrdinal;
1062  typedef typename Array<int>::difference_type size_type;
1063  const char tfecfFuncName[] = "setupExport: ";
1064  const char suffix[] = " Please report this bug to the Tpetra developers.";
1065 
1066  std::unique_ptr<std::string> prefix;
1067  if (this->verbose ()) {
1068  auto srcMap = this->getSourceMap ();
1069  auto comm = srcMap.is_null () ? Teuchos::null : srcMap->getComm ();
1070  const int myRank = comm.is_null () ? -1 : comm->getRank ();
1071  std::ostringstream os;
1072  os << "Proc " << myRank << ": Tpetra::Import::setupExport: ";
1073  prefix = std::unique_ptr<std::string> (new std::string (os.str ()));
1074  os << "Start" << std::endl;
1075  this->verboseOutputStream () << os.str ();
1076  }
1077 
1078  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1079  (this->getSourceMap ().is_null (), std::logic_error,
1080  "Source Map is null. " << suffix);
1081  const map_type& source = * (this->getSourceMap ());
1082 
1083  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1084  (! useRemotePIDs && (userRemotePIDs.size() > 0), std::invalid_argument,
1085  "remotePIDs are non-empty but their use has not been requested.");
1086  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1087  (userRemotePIDs.size () > 0 && remoteGIDs.size () != userRemotePIDs.size (),
1088  std::invalid_argument, "remotePIDs must either be of size zero or match "
1089  "the size of remoteGIDs.");
1090 
1091  // For each entry remoteGIDs[i], remoteProcIDs[i] will contain
1092  // the process ID of the process that owns that GID.
1093  ArrayView<GO> remoteGIDsView = remoteGIDs ();
1094  ArrayView<int> remoteProcIDsView;
1095 
1096  // lookup == IDNotPresent means that the source Map wasn't able to
1097  // figure out to which processes one or more of the GIDs in the
1098  // given list of remote GIDs belong.
1099  //
1100  // The previous abuse warning said "The target Map has GIDs not
1101  // found in the source Map." This statement could be confusing,
1102  // because it doesn't refer to ownership by the current process,
1103  // but rather to ownership by _any_ process participating in the
1104  // Map. (It could not possibly refer to ownership by the current
1105  // process, since remoteGIDs is exactly the list of GIDs owned by
1106  // the target Map but not owned by the source Map. It was
1107  // constructed that way by setupSamePermuteRemote().)
1108  //
1109  // What this statement means is that the source and target Maps
1110  // don't contain the same set of GIDs globally (over all
1111  // processes). That is, there is at least one GID owned by some
1112  // process in the target Map, which is not owned by _any_ process
1113  // in the source Map.
1114  Array<int> newRemotePIDs;
1115  LookupStatus lookup = AllIDsPresent;
1116 
1117  if (! useRemotePIDs) {
1118  newRemotePIDs.resize (remoteGIDsView.size ());
1119  if (this->verbose ()) {
1120  std::ostringstream os;
1121  os << *prefix << "Call sourceMap.getRemoteIndexList" << endl;
1122  this->verboseOutputStream () << os.str ();
1123  }
1124  lookup = source.getRemoteIndexList (remoteGIDsView, newRemotePIDs ());
1125  }
1126  Array<int>& remoteProcIDs = useRemotePIDs ? userRemotePIDs : newRemotePIDs;
1127 
1128  if (lookup == IDNotPresent) {
1129  // There is at least one GID owned by the calling process in the
1130  // target Map, which is not owned by any process in the source
1131  // Map.
1132  this->TransferData_->isLocallyComplete_ = false;
1133 
1134  // mfh 12 Sep 2016: I disagree that this is "abuse"; it may be
1135  // correct behavior, depending on the circumstances.
1136  TPETRA_ABUSE_WARNING
1137  (true, std::runtime_error, "::setupExport(): the source Map wasn't "
1138  "able to figure out which process owns one or more of the GIDs in the "
1139  "list of remote GIDs. This probably means that there is at least one "
1140  "GID owned by some process in the target Map which is not owned by any"
1141  " process in the source Map. (That is, the source and target Maps do "
1142  "not contain the same set of GIDs globally.)");
1143 
1144  // Ignore remote GIDs that aren't owned by any process in the
1145  // source Map. getRemoteIndexList() gives each of these a
1146  // process ID of -1.
1147 
1148  const size_type numInvalidRemote =
1149  std::count_if (remoteProcIDs.begin (), remoteProcIDs.end (),
1150  std::bind (std::equal_to<int> (), -1, std::placeholders::_1));
1151  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1152  (numInvalidRemote == 0, std::logic_error, "Calling getRemoteIndexList "
1153  "on the source Map returned IDNotPresent, but none of the returned "
1154  "\"remote\" process ranks are -1. Please report this bug to the "
1155  "Tpetra developers.");
1156 
1157 #ifdef HAVE_TPETRA_MMM_TIMINGS
1158  using Teuchos::TimeMonitor;
1159  std::string label;
1160  if(!plist.is_null())
1161  label = plist->get("Timer Label",label);
1162  std::string prefix = std::string("Tpetra ")+ label + std::string(":iport_ctor:setupExport:1 ");
1163  auto MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix)));
1164 #else
1165  (void)plist;
1166 #endif
1167 
1168  // If all of them are invalid, we can delete the whole array.
1169  const size_type totalNumRemote = this->getNumRemoteIDs ();
1170  if (numInvalidRemote == totalNumRemote) {
1171  // all remotes are invalid; we have no remotes; we can delete the remotes
1172  remoteProcIDs.clear ();
1173  remoteGIDs.clear (); // invalidates remoteGIDsView
1174  this->TransferData_->remoteLIDs_ =
1175  decltype (this->TransferData_->remoteLIDs_) ();
1176  }
1177  else {
1178  // Some remotes are valid; we need to keep the valid ones.
1179  // Pack and resize remoteProcIDs, remoteGIDs, and remoteLIDs_.
1180  size_type numValidRemote = 0;
1181 #ifdef HAVE_TPETRA_DEBUG
1182  ArrayView<GO> remoteGIDsPtr = remoteGIDsView;
1183 #else
1184  GO* const remoteGIDsPtr = remoteGIDsView.getRawPtr ();
1185 #endif // HAVE_TPETRA_DEBUG
1186 
1187  // Don't mark the DualView modified, since we'll reallocate it.
1188  auto remoteLIDs = this->TransferData_->remoteLIDs_.view_host ();
1189 
1190  for (size_type r = 0; r < totalNumRemote; ++r) {
1191  // Pack in all the valid remote PIDs and GIDs.
1192  if (remoteProcIDs[r] != -1) {
1193  remoteProcIDs[numValidRemote] = remoteProcIDs[r];
1194  remoteGIDsPtr[numValidRemote] = remoteGIDsPtr[r];
1195  remoteLIDs[numValidRemote] = remoteLIDs[r];
1196  ++numValidRemote;
1197  }
1198  }
1199  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1200  (numValidRemote != totalNumRemote - numInvalidRemote,
1201  std::logic_error, "After removing invalid remote GIDs and packing "
1202  "the valid remote GIDs, numValidRemote = " << numValidRemote
1203  << " != totalNumRemote - numInvalidRemote = "
1204  << totalNumRemote - numInvalidRemote
1205  << ". Please report this bug to the Tpetra developers.");
1206 
1207  remoteProcIDs.resize (numValidRemote);
1208  remoteGIDs.resize (numValidRemote);
1209 
1210  Kokkos::resize (remoteLIDs, numValidRemote);
1211  this->TransferData_->remoteLIDs_ = decltype (this->TransferData_->remoteLIDs_) ();
1212  makeDualViewFromOwningHostView (this->TransferData_->remoteLIDs_, remoteLIDs);
1213  }
1214  // Revalidate the view after clear or resize.
1215  remoteGIDsView = remoteGIDs ();
1216  }
1217 
1218  // Sort remoteProcIDs in ascending order, and apply the resulting
1219  // permutation to remoteGIDs and remoteLIDs_. This ensures that
1220  // remoteProcIDs[i], remoteGIDs[i], and remoteLIDs_[i] all refer
1221  // to the same thing.
1222  {
1223  this->TransferData_->remoteLIDs_.modify_host ();
1224  auto remoteLIDs = this->TransferData_->remoteLIDs_.view_host ();
1225  sort3 (remoteProcIDs.begin (),
1226  remoteProcIDs.end (),
1227  remoteGIDsView.getRawPtr (),
1228  remoteLIDs.data ());
1229  this->TransferData_->remoteLIDs_.sync_device ();
1230  }
1231 
1232  // Call the Distributor's createFromRecvs() method to turn the
1233  // remote GIDs and their owning processes into a send-and-receive
1234  // communication plan. remoteGIDs and remoteProcIDs_ are input;
1235  // exportGIDs and exportProcIDs_ are output arrays which are
1236  // allocated by createFromRecvs().
1237  Array<GO> exportGIDs;
1238 
1239 #ifdef HAVE_TPETRA_MMM_TIMINGS
1240  using Teuchos::TimeMonitor;
1241  std::string label;
1242  if(!plist.is_null())
1243  label = plist->get("Timer Label",label);
1244  std::string prefix2 = std::string("Tpetra ")+ label + std::string(":iport_ctor:setupExport:3 ");
1245  auto MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix2)));
1246 #endif
1247 
1248  if (this->verbose ()) {
1249  std::ostringstream os;
1250  os << *prefix << "Call createFromRecvs" << endl;
1251  this->verboseOutputStream () << endl;
1252  }
1253  this->TransferData_->distributor_.createFromRecvs (remoteGIDsView ().getConst (),
1254  remoteProcIDs, exportGIDs,
1255  this->TransferData_->exportPIDs_);
1256 
1257  // Find the LIDs corresponding to the (outgoing) GIDs in
1258  // exportGIDs. For sparse matrix-vector multiply, this tells the
1259  // calling process how to index into the source vector to get the
1260  // elements which it needs to send.
1261 #ifdef HAVE_TPETRA_MMM_TIMINGS
1262  prefix2 = std::string("Tpetra ")+ label + std::string(":iport_ctor:setupExport:4 ");
1263  MM = Teuchos::null; MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix2)));
1264 #endif
1265 
1266  // NOTE (mfh 03 Mar 2014) This is now a candidate for a
1267  // thread-parallel kernel, but only if using the new thread-safe
1268  // Map implementation.
1269  const size_type numExportIDs = exportGIDs.size ();
1270  if (numExportIDs > 0) {
1271  typename decltype (this->TransferData_->exportLIDs_)::t_host
1272  exportLIDs (view_alloc_no_init ("exportLIDs"), numExportIDs);
1273  ArrayView<const GO> expGIDs = exportGIDs ();
1274 
1275  for (size_type k = 0; k < numExportIDs; ++k) {
1276  exportLIDs[k] = source.getLocalElement (expGIDs[k]);
1277  }
1278  makeDualViewFromOwningHostView (this->TransferData_->exportLIDs_, exportLIDs);
1279  }
1280 
1281  if (this->verbose ()) {
1282  std::ostringstream os;
1283  os << *prefix << "Done!" << endl;
1284  this->verboseOutputStream () << os.str ();
1285  }
1286  }
1287 
1288  template <class LocalOrdinal, class GlobalOrdinal, class Node>
1289  void
1290  Import<LocalOrdinal,GlobalOrdinal,Node>::
1291  findUnionTargetGIDs(Teuchos::Array<GlobalOrdinal>& unionTgtGIDs,
1292  Teuchos::Array<std::pair<int,GlobalOrdinal>>& remotePGIDs,
1293  typename Teuchos::Array<GlobalOrdinal>::size_type& numSameGIDs,
1294  typename Teuchos::Array<GlobalOrdinal>::size_type& numPermuteGIDs,
1295  typename Teuchos::Array<GlobalOrdinal>::size_type& numRemoteGIDs,
1296  const Teuchos::ArrayView<const GlobalOrdinal>& sameGIDs1,
1297  const Teuchos::ArrayView<const GlobalOrdinal>& sameGIDs2,
1298  Teuchos::Array<GlobalOrdinal>& permuteGIDs1,
1299  Teuchos::Array<GlobalOrdinal>& permuteGIDs2,
1300  Teuchos::Array<GlobalOrdinal>& remoteGIDs1,
1301  Teuchos::Array<GlobalOrdinal>& remoteGIDs2,
1302  Teuchos::Array<int>& remotePIDs1,
1303  Teuchos::Array<int>& remotePIDs2) const
1304  {
1305 
1306  typedef GlobalOrdinal GO;
1307  typedef typename Teuchos::Array<GO>::size_type size_type;
1308 
1309  const size_type numSameGIDs1 = sameGIDs1.size();
1310  const size_type numSameGIDs2 = sameGIDs2.size();
1311 
1312  // Sort the permute GIDs
1313  std::sort(permuteGIDs1.begin(), permuteGIDs1.end());
1314  std::sort(permuteGIDs2.begin(), permuteGIDs2.end());
1315 
1316  // Get the union of the two target maps
1317  // Reserve the maximum possible size to guard against reallocations from
1318  // push_back operations.
1319  unionTgtGIDs.reserve(numSameGIDs1 + numSameGIDs2 +
1320  permuteGIDs1.size() + permuteGIDs2.size() +
1321  remoteGIDs1.size() + remoteGIDs2.size());
1322 
1323  // Copy the same GIDs to unionTgtGIDs. Cases for numSameGIDs1 !=
1324  // numSameGIDs2 must be treated separately.
1325  typename Teuchos::Array<GO>::iterator permuteGIDs1_end;
1326  typename Teuchos::Array<GO>::iterator permuteGIDs2_end;
1327  if (numSameGIDs2 > numSameGIDs1) {
1328 
1329  numSameGIDs = numSameGIDs2;
1330  permuteGIDs2_end = permuteGIDs2.end();
1331 
1332  // Copy the same GIDs from tgtGIDs to the union
1333  std::copy(sameGIDs2.begin(), sameGIDs2.end(), std::back_inserter(unionTgtGIDs));
1334 
1335  // Remove GIDs from permuteGIDs1 that have already been copied in to unionTgtGIDs
1336  // set_difference allows the last (output) argument to alias the first.
1337  permuteGIDs1_end = std::set_difference(permuteGIDs1.begin(), permuteGIDs1.end(),
1338  unionTgtGIDs.begin()+numSameGIDs1, unionTgtGIDs.end(),
1339  permuteGIDs1.begin());
1340 
1341  } else {
1342 
1343  numSameGIDs = numSameGIDs1;
1344  permuteGIDs1_end = permuteGIDs1.end();
1345 
1346  // Copy the same GIDs from tgtGIDs to the union
1347  std::copy(sameGIDs1.begin(), sameGIDs1.end(), std::back_inserter(unionTgtGIDs));
1348 
1349  // Remove GIDs from permuteGIDs2 that have already been copied in to unionTgtGIDs
1350  // set_difference allows the last (output) argument to alias the first.
1351  permuteGIDs2_end = std::set_difference(permuteGIDs2.begin(), permuteGIDs2.end(),
1352  unionTgtGIDs.begin()+numSameGIDs2, unionTgtGIDs.end(),
1353  permuteGIDs2.begin());
1354 
1355  }
1356 
1357  // Get the union of the permute GIDs and push it back on unionTgtGIDs
1358  std::set_union(permuteGIDs1.begin(), permuteGIDs1_end,
1359  permuteGIDs2.begin(), permuteGIDs2_end,
1360  std::back_inserter(unionTgtGIDs));
1361 
1362  // Sort the PID,GID pairs and find the unique set
1363  Teuchos::Array<std::pair<int,GO>> remotePGIDs1(remoteGIDs1.size());
1364  for (size_type k=0; k<remoteGIDs1.size(); k++)
1365  remotePGIDs1[k] = std::make_pair(remotePIDs1[k], remoteGIDs1[k]);
1366  std::sort(remotePGIDs1.begin(), remotePGIDs1.end());
1367 
1368  Teuchos::Array<std::pair<int,GO>> remotePGIDs2(remoteGIDs2.size());
1369  for (size_type k=0; k<remoteGIDs2.size(); k++)
1370  remotePGIDs2[k] = std::make_pair(remotePIDs2[k], remoteGIDs2[k]);
1371  std::sort(remotePGIDs2.begin(), remotePGIDs2.end());
1372 
1373  remotePGIDs.reserve(remotePGIDs1.size()+remotePGIDs2.size());
1374  std::merge(remotePGIDs1.begin(), remotePGIDs1.end(),
1375  remotePGIDs2.begin(), remotePGIDs2.end(),
1376  std::back_inserter(remotePGIDs));
1377  auto it = std::unique(remotePGIDs.begin(), remotePGIDs.end());
1378  remotePGIDs.resize(std::distance(remotePGIDs.begin(), it));
1379 
1380  // Finally, insert remote GIDs
1381  const size_type oldSize = unionTgtGIDs.size();
1382  unionTgtGIDs.resize(oldSize+remotePGIDs.size());
1383  for (size_type start=oldSize, k=0; k<remotePGIDs.size(); k++)
1384  unionTgtGIDs[start+k] = remotePGIDs[k].second;
1385 
1386  // Compute output only quantities
1387  numRemoteGIDs = remotePGIDs.size();
1388  numPermuteGIDs = unionTgtGIDs.size() - numSameGIDs - numRemoteGIDs;
1389 
1390  return;
1391  }
1392 
1393  template <class LocalOrdinal, class GlobalOrdinal, class Node>
1394  Teuchos::RCP<const Import<LocalOrdinal, GlobalOrdinal, Node> >
1395  Import<LocalOrdinal,GlobalOrdinal,Node>::
1396  setUnion (const Import<LocalOrdinal, GlobalOrdinal, Node>& rhs) const
1397  {
1398  using ::Tpetra::Details::Behavior;
1399  using Teuchos::Array;
1400  using Teuchos::ArrayView;
1401  using Teuchos::as;
1402  using Teuchos::Comm;
1403  using Teuchos::RCP;
1404  using Teuchos::rcp;
1405  using Teuchos::outArg;
1406  using Teuchos::REDUCE_MIN;
1407  using Teuchos::reduceAll;
1408  using GST = Tpetra::global_size_t;
1409  using LO = LocalOrdinal;
1410  using GO = GlobalOrdinal;
1411  using import_type = Import<LO, GO, Node>;
1412  using size_type = typename Array<GO>::size_type;
1413 
1414 #ifdef HAVE_TPETRA_MMM_TIMINGS
1415  using Teuchos::TimeMonitor;
1416  std::string label = std::string("Tpetra::Import::setUnion");
1417  TimeMonitor MM(*TimeMonitor::getNewTimer(label));
1418 #endif
1419 
1420  RCP<const map_type> srcMap = this->getSourceMap ();
1421  RCP<const map_type> tgtMap1 = this->getTargetMap ();
1422  RCP<const map_type> tgtMap2 = rhs.getTargetMap ();
1423  RCP<const Comm<int> > comm = srcMap->getComm ();
1424 
1425  const bool debug = Behavior::debug ("Import::setUnion") ||
1426  Behavior::debug ("Tpetra::Import::setUnion");
1427 
1428  if (debug) {
1429  TEUCHOS_TEST_FOR_EXCEPTION
1430  (! srcMap->isSameAs (* (rhs.getSourceMap ())), std::invalid_argument,
1431  "Tpetra::Import::setUnion: The source Map of the input Import must be the "
1432  "same as (in the sense of Map::isSameAs) the source Map of this Import.");
1433  const Comm<int>& comm1 = * (tgtMap1->getComm ());
1434  const Comm<int>& comm2 = * (tgtMap2->getComm ());
1435  TEUCHOS_TEST_FOR_EXCEPTION
1436  (! ::Tpetra::Details::congruent (comm1, comm2),
1437  std::invalid_argument, "Tpetra::Import::setUnion: "
1438  "The target Maps must have congruent communicators.");
1439  }
1440 
1441  // It's probably worth the one all-reduce to check whether the two
1442  // Maps are the same. If so, we can just return a copy of *this.
1443  // isSameAs() bypasses the all-reduce if the pointers are equal.
1444  if (tgtMap1->isSameAs (*tgtMap2)) {
1445  return rcp (new import_type (*this));
1446  }
1447 
1448  // Alas, the two target Maps are not the same. That means we have
1449  // to compute their union, and the union Import object.
1450 
1451  // Get the same GIDs (same GIDs are a subview of the first numSame target
1452  // GIDs)
1453  const size_type numSameGIDs1 = this->getNumSameIDs();
1454  ArrayView<const GO> sameGIDs1 = (tgtMap1->getLocalElementList())(0,numSameGIDs1);
1455 
1456  const size_type numSameGIDs2 = rhs.getNumSameIDs();
1457  ArrayView<const GO> sameGIDs2 = (tgtMap2->getLocalElementList())(0,numSameGIDs2);
1458 
1459  // Get permute GIDs
1460  ArrayView<const LO> permuteToLIDs1 = this->getPermuteToLIDs();
1461  Array<GO> permuteGIDs1(permuteToLIDs1.size());
1462  for (size_type k=0; k<permuteGIDs1.size(); k++)
1463  permuteGIDs1[k] = tgtMap1->getGlobalElement(permuteToLIDs1[k]);
1464 
1465  ArrayView<const LO> permuteToLIDs2 = rhs.getPermuteToLIDs();
1466  Array<GO> permuteGIDs2(permuteToLIDs2.size());
1467  for (size_type k=0; k<permuteGIDs2.size(); k++)
1468  permuteGIDs2[k] = tgtMap2->getGlobalElement(permuteToLIDs2[k]);
1469 
1470  // Get remote GIDs
1471  ArrayView<const LO> remoteLIDs1 = this->getRemoteLIDs();
1472  Array<GO> remoteGIDs1(remoteLIDs1.size());
1473  for (size_type k=0; k<remoteLIDs1.size(); k++)
1474  remoteGIDs1[k] = this->getTargetMap()->getGlobalElement(remoteLIDs1[k]);
1475 
1476  ArrayView<const LO> remoteLIDs2 = rhs.getRemoteLIDs();
1477  Array<GO> remoteGIDs2(remoteLIDs2.size());
1478  for (size_type k=0; k<remoteLIDs2.size(); k++)
1479  remoteGIDs2[k] = rhs.getTargetMap()->getGlobalElement(remoteLIDs2[k]);
1480 
1481  // Get remote PIDs
1482  Array<int> remotePIDs1;
1483  Tpetra::Import_Util::getRemotePIDs(*this, remotePIDs1);
1484 
1485  Array<int> remotePIDs2;
1486  Tpetra::Import_Util::getRemotePIDs(rhs, remotePIDs2);
1487 
1488  // Get the union of the target GIDs
1489  Array<GO> unionTgtGIDs;
1490  Array<std::pair<int,GO>> remotePGIDs;
1491  size_type numSameIDsUnion, numPermuteIDsUnion, numRemoteIDsUnion;
1492 
1493  findUnionTargetGIDs(unionTgtGIDs, remotePGIDs,
1494  numSameIDsUnion, numPermuteIDsUnion, numRemoteIDsUnion,
1495  sameGIDs1, sameGIDs2, permuteGIDs1, permuteGIDs2,
1496  remoteGIDs1, remoteGIDs2, remotePIDs1, remotePIDs2);
1497 
1498  // Extract GIDs and compute LIDS, PIDs for the remotes in the union
1499  Array<LO> remoteLIDsUnion(numRemoteIDsUnion);
1500  Array<GO> remoteGIDsUnion(numRemoteIDsUnion);
1501  Array<int> remotePIDsUnion(numRemoteIDsUnion);
1502  const size_type unionRemoteIDsStart = numSameIDsUnion + numPermuteIDsUnion;
1503  for (size_type k = 0; k < numRemoteIDsUnion; ++k) {
1504  remoteLIDsUnion[k] = unionRemoteIDsStart + k;
1505  remotePIDsUnion[k] = remotePGIDs[k].first;
1506  remoteGIDsUnion[k] = remotePGIDs[k].second;
1507  }
1508 
1509  // Compute the permute-to LIDs (in the union target Map).
1510  // Convert the permute GIDs to permute-from LIDs in the source Map.
1511  Array<LO> permuteToLIDsUnion(numPermuteIDsUnion);
1512  Array<LO> permuteFromLIDsUnion(numPermuteIDsUnion);
1513 
1514  for (size_type k = 0; k < numPermuteIDsUnion; ++k) {
1515  size_type idx = numSameIDsUnion + k;
1516  permuteToLIDsUnion[k] = static_cast<LO>(idx);
1517  permuteFromLIDsUnion[k] = srcMap->getLocalElement(unionTgtGIDs[idx]);
1518  }
1519 
1520 #ifdef HAVE_TPETRA_MMM_TIMINGS
1521  MM.disableTimer(label);
1522  label = "Tpetra::Import::setUnion : Construct Target Map";
1523  TimeMonitor MM2(*TimeMonitor::getNewTimer(label));
1524 #endif
1525 
1526  // Create the union target Map.
1527  const GST INVALID = Teuchos::OrdinalTraits<GST>::invalid ();
1528  const GO indexBaseUnion = std::min(tgtMap1->getIndexBase(), tgtMap2->getIndexBase());
1529  RCP<const map_type> unionTgtMap =
1530  rcp(new map_type(INVALID, unionTgtGIDs(), indexBaseUnion, comm));
1531 
1532 #ifdef HAVE_TPETRA_MMM_TIMINGS
1533  MM2.disableTimer(label);
1534  label = "Tpetra::Import::setUnion : Export GIDs";
1535  TimeMonitor MM3(*TimeMonitor::getNewTimer(label));
1536 #endif
1537 
1538  // Thus far, we have computed the following in the union Import:
1539  // - numSameIDs
1540  // - numPermuteIDs and permuteFromLIDs
1541  // - numRemoteIDs, remoteGIDs, remoteLIDs, and remotePIDs
1542  //
1543  // Now it's time to compute the export IDs and initialize the
1544  // Distributor.
1545 
1546  Array<GO> exportGIDsUnion;
1547  Array<LO> exportLIDsUnion;
1548  Array<int> exportPIDsUnion;
1549  Distributor distributor (comm, this->TransferData_->out_);
1550 
1551 #ifdef TPETRA_IMPORT_SETUNION_USE_CREATE_FROM_SENDS
1552  // Compute the export IDs without communication, by merging the
1553  // lists of (export LID, export PID) pairs from the two input
1554  // Import objects. The export LIDs in both input Import objects
1555  // are LIDs in the source Map. Then, use the export PIDs to
1556  // initialize the Distributor via createFromSends.
1557 
1558  // const size_type numExportIDs1 = this->getNumExportIDs ();
1559  ArrayView<const LO> exportLIDs1 = this->getExportLIDs ();
1560  ArrayView<const LO> exportPIDs1 = this->getExportPIDs ();
1561 
1562  // const size_type numExportIDs2 = rhs.getNumExportIDs ();
1563  ArrayView<const LO> exportLIDs2 = rhs.getExportLIDs ();
1564  ArrayView<const LO> exportPIDs2 = rhs.getExportPIDs ();
1565 
1566  // We have to keep the export LIDs in PID-sorted order, then merge
1567  // them. So, first key-value merge (LID,PID) pairs, treating PIDs
1568  // as values, merging values by replacement. Then, sort the
1569  // (LID,PID) pairs again by PID.
1570 
1571  // Sort (LID,PID) pairs by LID for the later merge, and make
1572  // each sequence unique by LID.
1573  Array<LO> exportLIDs1Copy (exportLIDs1.begin (), exportLIDs1.end ());
1574  Array<int> exportPIDs1Copy (exportLIDs1.begin (), exportLIDs1.end ());
1575  sort2 (exportLIDs1Copy.begin (), exportLIDs1Copy.end (),
1576  exportPIDs1Copy.begin ());
1577  typename ArrayView<LO>::iterator exportLIDs1_end = exportLIDs1Copy.end ();
1578  typename ArrayView<LO>::iterator exportPIDs1_end = exportPIDs1Copy.end ();
1579  merge2 (exportLIDs1_end, exportPIDs1_end,
1580  exportLIDs1Copy.begin (), exportLIDs1_end,
1581  exportPIDs1Copy.begin (), exportPIDs1_end,
1582  project1st<LO, LO> ());
1583 
1584  Array<LO> exportLIDs2Copy (exportLIDs2.begin (), exportLIDs2.end ());
1585  Array<int> exportPIDs2Copy (exportLIDs2.begin (), exportLIDs2.end ());
1586  sort2 (exportLIDs2Copy.begin (), exportLIDs2Copy.end (),
1587  exportPIDs2Copy.begin ());
1588  typename ArrayView<LO>::iterator exportLIDs2_end = exportLIDs2Copy.end ();
1589  typename ArrayView<LO>::iterator exportPIDs2_end = exportPIDs2Copy.end ();
1590  merge2 (exportLIDs2_end, exportPIDs2_end,
1591  exportLIDs2Copy.begin (), exportLIDs2_end,
1592  exportPIDs2Copy.begin (), exportPIDs2_end,
1593  project1st<LO, LO> ());
1594 
1595  // Merge export (LID,PID) pairs. In this merge operation, the
1596  // LIDs are the "keys" and the PIDs their "values." We combine
1597  // the "values" (PIDs) in the pairs by replacement, rather than
1598  // by adding them together.
1599  keyValueMerge (exportLIDs1Copy.begin (), exportLIDs1Copy.end (),
1600  exportPIDs1Copy.begin (), exportPIDs1Copy.end (),
1601  exportLIDs2Copy.begin (), exportLIDs2Copy.end (),
1602  exportPIDs2Copy.begin (), exportPIDs2Copy.end (),
1603  std::back_inserter (exportLIDsUnion),
1604  std::back_inserter (exportPIDsUnion),
1605  project1st<int, int> ());
1606 
1607  // Resort the merged (LID,PID) pairs by PID.
1608  sort2 (exportPIDsUnion.begin (), exportPIDsUnion.end (),
1609  exportLIDsUnion.begin ());
1610 
1611  // Initialize the Distributor. Using createFromSends instead of
1612  // createFromRecvs avoids the initialization and use of a
1613  // temporary Distributor object.
1614  (void) distributor.createFromSends (exportPIDsUnion ().getConst ());
1615 #else // NOT TPETRA_IMPORT_SETUNION_USE_CREATE_FROM_SENDS
1616 
1617  // Call the Distributor's createFromRecvs() method to turn the
1618  // remote GIDs and their owning processes into a send-and-receive
1619  // communication plan. remoteGIDsUnion and remotePIDsUnion are
1620  // input; exportGIDsUnion and exportPIDsUnion are output arrays
1621  // which are allocated by createFromRecvs().
1622  distributor.createFromRecvs (remoteGIDsUnion().getConst(),
1623  remotePIDsUnion().getConst(),
1624  exportGIDsUnion, exportPIDsUnion);
1625 
1626  // Find the (source Map) LIDs corresponding to the export GIDs.
1627  const size_type numExportIDsUnion = exportGIDsUnion.size ();
1628  exportLIDsUnion.resize (numExportIDsUnion);
1629  for (size_type k = 0; k < numExportIDsUnion; ++k) {
1630  exportLIDsUnion[k] = srcMap->getLocalElement (exportGIDsUnion[k]);
1631  }
1632 #endif // TPETRA_IMPORT_SETUNION_USE_CREATE_FROM_SENDS
1633 
1634  // Create and return the union Import. This uses the "expert" constructor
1635 #ifdef HAVE_TPETRA_MMM_TIMINGS
1636  MM3.disableTimer(label);
1637  label = "Tpetra::Import::setUnion : Construct Import";
1638  TimeMonitor MM4(*TimeMonitor::getNewTimer(label));
1639 #endif
1640  RCP<const import_type> unionImport =
1641  rcp (new import_type (srcMap, unionTgtMap,
1642  as<size_t> (numSameIDsUnion),
1643  permuteToLIDsUnion, permuteFromLIDsUnion,
1644  remoteLIDsUnion, exportLIDsUnion,
1645  exportPIDsUnion, distributor,
1646  this->TransferData_->out_));
1647  return unionImport;
1648  }
1649 
1650  template <class LocalOrdinal, class GlobalOrdinal, class Node>
1651  Teuchos::RCP<const Import<LocalOrdinal, GlobalOrdinal, Node> >
1652  Import<LocalOrdinal,GlobalOrdinal,Node>::
1653  setUnion () const
1654  {
1655  using Teuchos::Array;
1656  using Teuchos::ArrayView;
1657  using Teuchos::as;
1658  using Teuchos::Comm;
1659  using Teuchos::RCP;
1660  using Teuchos::rcp;
1661  using Teuchos::outArg;
1662  using Teuchos::REDUCE_MIN;
1663  using Teuchos::reduceAll;
1664  typedef LocalOrdinal LO;
1665  typedef GlobalOrdinal GO;
1666  Teuchos::RCP<const Import<LocalOrdinal, GlobalOrdinal, Node> > unionImport;
1667  RCP<const map_type> srcMap = this->getSourceMap ();
1668  RCP<const map_type> tgtMap = this->getTargetMap ();
1669  RCP<const Comm<int> > comm = srcMap->getComm ();
1670 
1671  ArrayView<const GO> srcGIDs = srcMap->getLocalElementList ();
1672  ArrayView<const GO> tgtGIDs = tgtMap->getLocalElementList ();
1673 
1674  // All elements in srcMap will be in the "new" target map, so...
1675  size_t numSameIDsNew = srcMap->getLocalNumElements ();
1676  size_t numRemoteIDsNew = this->getNumRemoteIDs ();
1677  Array<LO> permuteToLIDsNew, permuteFromLIDsNew; // empty on purpose
1678 
1679  // Grab some old data
1680  ArrayView<const LO> remoteLIDsOld = this->getRemoteLIDs ();
1681  ArrayView<const LO> exportLIDsOld = this->getExportLIDs ();
1682 
1683  // Build up the new map (same part)
1684  Array<GO> GIDs(numSameIDsNew + numRemoteIDsNew);
1685  for(size_t i=0; i<numSameIDsNew; i++)
1686  GIDs[i] = srcGIDs[i];
1687 
1688  // Build up the new map (remote part) and remotes list
1689  Array<LO> remoteLIDsNew(numRemoteIDsNew);
1690  for(size_t i=0; i<numRemoteIDsNew; i++) {
1691  GIDs[numSameIDsNew + i] = tgtGIDs[remoteLIDsOld[i]];
1692  remoteLIDsNew[i] = numSameIDsNew+i;
1693  }
1694 
1695  // Build the new target map
1696  GO GO_INVALID = Teuchos::OrdinalTraits<GO>::invalid();
1697  RCP<const map_type> targetMapNew =
1698  rcp (new map_type (GO_INVALID, GIDs, tgtMap->getIndexBase (),
1699  tgtMap->getComm ()));
1700 
1701  // Exports are trivial (since the sourcemap doesn't change)
1702  Array<int> exportPIDsnew (this->getExportPIDs ());
1703  Array<LO> exportLIDsnew (this->getExportLIDs ());
1704 
1705  // Copy the Distributor (due to how the Import constructor works)
1706  Distributor D (this->getDistributor ());
1707 
1708  // Build the importer using the "expert" constructor
1709  unionImport = rcp(new Import<LocalOrdinal, GlobalOrdinal, Node>(srcMap,
1710  targetMapNew,
1711  numSameIDsNew,
1712  permuteToLIDsNew,
1713  permuteFromLIDsNew,
1714  remoteLIDsNew,
1715  exportLIDsnew,
1716  exportPIDsnew,D));
1717 
1718  return unionImport;
1719  }
1720 
1721  template <class LocalOrdinal, class GlobalOrdinal, class Node>
1722  Teuchos::RCP<const Import<LocalOrdinal, GlobalOrdinal, Node> >
1723  Import<LocalOrdinal,GlobalOrdinal,Node>::
1724  createRemoteOnlyImport (const Teuchos::RCP<const map_type>& remoteTarget) const
1725  {
1726  using ::Tpetra::Details::Behavior;
1727  using ::Tpetra::Details::gathervPrint;
1728  using Teuchos::outArg;
1729  using Teuchos::rcp;
1730  using Teuchos::REDUCE_MIN;
1731  using Teuchos::reduceAll;
1732  using std::endl;
1733  using LO = LocalOrdinal;
1734  using GO = GlobalOrdinal;
1735  using import_type = Import<LocalOrdinal,GlobalOrdinal,Node>;
1736 
1737  const char funcPrefix[] = "Tpetra::createRemoteOnlyImport: ";
1738  int lclSuccess = 1;
1739  int gblSuccess = 1;
1740  const bool debug = Behavior::debug ();
1741 
1742  const size_t NumRemotes = this->getNumRemoteIDs ();
1743  std::unique_ptr<std::string> procPrefix;
1744  Teuchos::RCP<const Teuchos::Comm<int>> comm;
1745  if (debug) {
1746  comm = remoteTarget.is_null () ? Teuchos::null :
1747  remoteTarget->getComm ();
1748  std::ostringstream os;
1749  os << "Proc ";
1750  if (comm.is_null ()) {
1751  os << "?";
1752  }
1753  else {
1754  os << comm->getRank ();
1755  }
1756  os << ": ";
1757  procPrefix = std::unique_ptr<std::string> (new std::string (os.str ()));
1758  }
1759 
1760  if (debug) {
1761  std::ostringstream lclErr;
1762  if (remoteTarget.is_null ()) {
1763  lclSuccess = -1;
1764  }
1765  else if (NumRemotes != remoteTarget->getLocalNumElements ()) {
1766  lclSuccess = 0;
1767  lclErr << *procPrefix << "getNumRemoteIDs() = " << NumRemotes
1768  << " != remoteTarget->getLocalNumElements() = "
1769  << remoteTarget->getLocalNumElements () << "." << endl;
1770  }
1771 
1772  if (comm.is_null ()) {
1773  lclSuccess = gblSuccess;
1774  }
1775  else {
1776  reduceAll (*comm, REDUCE_MIN, lclSuccess, outArg (gblSuccess));
1777  }
1778  TEUCHOS_TEST_FOR_EXCEPTION
1779  (gblSuccess == -1, std::invalid_argument, funcPrefix
1780  << "Input target Map is null on at least one process.");
1781 
1782  if (gblSuccess != 1) {
1783  if (comm.is_null ()) {
1784  TEUCHOS_TEST_FOR_EXCEPTION
1785  (true, std::runtime_error, lclErr.str ());
1786  }
1787  else {
1788  std::ostringstream gblErr;
1789  gblErr << funcPrefix << endl;
1790  gathervPrint (gblErr, lclErr.str (), *comm);
1791  TEUCHOS_TEST_FOR_EXCEPTION
1792  (true, std::runtime_error, gblErr.str ());
1793  }
1794  }
1795  }
1796 
1797  // Compute the new Remote LIDs
1798  Teuchos::ArrayView<const LO> oldRemoteLIDs = this->getRemoteLIDs ();
1799  Teuchos::Array<LO> newRemoteLIDs (NumRemotes);
1800  const map_type& tgtMap = * (this->getTargetMap ());
1801  size_t badCount = 0;
1802 
1803  std::unique_ptr<std::vector<size_t>> badIndices;
1804  if (debug) {
1805  badIndices = std::unique_ptr<std::vector<size_t>> (new std::vector<size_t>);
1806  }
1807 
1808  for (size_t i = 0; i < NumRemotes; ++i) {
1809  const LO oldLclInd = oldRemoteLIDs[i];
1810  if (oldLclInd == Teuchos::OrdinalTraits<LO>::invalid ()) {
1811  ++badCount;
1812  if (debug) { badIndices->push_back (i); }
1813  continue;
1814  }
1815  const GO gblInd = tgtMap.getGlobalElement (oldLclInd);
1816  if (gblInd == Teuchos::OrdinalTraits<GO>::invalid ()) {
1817  ++badCount;
1818  if (debug) { badIndices->push_back (i); }
1819  continue;
1820  }
1821  const LO newLclInd = remoteTarget->getLocalElement (gblInd);
1822  if (newLclInd == Teuchos::OrdinalTraits<LO>::invalid ()) {
1823  ++badCount;
1824  if (debug) { badIndices->push_back (i); }
1825  continue;
1826  }
1827  newRemoteLIDs[i] = newLclInd;
1828  // Now we make sure these guys are in sorted order (AztecOO-ML ordering)
1829  if (i > 0 && newRemoteLIDs[i] < newRemoteLIDs[i-1]) {
1830  ++badCount;
1831  if (debug) { badIndices->push_back (i); }
1832  }
1833  }
1834 
1835  if (badCount != 0) {
1836  lclSuccess = 0;
1837  }
1838 
1839  if (debug) {
1840  if (comm.is_null ()) {
1841  lclSuccess = gblSuccess;
1842  }
1843  else {
1844  reduceAll (*comm, REDUCE_MIN, lclSuccess, outArg (gblSuccess));
1845  }
1846  std::ostringstream lclErr;
1847  if (lclSuccess != 1) {
1848  lclErr << *procPrefix << "Count of bad indices: " << badCount
1849  << ", bad indices: [";
1850  // TODO (mfh 04 Sep 2019) Limit the maximum number of bad
1851  // indices to print.
1852  for (size_t k = 0; k < badCount; ++k) {
1853  const size_t badIndex = (*badIndices)[k];
1854  lclErr << "(" << badIndex << ","
1855  << oldRemoteLIDs[badIndex] << ")";
1856  if (k + size_t (1) < badCount) {
1857  lclErr << ", ";
1858  }
1859  }
1860  lclErr << "]" << endl;
1861  }
1862 
1863  if (gblSuccess != 1) {
1864  std::ostringstream gblErr;
1865  gblErr << funcPrefix << "this->getRemoteLIDs() has \"bad\" "
1866  "indices on one or more processes. \"Bad\" means that the "
1867  "indices are invalid, they don't exist in the target Map, "
1868  "they don't exist in remoteTarget, or they are not in "
1869  "sorted order. In what follows, I will show the \"bad\" "
1870  "indices as (k, LID) pairs, where k is the zero-based "
1871  "index of the LID in this->getRemoteLIDs()." << endl;
1872  if (comm.is_null ()) {
1873  gblErr << lclErr.str ();
1874  }
1875  else {
1876  gathervPrint (gblErr, lclErr.str (), *comm);
1877  }
1878  TEUCHOS_TEST_FOR_EXCEPTION
1879  (true, std::runtime_error, gblErr.str ());
1880  }
1881  }
1882  else { // not debug
1883  TEUCHOS_TEST_FOR_EXCEPTION
1884  (lclSuccess == 0, std::runtime_error, funcPrefix
1885  << "this->getRemoteLIDs() has " << badCount
1886  << "ind" << (badCount == 1 ? "ex" : "ices")
1887  << " \"bad\" indices on this process." << endl);
1888  }
1889 
1890  // Copy ExportPIDs and such
1891  // NOTE: Be careful: The "Expert" Import constructor we use does a "swap"
1892  // for most of the LID/PID lists and the Distributor, meaning it
1893  // ruins the existing object if we pass things in directly. Hence
1894  // we copy them first.
1895  Teuchos::Array<int> newExportPIDs (this->getExportPIDs ());
1896  Teuchos::Array<LO> newExportLIDs (this->getExportLIDs ());
1897  Teuchos::Array<LO> dummy;
1898  Distributor newDistor (this->getDistributor ());
1899 
1900  return rcp (new import_type (this->getSourceMap (), remoteTarget,
1901  static_cast<size_t> (0), dummy, dummy,
1902  newRemoteLIDs, newExportLIDs,
1903  newExportPIDs, newDistor));
1904  }
1905 
1906 } // namespace Tpetra
1907 
1908 #define TPETRA_IMPORT_CLASS_INSTANT(LO, GO, NODE) \
1909  template class Import< LO , GO , NODE >;
1910 
1911 // Explicit instantiation macro.
1912 // Only invoke this when in the Tpetra namespace.
1913 // Most users do not need to use this.
1914 //
1915 // LO: The local ordinal type.
1916 // GO: The global ordinal type.
1917 // NODE: The Kokkos Node type.
1918 #define TPETRA_IMPORT_INSTANT(LO, GO, NODE) \
1919  TPETRA_IMPORT_CLASS_INSTANT(LO, GO, NODE)
1920 
1921 #endif // TPETRA_IMPORT_DEF_HPP
Tpetra::Details::congruent
bool congruent(const Teuchos::Comm< int > &comm1, const Teuchos::Comm< int > &comm2)
Whether the two communicators are congruent.
Definition: Tpetra_Util.cpp:34
Tpetra::Import
Communication plan for data redistribution from a uniquely-owned to a (possibly) multiply-owned distr...
Definition: Tpetra_Import_decl.hpp:77
Tpetra::merge2
void merge2(IT1 &indResultOut, IT2 &valResultOut, IT1 indBeg, IT1 indEnd, IT2 valBeg, IT2)
Merge values in place, additively, with the same index.
Definition: Tpetra_Util.hpp:729
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::verboseOutputStream
Teuchos::FancyOStream & verboseOutputStream() const
Valid (nonnull) output stream for verbose output.
Tpetra::Import::createRemoteOnlyImport
Teuchos::RCP< const Import< LocalOrdinal, GlobalOrdinal, Node > > createRemoteOnlyImport(const Teuchos::RCP< const map_type > &remoteTarget) const
Returns an importer that contains only the remote entries of this.
Definition: Tpetra_Import_def.hpp:1724
Tpetra_Details_Profiling.hpp
Declaration of Tpetra::Details::Profiling, a scope guard for Kokkos Profiling.
Tpetra::LookupStatus
LookupStatus
Return status of Map remote index lookup (getRemoteIndexList()).
Definition: Tpetra_ConfigDefs.hpp:86
Tpetra_Details_gathervPrint.hpp
Declaration of a function that prints strings from each process.
Tpetra::sort3
void sort3(const IT1 &first1, const IT1 &last1, const IT2 &first2, const IT3 &first3, const bool stableSort=false)
Sort the first array, and apply the same permutation to the second and third arrays.
Definition: Tpetra_Util.hpp:659
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getExportPIDs
Teuchos::ArrayView< const int > getExportPIDs() const
List of processes to which entries will be sent.
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getPermuteToLIDs
Teuchos::ArrayView< const LocalOrdinal > getPermuteToLIDs() const
List of local IDs in the target Map that are permuted.
Tpetra::Import::describe
virtual void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
Definition: Tpetra_Import_def.hpp:908
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getTargetMap
Teuchos::RCP< const map_type > getTargetMap() const
The target Map used to construct this Export or Import.
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getNumSameIDs
size_t getNumSameIDs() const
Number of initial identical IDs.
Tpetra::sort
void sort(View &view, const size_t &size)
Convenience wrapper for std::sort for host-accessible views.
Definition: Tpetra_Util.hpp:614
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getNumRemoteIDs
size_t getNumRemoteIDs() const
Number of entries not on the calling process.
Tpetra::Distributor::createFromSends
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
Definition: Tpetra_Distributor.cpp:421
Tpetra::Import::setUnion
Teuchos::RCP< const Import< LocalOrdinal, GlobalOrdinal, Node > > setUnion() const
Return the union of this Import this-&gt;getSourceMap()
Definition: Tpetra_Import_def.hpp:1653
Tpetra::global_size_t
size_t global_size_t
Global size_t object.
Definition: Tpetra_ConfigDefs.hpp:71
Tpetra::Import::Import
Import(const Teuchos::RCP< const map_type > &source, const Teuchos::RCP< const map_type > &target)
Construct an Import from the source and target Maps.
Definition: Tpetra_Import_def.hpp:158
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getRemoteLIDs
Teuchos::ArrayView< const LocalOrdinal > getRemoteLIDs() const
List of entries in the target Map to receive from other processes.
Tpetra::Map::getGlobalElement
global_ordinal_type getGlobalElement(local_ordinal_type localIndex) const
The global index corresponding to the given local index.
Definition: Tpetra_Map_def.hpp:1259
Tpetra::keyValueMerge
void keyValueMerge(KeyInputIterType keyBeg1, KeyInputIterType keyEnd1, ValueInputIterType valBeg1, ValueInputIterType valEnd1, KeyInputIterType keyBeg2, KeyInputIterType keyEnd2, ValueInputIterType valBeg2, ValueInputIterType valEnd2, KeyOutputIterType keyOut, ValueOutputIterType valOut, BinaryFunction f)
Merge two sorted (by keys) sequences of unique (key,value) pairs by combining pairs with equal keys...
Definition: Tpetra_Util.hpp:886
Tpetra::Distributor::createFromSendsAndRecvs
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
Definition: Tpetra_Distributor.cpp:428
Tpetra::Details::view_alloc_no_init
auto view_alloc_no_init(const std::string &label) ->
Use in place of the string label as the first argument of Kokkos::View&#39;s constructor, in case you want to allocate without initializing.
Definition: Tpetra_Details_DualViewUtil.cpp:16
Tpetra::Export
Communication plan for data redistribution from a (possibly) multiply-owned to a uniquely-owned distr...
Definition: Tpetra_Export_decl.hpp:85
Tpetra::sort2
void sort2(const IT1 &first1, const IT1 &last1, const IT2 &first2, const bool stableSort=false)
Sort the first array, and apply the resulting permutation to the second array.
Definition: Tpetra_Util.hpp:557
TPETRA_ABUSE_WARNING
#define TPETRA_ABUSE_WARNING(throw_exception_test, Exception, msg)
Handle an abuse warning, according to HAVE_TPETRA_THROW_ABUSE_WARNINGS and HAVE_TPETRA_PRINT_ABUSE_WA...
Definition: Tpetra_Util.hpp:119
Tpetra::ImportExportData
Implementation detail of Import and Export.
Definition: Tpetra_ImportExportData_decl.hpp:51
Tpetra::Distributor
Sets up and executes a communication plan for a Tpetra DistObject.
Definition: Tpetra_Distributor.hpp:103
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::TransferData_
Teuchos::RCP< ImportExportData< LocalOrdinal, GlobalOrdinal, Node > > TransferData_
All the data needed for executing the Export communication plan.
Definition: Tpetra_Details_Transfer_decl.hpp:239
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getSourceMap
Teuchos::RCP< const map_type > getSourceMap() const
The source Map used to construct this Export or Import.
Tpetra::Import::findUnionTargetGIDs
void findUnionTargetGIDs(Teuchos::Array< GlobalOrdinal > &unionTgtGIDs, Teuchos::Array< std::pair< int, GlobalOrdinal >> &remotePGIDs, typename Teuchos::Array< GlobalOrdinal >::size_type &numSameGIDs, typename Teuchos::Array< GlobalOrdinal >::size_type &numPermuteGIDs, typename Teuchos::Array< GlobalOrdinal >::size_type &numRemoteGIDs, const Teuchos::ArrayView< const GlobalOrdinal > &sameGIDs1, const Teuchos::ArrayView< const GlobalOrdinal > &sameGIDs2, Teuchos::Array< GlobalOrdinal > &permuteGIDs1, Teuchos::Array< GlobalOrdinal > &permuteGIDs2, Teuchos::Array< GlobalOrdinal > &remoteGIDs1, Teuchos::Array< GlobalOrdinal > &remoteGIDs2, Teuchos::Array< int > &remotePIDs1, Teuchos::Array< int > &remotePIDs2) const
Find the union of the target IDs from two Import objects.
Definition: Tpetra_Import_def.hpp:1291
Tpetra::Details::DeepCopyCounter::start
void start()
Start the deep_copy counter.
Definition: Tpetra_Details_KokkosCounter.cpp:45
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getExportLIDs
Teuchos::ArrayView< const LocalOrdinal > getExportLIDs() const
List of entries in the source Map that will be sent to other processes.
Tpetra::Map
A parallel distribution of indices over processes.
Definition: Tpetra_Map_decl.hpp:198
Tpetra_Import_Util.hpp
Internal functions and macros designed for use with Tpetra::Import and Tpetra::Export objects...
Tpetra_Util.hpp
Stand-alone utility functions and macros.
Tpetra::Details::makeDualViewFromOwningHostView
void makeDualViewFromOwningHostView(Kokkos::DualView< ElementType *, DeviceType > &dv, const typename Kokkos::DualView< ElementType *, DeviceType >::t_host &hostView)
Initialize dv such that its host View is hostView.
Definition: Tpetra_Details_DualViewUtil.hpp:39
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::verbose
bool verbose() const
Whether to print verbose debugging output.
Tpetra::Import::print
virtual void print(std::ostream &os) const
Print the Import&#39;s data to the given output stream.
Definition: Tpetra_Import_def.hpp:917
Tpetra::Distributor::createFromRecvs
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
Tpetra::project1st
Binary function that returns its first argument.
Definition: Tpetra_ConfigDefs.hpp:123
Tpetra_Details_Behavior.hpp
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra&#39;s behavior.
Generated on Thu Nov 21 2024 09:19:26 for Tpetra parallel linear algebra by   doxygen 1.8.5