Tpetra parallel linear algebra  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
Tpetra_Details_Transfer_def.hpp
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Tpetra: Templated Linear Algebra Services Package
5 // Copyright (2008) Sandia Corporation
6 //
7 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8 // the U.S. Government retains certain rights in this software.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ************************************************************************
40 // @HEADER
41 
42 #ifndef TPETRA_DETAILS_TRANSFER_DEF_HPP
43 #define TPETRA_DETAILS_TRANSFER_DEF_HPP
44 
45 #include "Tpetra_Details_gathervPrint.hpp"
46 #include "Tpetra_Distributor.hpp"
47 #include "Tpetra_ImportExportData.hpp"
48 #include "Tpetra_Map.hpp"
49 #include "Teuchos_CommHelpers.hpp"
50 #include "Teuchos_TypeNameTraits.hpp"
51 #include <sstream>
52 
53 namespace { // (anonymous)
54 
55  // Assume that dv is sync'd.
56  template<class ElementType, class DeviceType>
57  Teuchos::ArrayView<const ElementType>
58  makeConstArrayViewFromDualView (const Kokkos::DualView<ElementType*, DeviceType>& dv)
59  {
60  TEUCHOS_ASSERT( ! dv.need_sync_host () );
61  auto hostView = dv.view_host ();
62  const auto size = hostView.extent (0);
63  return Teuchos::ArrayView<const ElementType> (size == 0 ? nullptr : hostView.data (), size);
64  }
65 
66 } // namespace (anonymous)
67 
68 namespace Tpetra {
69 namespace Details {
70 
71 template <class LO, class GO, class NT>
72 Transfer<LO, GO, NT>::
73 Transfer (const Teuchos::RCP<const map_type>& source,
74  const Teuchos::RCP<const map_type>& target,
75  const Teuchos::RCP<Teuchos::FancyOStream>& out,
76  const Teuchos::RCP<Teuchos::ParameterList>& plist,
77  const std::string& className) :
78  TransferData_ (new ImportExportData<LO, GO, NT> (source, target, out, plist))
79 {
80  TEUCHOS_ASSERT( ! TransferData_->out_.is_null () );
81  this->setParameterList (plist, className);
82 }
83 
84 template <class LO, class GO, class NT>
86 Transfer (const Transfer<LO, GO, NT>& rhs, reverse_tag)
87 {
88  TEUCHOS_ASSERT( ! (rhs.TransferData_).is_null () );
89  this->TransferData_ = rhs.TransferData_->reverseClone ();
90  TEUCHOS_ASSERT( ! this->TransferData_->out_.is_null () );
91 }
92 
93 template <class LO, class GO, class NT>
94 void
96 setParameterList (const Teuchos::RCP<Teuchos::ParameterList>& plist,
97  const std::string& className)
98 {
99  using ::Tpetra::Details::Behavior;
100 
101  const bool verboseEnv = Behavior::verbose (className.c_str ()) ||
102  Behavior::verbose ((std::string ("Tpetra::") + className).c_str ());
103 
104  bool verboseParam = false;
105  if (! plist.is_null ()) {
106  // FIXME (mfh 03 Feb 2019) Phase out these parameters in favor of
107  // TPETRA_VERBOSE.
108  if (plist->isType<bool> ("Verbose")) {
109  verboseParam = plist->get<bool> ("Verbose");
110  }
111  else if (plist->isType<bool> ("Debug")) { // backwards compat
112  verboseParam = plist->get<bool> ("Debug");
113  }
114  }
115  this->TransferData_->verbose_ = verboseEnv || verboseParam;
116 }
117 
118 template <class LO, class GO, class NT>
119 size_t
121 getNumSameIDs () const {
122  return TransferData_->numSameIDs_;
123 }
124 
125 template <class LO, class GO, class NT>
126 size_t
129  return static_cast<size_t> (TransferData_->permuteFromLIDs_.extent (0));
130 }
131 
132 template <class LO, class GO, class NT>
133 Kokkos::DualView<const LO*, typename Transfer<LO, GO, NT>::device_type>
136  const auto& dv = TransferData_->permuteFromLIDs_;
137  TEUCHOS_TEST_FOR_EXCEPTION
138  (dv.need_sync_device (), std::logic_error,
139  "Tpetra::Details::Transfer::getPermuteFromLIDs_dv: "
140  "DualView needs sync to device" );
141  TEUCHOS_TEST_FOR_EXCEPTION
142  (dv.need_sync_host (), std::logic_error,
143  "Tpetra::Details::Transfer::getPermuteFromLIDs_dv: "
144  "DualView needs sync to host" );
145  return dv;
146 }
147 
148 template <class LO, class GO, class NT>
149 Teuchos::ArrayView<const LO>
152  return makeConstArrayViewFromDualView (TransferData_->permuteFromLIDs_);
153 }
154 
155 template <class LO, class GO, class NT>
156 Kokkos::DualView<const LO*, typename Transfer<LO, GO, NT>::device_type>
159  const auto& dv = TransferData_->permuteToLIDs_;
160  TEUCHOS_TEST_FOR_EXCEPTION
161  (dv.need_sync_device (), std::logic_error,
162  "Tpetra::Details::Transfer::getPermuteToLIDs_dv: "
163  "DualView needs sync to device" );
164  TEUCHOS_TEST_FOR_EXCEPTION
165  (dv.need_sync_host (), std::logic_error,
166  "Tpetra::Details::Transfer::getPermuteToLIDs_dv: "
167  "DualView needs sync to host" );
168  return dv;
169 }
170 
171 template <class LO, class GO, class NT>
172 Teuchos::ArrayView<const LO>
175  return makeConstArrayViewFromDualView (TransferData_->permuteToLIDs_);
176 }
177 
178 template <class LO, class GO, class NT>
179 size_t
181 getNumRemoteIDs () const {
182  return static_cast<size_t> (TransferData_->remoteLIDs_.extent (0));
183 }
184 
185 template <class LO, class GO, class NT>
186 Kokkos::DualView<const LO*, typename Transfer<LO, GO, NT>::device_type>
189  const auto& dv = TransferData_->remoteLIDs_;
190  TEUCHOS_TEST_FOR_EXCEPTION
191  (dv.need_sync_device (), std::logic_error,
192  "Tpetra::Details::Transfer::getRemoteLIDs_dv: "
193  "DualView needs sync to device" );
194  TEUCHOS_TEST_FOR_EXCEPTION
195  (dv.need_sync_host (), std::logic_error,
196  "Tpetra::Details::Transfer::getRemoteLIDs_dv: "
197  "DualView needs sync to host" );
198  return dv;
199 }
200 
201 template <class LO, class GO, class NT>
202 Teuchos::ArrayView<const LO>
204 getRemoteLIDs () const {
205  return makeConstArrayViewFromDualView (TransferData_->remoteLIDs_);
206 }
207 
208 template <class LO, class GO, class NT>
209 size_t
211 getNumExportIDs () const {
212  return static_cast<size_t> (TransferData_->exportLIDs_.extent (0));
213 }
214 
215 template <class LO, class GO, class NT>
216 Kokkos::DualView<const LO*, typename Transfer<LO, GO, NT>::device_type>
219  const auto& dv = TransferData_->exportLIDs_;
220  TEUCHOS_TEST_FOR_EXCEPTION
221  (dv.need_sync_device (), std::logic_error,
222  "Tpetra::Details::Transfer::getExportLIDs_dv: "
223  "DualView needs sync to device" );
224  TEUCHOS_TEST_FOR_EXCEPTION
225  (dv.need_sync_host (), std::logic_error,
226  "Tpetra::Details::Transfer::getExportLIDs_dv: "
227  "DualView needs sync to host" );
228  return dv;
229 }
230 
231 template <class LO, class GO, class NT>
232 Teuchos::ArrayView<const LO>
234 getExportLIDs () const {
235  return makeConstArrayViewFromDualView (TransferData_->exportLIDs_);
236 }
237 
238 template <class LO, class GO, class NT>
239 Teuchos::ArrayView<const int>
241 getExportPIDs () const {
242  return TransferData_->exportPIDs_ ();
243 }
244 
245 template <class LO, class GO, class NT>
246 Teuchos::RCP<const typename Transfer<LO, GO, NT>::map_type>
248 getSourceMap () const {
249  return TransferData_->source_;
250 }
251 
252 template <class LO, class GO, class NT>
253 Teuchos::RCP<const typename Transfer<LO, GO, NT>::map_type>
255 getTargetMap () const {
256  return TransferData_->target_;
257 }
258 
259 template <class LO, class GO, class NT>
262 getDistributor () const {
263  return TransferData_->distributor_;
264 }
265 
266 template <class LO, class GO, class NT>
267 bool
270  return TransferData_->isLocallyComplete_;
271 }
272 
273 template <class LO, class GO, class NT>
274 void
276 describe (Teuchos::FancyOStream& out,
277  const Teuchos::EVerbosityLevel verbLevel) const
278 {
279  this->describeImpl (out, "Tpetra::Details::Transfer", verbLevel);
280 }
281 
282 template<class LO, class GO, class NT>
283 Teuchos::FancyOStream&
286 {
287  Teuchos::FancyOStream* outPtr = TransferData_->out_.getRawPtr ();
288  TEUCHOS_ASSERT( outPtr != nullptr );
289  return *outPtr;
290 }
291 
292 template<class LO, class GO, class NT>
293 bool
295 verbose () const {
296  return TransferData_->verbose_;
297 }
298 
299 template<class LO, class GO, class NT>
300 void
302 describeImpl (Teuchos::FancyOStream& out,
303  const std::string& className,
304  const Teuchos::EVerbosityLevel verbLevel) const
305 {
306  using Teuchos::TypeNameTraits;
307  using Teuchos::VERB_DEFAULT;
308  using Teuchos::VERB_NONE;
309  using Teuchos::VERB_LOW;
310  using std::endl;
311  const Teuchos::EVerbosityLevel vl =
312  (verbLevel == VERB_DEFAULT) ? VERB_LOW : verbLevel;
313 
314  if (vl == VERB_NONE) {
315  return; // don't print anything
316  }
317  // If this Transfer's source Map or Comm is null, then the Transfer
318  // does not participate in collective operations with the other
319  // processes. In that case, it is not even legal to call this
320  // method. The reasonable thing to do in that case is nothing.
321  auto srcMap = this->getSourceMap ();
322  if (srcMap.is_null ()) {
323  return;
324  }
325  auto comm = srcMap->getComm ();
326  if (comm.is_null ()) {
327  return;
328  }
329  if (this->getTargetMap ().is_null () ||
330  this->getTargetMap ()->getComm ().is_null ()) {
331  return;
332  }
333 
334  const int myRank = comm->getRank ();
335  const int numProcs = comm->getSize ();
336 
337  // Only Process 0 should touch the output stream, but this method in
338  // general may need to do communication. Thus, we may need to
339  // preserve the current tab level across multiple "if (myRank == 0)
340  // { ... }" inner scopes. This is why we sometimes create OSTab
341  // instances by pointer, instead of by value.
342  Teuchos::RCP<Teuchos::OSTab> tab0, tab1;
343 
344  if (myRank == 0) {
345  // At every verbosity level but VERB_NONE, Process 0 prints.
346  // By convention, describe() always begins with a tab before
347  // printing.
348  tab0 = Teuchos::rcp (new Teuchos::OSTab (out));
349 
350  out << "\"" << className << "\":" << endl;
351  tab1 = Teuchos::rcp (new Teuchos::OSTab (out));
352 
353  {
354  out << "Template parameters:" << endl;
355  Teuchos::OSTab tab2 (out);
356  out << "LocalOrdinal: " << TypeNameTraits<LO>::name () << endl
357  << "GlobalOrdinal: " << TypeNameTraits<GO>::name () << endl
358  << "Node: " << TypeNameTraits<NT>::name () << endl;
359  }
360 
361  const std::string label = this->getObjectLabel ();
362  if (label != "") {
363  out << "Label: " << label << endl;
364  }
365  out << "Number of processes: " << numProcs << endl;
366  }
367 
368  if (vl > VERB_LOW) {
369  // At higher verbosity levels, describe() is allowed to
370  // communicate in order to print information from other
371  // processes in the object's communicator.
372  this->globalDescribe (out, vl);
373  }
374 
375  // It's illegal to call describe() on a process where either Map is
376  // null. (That implies the process in question is not participating
377  // in collective operations with either Map, and describe is
378  // collective over the Maps' communicator.) Thus, we don't have to
379  // define behavior when either Map is NULL on any process. Thus,
380  // it's OK that the code below isn't quite right (that is, won't
381  // print anything) if either Map is NULL on Process 0.
382 
383  if (myRank == 0) {
384  out << "Source Map:" << endl;
385  }
386  // This is collective over the Map's communicator.
387  this->getSourceMap ()->describe (out, vl);
388 
389  if (myRank == 0) {
390  out << "Target Map:" << endl;
391  }
392  // This is collective over the Map's communicator.
393  this->getTargetMap ()->describe (out, vl);
394 
395  if (myRank == 0) {
396  out << "Distributor:" << endl;
397  }
398  this->getDistributor ().describe (out, vl);
399 }
400 
401 template<class LO, class GO, class NT>
402 void
404 globalDescribe (Teuchos::FancyOStream& out,
405  const Teuchos::EVerbosityLevel vl) const
406 {
407  using Teuchos::Comm;
408  using Teuchos::OSTab;
409  using Teuchos::RCP;
410  using Teuchos::toString;
411  using std::endl;
412 
413  // If this Transfer's source Map or Comm is null, then the Transfer
414  // does not participate in collective operations with the other
415  // processes. In that case, it is not even legal to call this
416  // method. The reasonable thing to do in that case is nothing.
417  auto srcMap = this->getSourceMap ();
418  if (srcMap.is_null ()) {
419  return;
420  }
421  RCP<const Teuchos::Comm<int> > comm = srcMap->getComm ();
422  if (comm.is_null ()) {
423  return;
424  }
425 
426  const std::string myStr = localDescribeToString (vl);
427  ::Tpetra::Details::gathervPrint (out, myStr, *comm);
428 }
429 
430 template<class LO, class GO, class NT>
431 std::string
432 Transfer<LO, GO, NT>::
433 localDescribeToString (const Teuchos::EVerbosityLevel vl) const
434 {
435  using Teuchos::OSTab;
436  using Teuchos::RCP;
437  using std::endl;
438 
439  RCP<std::ostringstream> outString (new std::ostringstream);
440  RCP<Teuchos::FancyOStream> outp = Teuchos::getFancyOStream (outString);
441  Teuchos::FancyOStream& out = *outp; // only valid during this scope
442 
443  RCP<const Teuchos::Comm<int> > comm = this->getSourceMap ()->getComm ();
444  if (this->getSourceMap ().is_null () ||
445  this->getSourceMap ()->getComm ().is_null ()) {
446  // If this Transfer does not participate in the communicator,
447  // it's not even legal to call this method. However, we need to
448  // do something in this case. The reasonable thing to do is not
449  // to print anything.
450  return std::string ("");
451  }
452  else {
453  const int myRank = comm->getRank ();
454  const int numProcs = comm->getSize ();
455 
456  out << "Process " << myRank << " of " << numProcs << ":" << endl;
457  OSTab tab1 (out);
458 
459  out << "numSameIDs: " << getNumSameIDs () << endl;
460  out << "numPermuteIDs: " << getNumPermuteIDs () << endl;
461  out << "numRemoteIDs: " << getNumRemoteIDs () << endl;
462  out << "numExportIDs: " << getNumExportIDs () << endl;
463 
464  // Only print the actual contents of these arrays at the two
465  // highest verbosity levels. Otherwise, just print their counts.
466  if (vl <= Teuchos::VERB_MEDIUM) {
467  out << "permuteFromLIDs count: " << getPermuteFromLIDs ().size () << endl
468  << "permuteToLIDs count: " << getPermuteToLIDs ().size () << endl
469  << "remoteLIDs count: " << getRemoteLIDs ().size () << endl
470  << "exportLIDs count: " << getExportLIDs ().size () << endl
471  << "exportPIDs count: " << getExportPIDs () << endl;
472  }
473  else { // vl = VERB_HIGH or VERB_EXTREME
474  // Build RemoteGIDs
475  RCP<const Map<LO,GO,NT> > tmap = getTargetMap();
476  RCP<const Map<LO,GO,NT> > smap = getSourceMap();
477  Teuchos::Array<GO> RemoteGIDs(getRemoteLIDs().size());
478  Teuchos::Array<int> RemotePIDs(getRemoteLIDs().size());
479  for(size_t i=0; i<(size_t)getRemoteLIDs().size(); i++)
480  RemoteGIDs[i] = tmap->getGlobalElement(getRemoteLIDs()[i]);
481 
482  Teuchos::Array<int> ExportGIDs(getExportLIDs().size());
483  for(size_t i=0; i<(size_t)getExportLIDs().size(); i++)
484  ExportGIDs[i] = smap->getGlobalElement(getExportLIDs()[i]);
485 
486  // Build RemotePIDs (taken from Tpetra_Import_Util.hpp)
487  const Tpetra::Distributor & D=getDistributor();
488  size_t NumReceives = D.getNumReceives();
489  Teuchos::ArrayView<const int> ProcsFrom = D.getProcsFrom();
490  Teuchos::ArrayView<const size_t> LengthsFrom = D.getLengthsFrom();
491  for (size_t i = 0, j = 0; i < NumReceives; ++i) {
492  const int pid = ProcsFrom[i];
493  for (size_t k = 0; k < LengthsFrom[i]; ++k) {
494  RemotePIDs[j] = pid;
495  j++;
496  }
497  }
498 
499  out << "distor.NumRecvs : "<<NumReceives<<endl
500  << "distor.ProcsFrom : "<<toString(ProcsFrom)<<endl
501  << "distor.LengthsFrom: "<<toString(LengthsFrom)<<endl;
502 
503  out << "distor.NumSends : "<<D.getNumSends()<<endl
504  << "distor.ProcsTo : "<<toString(D.getProcsTo())<<endl
505  << "distor.LengthsTo : "<<toString(D.getLengthsTo())<<endl;
506 
507  out << "distor.hasSelfMsg : "<<D.hasSelfMessage()<<endl;
508 
509  out << "permuteFromLIDs: " << toString (getPermuteFromLIDs ()) << endl
510  << "permuteToLIDs: " << toString (getPermuteToLIDs ()) << endl
511  << "remoteLIDs: " << toString (getRemoteLIDs ()) << endl
512  << "remoteGIDs: " << toString (RemoteGIDs ()) << endl
513  << "remotePIDs: " << toString (RemotePIDs ()) << endl
514  << "exportLIDs: " << toString (getExportLIDs ()) << endl
515  << "exportGIDs: " << toString (ExportGIDs ()) << endl
516  << "exportPIDs: " << toString (getExportPIDs ()) << endl;
517  }
518 
519  out.flush (); // make sure the ostringstream got everything
520  return outString->str ();
521  }
522 }
523 
524 } // namespace Details
525 } // namespace Tpetra
526 
527 #endif // TPETRA_DETAILS_TRANSFER_DEF_HPP
Kokkos::DualView< const LO *, device_type > getRemoteLIDs_dv() const
List of entries in the target Map to receive from other processes, as a const DualView (that is sync&#39;...
void describeImpl(Teuchos::FancyOStream &out, const std::string &className, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Implementation of describe() for subclasses (Tpetra::Import and Tpetra::Export).
Common base class of Import and Export.
size_t getNumReceives() const
The number of processes from which we will receive data.
Teuchos::FancyOStream & verboseOutputStream() const
Valid (nonnull) output stream for verbose output.
Teuchos::ArrayView< const int > getExportPIDs() const
List of processes to which entries will be sent.
Teuchos::ArrayView< const LO > getPermuteToLIDs() const
List of local IDs in the target Map that are permuted.
Teuchos::RCP< const map_type > getTargetMap() const
The target Map used to construct this Export or Import.
size_t getNumSameIDs() const
Number of initial identical IDs.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
size_t getNumRemoteIDs() const
Number of entries not on the calling process.
Kokkos::DualView< const LO *, device_type > getPermuteFromLIDs_dv() const
List of local IDs in the source Map that are permuted, as a const DualView (that is sync&#39;d to both ho...
Teuchos::ArrayView< const LO > getRemoteLIDs() const
List of entries in the target Map to receive from other processes.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
Kokkos::DualView< const LO *, device_type > getPermuteToLIDs_dv() const
List of local IDs in the target Map that are permuted, as a const DualView (that is sync&#39;d to both ho...
size_t getNumPermuteIDs() const
Number of IDs to permute but not to communicate.
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
Implementation detail of Import and Export.
Sets up and executes a communication plan for a Tpetra DistObject.
static bool verbose()
Whether Tpetra is in verbose mode.
Teuchos::RCP< ImportExportData< LO, GO, NT > > TransferData_
All the data needed for executing the Export communication plan.
Kokkos::DualView< const LO *, device_type > getExportLIDs_dv() const
List of entries in the source Map that will be sent to other processes, as a const DualView (that is ...
Teuchos::RCP< const map_type > getSourceMap() const
The source Map used to construct this Export or Import.
size_t getNumExportIDs() const
Number of entries that must be sent by the calling process to other processes.
bool isLocallyComplete() const
Is this Export or Import locally complete?
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
Teuchos::ArrayView< const LO > getExportLIDs() const
List of entries in the source Map that will be sent to other processes.
Teuchos::ArrayView< const LO > getPermuteFromLIDs() const
List of local IDs in the source Map that are permuted.
::Tpetra::Distributor & getDistributor() const
The Distributor that this Export or Import object uses to move data.
virtual void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
size_t getNumSends() const
The number of processes to which we will send data.
bool verbose() const
Whether to print verbose debugging output.