42 #ifndef TPETRA_DETAILS_TRANSFER_DEF_HPP
43 #define TPETRA_DETAILS_TRANSFER_DEF_HPP
46 #include "Tpetra_Distributor.hpp"
47 #include "Tpetra_ImportExportData.hpp"
48 #include "Tpetra_Map.hpp"
49 #include "Teuchos_CommHelpers.hpp"
50 #include "Teuchos_TypeNameTraits.hpp"
56 template<
class ElementType,
class DeviceType>
57 Teuchos::ArrayView<const ElementType>
58 makeConstArrayViewFromDualView (
const Kokkos::DualView<ElementType*, DeviceType>& dv)
60 TEUCHOS_ASSERT( ! dv.need_sync_host () );
61 auto hostView = dv.view_host ();
62 const auto size = hostView.extent (0);
63 return Teuchos::ArrayView<const ElementType> (size == 0 ?
nullptr : hostView.data (), size);
66 template<
class DeviceType,
class LocalOrdinal>
67 struct OrderedViewFunctor {
68 OrderedViewFunctor (
const Kokkos::View<LocalOrdinal*, DeviceType>& viewToCheck) :
69 viewToCheck_ (viewToCheck) {}
70 KOKKOS_INLINE_FUNCTION
void operator() (
const size_t i,
unsigned int& isUnordered)
const {
71 isUnordered |=
static_cast<unsigned int>(viewToCheck_(i)+1 != viewToCheck_(i+1));
73 Kokkos::View<const LocalOrdinal*, DeviceType> viewToCheck_;
76 template<
class DeviceType,
class LocalOrdinal>
78 isViewOrdered (
const Kokkos::View<LocalOrdinal*, DeviceType>& viewToCheck)
80 using Kokkos::parallel_reduce;
81 typedef DeviceType DT;
82 typedef typename DT::execution_space DES;
83 typedef Kokkos::RangePolicy<DES, size_t> range_type;
85 const size_t size = viewToCheck.extent (0);
86 unsigned int isUnordered = 0;
88 parallel_reduce (
"isViewOrdered",
89 range_type (0, size-1),
90 OrderedViewFunctor<DeviceType, LocalOrdinal> (viewToCheck),
92 return isUnordered == 0;
100 template <
class LO,
class GO,
class NT>
101 Transfer<LO, GO, NT>::
102 Transfer (
const Teuchos::RCP<const map_type>& source,
103 const Teuchos::RCP<const map_type>& target,
104 const Teuchos::RCP<Teuchos::FancyOStream>& out,
105 const Teuchos::RCP<Teuchos::ParameterList>& plist,
106 const std::string& className) :
107 TransferData_ (new
ImportExportData<LO, GO, NT> (source, target, out, plist))
110 this->setParameterList (plist, className);
113 template <
class LO,
class GO,
class NT>
119 TEUCHOS_ASSERT( ! this->TransferData_->out_.is_null () );
122 template <
class LO,
class GO,
class NT>
126 const std::string& className)
128 using ::Tpetra::Details::Behavior;
133 bool verboseParam =
false;
134 if (! plist.is_null ()) {
137 if (plist->isType<
bool> (
"Verbose")) {
138 verboseParam = plist->get<
bool> (
"Verbose");
140 else if (plist->isType<
bool> (
"Debug")) {
141 verboseParam = plist->get<
bool> (
"Debug");
144 this->TransferData_->verbose_ = verboseEnv || verboseParam;
147 template <
class LO,
class GO,
class NT>
151 return TransferData_->numSameIDs_;
154 template <
class LO,
class GO,
class NT>
158 return static_cast<size_t> (TransferData_->permuteFromLIDs_.extent (0));
161 template <
class LO,
class GO,
class NT>
162 Kokkos::DualView<const LO*, typename Transfer<LO, GO, NT>::device_type>
165 const auto& dv = TransferData_->permuteFromLIDs_;
166 TEUCHOS_TEST_FOR_EXCEPTION
167 (dv.need_sync_device (), std::logic_error,
168 "Tpetra::Details::Transfer::getPermuteFromLIDs_dv: "
169 "DualView needs sync to device" );
170 TEUCHOS_TEST_FOR_EXCEPTION
171 (dv.need_sync_host (), std::logic_error,
172 "Tpetra::Details::Transfer::getPermuteFromLIDs_dv: "
173 "DualView needs sync to host" );
177 template <
class LO,
class GO,
class NT>
178 Teuchos::ArrayView<const LO>
181 return makeConstArrayViewFromDualView (TransferData_->permuteFromLIDs_);
184 template <
class LO,
class GO,
class NT>
185 Kokkos::DualView<const LO*, typename Transfer<LO, GO, NT>::device_type>
188 const auto& dv = TransferData_->permuteToLIDs_;
189 TEUCHOS_TEST_FOR_EXCEPTION
190 (dv.need_sync_device (), std::logic_error,
191 "Tpetra::Details::Transfer::getPermuteToLIDs_dv: "
192 "DualView needs sync to device" );
193 TEUCHOS_TEST_FOR_EXCEPTION
194 (dv.need_sync_host (), std::logic_error,
195 "Tpetra::Details::Transfer::getPermuteToLIDs_dv: "
196 "DualView needs sync to host" );
200 template <
class LO,
class GO,
class NT>
201 Teuchos::ArrayView<const LO>
204 return makeConstArrayViewFromDualView (TransferData_->permuteToLIDs_);
207 template <
class LO,
class GO,
class NT>
211 return static_cast<size_t> (TransferData_->remoteLIDs_.extent (0));
214 template <
class LO,
class GO,
class NT>
215 Kokkos::DualView<const LO*, typename Transfer<LO, GO, NT>::device_type>
218 const auto& dv = TransferData_->remoteLIDs_;
219 TEUCHOS_TEST_FOR_EXCEPTION
220 (dv.need_sync_device (), std::logic_error,
221 "Tpetra::Details::Transfer::getRemoteLIDs_dv: "
222 "DualView needs sync to device" );
223 TEUCHOS_TEST_FOR_EXCEPTION
224 (dv.need_sync_host (), std::logic_error,
225 "Tpetra::Details::Transfer::getRemoteLIDs_dv: "
226 "DualView needs sync to host" );
230 template <
class LO,
class GO,
class NT>
231 Teuchos::ArrayView<const LO>
234 return makeConstArrayViewFromDualView (TransferData_->remoteLIDs_);
237 template <
class LO,
class GO,
class NT>
241 return static_cast<size_t> (TransferData_->exportLIDs_.extent (0));
244 template <
class LO,
class GO,
class NT>
245 Kokkos::DualView<const LO*, typename Transfer<LO, GO, NT>::device_type>
248 const auto& dv = TransferData_->exportLIDs_;
249 TEUCHOS_TEST_FOR_EXCEPTION
250 (dv.need_sync_device (), std::logic_error,
251 "Tpetra::Details::Transfer::getExportLIDs_dv: "
252 "DualView needs sync to device" );
253 TEUCHOS_TEST_FOR_EXCEPTION
254 (dv.need_sync_host (), std::logic_error,
255 "Tpetra::Details::Transfer::getExportLIDs_dv: "
256 "DualView needs sync to host" );
260 template <
class LO,
class GO,
class NT>
261 Teuchos::ArrayView<const LO>
264 return makeConstArrayViewFromDualView (TransferData_->exportLIDs_);
267 template <
class LO,
class GO,
class NT>
268 Teuchos::ArrayView<const int>
271 return TransferData_->exportPIDs_ ();
274 template <
class LO,
class GO,
class NT>
275 Teuchos::RCP<const typename Transfer<LO, GO, NT>::map_type>
278 return TransferData_->source_;
281 template <
class LO,
class GO,
class NT>
282 Teuchos::RCP<const typename Transfer<LO, GO, NT>::map_type>
285 return TransferData_->target_;
288 template <
class LO,
class GO,
class NT>
292 return TransferData_->distributor_;
295 template <
class LO,
class GO,
class NT>
299 return TransferData_->isLocallyComplete_;
302 template <
class LO,
class GO,
class NT>
306 return (getNumSameIDs() == std::min(getSourceMap()->getLocalNumElements(),
307 getTargetMap()->getLocalNumElements()));
310 template <
class LO,
class GO,
class NT>
318 bool ordered = (getNumSameIDs() == std::min(getSourceMap()->getLocalNumElements(),
319 getTargetMap()->getLocalNumElements()));
320 ordered &= (getTargetMap()->getLocalNumElements() == getNumSameIDs() + getNumRemoteIDs());
322 const auto& dv = TransferData_->remoteLIDs_;
323 TEUCHOS_TEST_FOR_EXCEPTION
324 (dv.need_sync_device (), std::logic_error,
325 "Tpetra::Details::Transfer::getRemoteLIDs_dv: "
326 "DualView needs sync to device" );
327 auto v_d = dv.view_device ();
328 ordered &= isViewOrdered<device_type, LO>(v_d);
330 TransferData_->remoteLIDsContiguous_ = ordered;
332 ordered = (getNumSameIDs() == std::min(getSourceMap()->getLocalNumElements(),
333 getTargetMap()->getLocalNumElements()));
334 ordered &= (getSourceMap()->getLocalNumElements() == getNumSameIDs() + getNumExportIDs());
336 const auto& dv = TransferData_->exportLIDs_;
337 TEUCHOS_TEST_FOR_EXCEPTION
338 (dv.need_sync_device (), std::logic_error,
339 "Tpetra::Details::Transfer::getRemoteLIDs_dv: "
340 "DualView needs sync to device" );
341 auto v_d = dv.view_device ();
342 ordered &= isViewOrdered<device_type, LO>(v_d);
344 TransferData_->exportLIDsContiguous_ = ordered;
347 template <
class LO,
class GO,
class NT>
349 Transfer<LO, GO, NT>::
350 areRemoteLIDsContiguous ()
const {
351 return TransferData_->remoteLIDsContiguous_;
354 template <
class LO,
class GO,
class NT>
356 Transfer<LO, GO, NT>::
357 areExportLIDsContiguous ()
const {
358 return TransferData_->exportLIDsContiguous_;
362 template <
class LO,
class GO,
class NT>
366 const Teuchos::EVerbosityLevel verbLevel)
const
368 this->describeImpl (out,
"Tpetra::Details::Transfer", verbLevel);
371 template<
class LO,
class GO,
class NT>
372 Teuchos::FancyOStream&
376 Teuchos::FancyOStream* outPtr = TransferData_->out_.getRawPtr ();
377 TEUCHOS_ASSERT( outPtr !=
nullptr );
381 template<
class LO,
class GO,
class NT>
385 return TransferData_->verbose_;
388 template<
class LO,
class GO,
class NT>
392 const std::string& className,
393 const Teuchos::EVerbosityLevel verbLevel)
const
395 using Teuchos::TypeNameTraits;
396 using Teuchos::VERB_DEFAULT;
397 using Teuchos::VERB_NONE;
398 using Teuchos::VERB_LOW;
400 const Teuchos::EVerbosityLevel vl =
401 (verbLevel == VERB_DEFAULT) ? VERB_LOW : verbLevel;
403 if (vl == VERB_NONE) {
410 auto srcMap = this->getSourceMap ();
411 if (srcMap.is_null ()) {
414 auto comm = srcMap->getComm ();
415 if (comm.is_null ()) {
418 if (this->getTargetMap ().is_null () ||
419 this->getTargetMap ()->getComm ().is_null ()) {
423 const int myRank = comm->getRank ();
424 const int numProcs = comm->getSize ();
431 Teuchos::RCP<Teuchos::OSTab> tab0, tab1;
437 tab0 = Teuchos::rcp (
new Teuchos::OSTab (out));
439 out <<
"\"" << className <<
"\":" << endl;
440 tab1 = Teuchos::rcp (
new Teuchos::OSTab (out));
443 out <<
"Template parameters:" << endl;
444 Teuchos::OSTab tab2 (out);
445 out <<
"LocalOrdinal: " << TypeNameTraits<LO>::name () << endl
446 <<
"GlobalOrdinal: " << TypeNameTraits<GO>::name () << endl
447 <<
"Node: " << TypeNameTraits<NT>::name () << endl;
450 const std::string label = this->getObjectLabel ();
452 out <<
"Label: " << label << endl;
454 out <<
"Number of processes: " << numProcs << endl;
461 this->globalDescribe (out, vl);
473 out <<
"Source Map:" << endl;
476 this->getSourceMap ()->describe (out, vl);
479 out <<
"Target Map:" << endl;
482 this->getTargetMap ()->describe (out, vl);
485 out <<
"Distributor:" << endl;
487 this->getDistributor ().describe (out, vl);
490 template<
class LO,
class GO,
class NT>
494 const Teuchos::EVerbosityLevel vl)
const
497 using Teuchos::OSTab;
499 using Teuchos::toString;
506 auto srcMap = this->getSourceMap ();
507 if (srcMap.is_null ()) {
510 RCP<const Teuchos::Comm<int> > comm = srcMap->getComm ();
511 if (comm.is_null ()) {
515 const std::string myStr = localDescribeToString (vl);
516 ::Tpetra::Details::gathervPrint (out, myStr, *comm);
519 template<
class LO,
class GO,
class NT>
521 Transfer<LO, GO, NT>::
522 localDescribeToString (
const Teuchos::EVerbosityLevel vl)
const
524 using Teuchos::OSTab;
528 RCP<std::ostringstream> outString (
new std::ostringstream);
529 RCP<Teuchos::FancyOStream> outp = Teuchos::getFancyOStream (outString);
530 Teuchos::FancyOStream& out = *outp;
532 RCP<const Teuchos::Comm<int> > comm = this->getSourceMap ()->getComm ();
533 if (this->getSourceMap ().is_null () ||
534 this->getSourceMap ()->getComm ().is_null ()) {
539 return std::string (
"");
542 const int myRank = comm->getRank ();
543 const int numProcs = comm->getSize ();
545 out <<
"Process " << myRank <<
" of " << numProcs <<
":" << endl;
548 out <<
"numSameIDs: " << getNumSameIDs () << endl;
549 out <<
"numPermuteIDs: " << getNumPermuteIDs () << endl;
550 out <<
"numRemoteIDs: " << getNumRemoteIDs () << endl;
551 out <<
"numExportIDs: " << getNumExportIDs () << endl;
555 if (vl <= Teuchos::VERB_MEDIUM) {
556 out <<
"permuteFromLIDs count: " << getPermuteFromLIDs ().size () << endl
557 <<
"permuteToLIDs count: " << getPermuteToLIDs ().size () << endl
558 <<
"remoteLIDs count: " << getRemoteLIDs ().size () << endl
559 <<
"exportLIDs count: " << getExportLIDs ().size () << endl
560 <<
"exportPIDs count: " << getExportPIDs () << endl;
564 RCP<const Map<LO,GO,NT> > tmap = getTargetMap();
565 RCP<const Map<LO,GO,NT> > smap = getSourceMap();
566 Teuchos::Array<GO> RemoteGIDs(getRemoteLIDs().size());
567 Teuchos::Array<int> RemotePIDs(getRemoteLIDs().size());
568 for(
size_t i=0; i<(size_t)getRemoteLIDs().size(); i++)
569 RemoteGIDs[i] = tmap->getGlobalElement(getRemoteLIDs()[i]);
571 Teuchos::Array<int> ExportGIDs(getExportLIDs().size());
572 for(
size_t i=0; i<(size_t)getExportLIDs().size(); i++)
573 ExportGIDs[i] = smap->getGlobalElement(getExportLIDs()[i]);
578 Teuchos::ArrayView<const int> ProcsFrom = D.
getProcsFrom();
579 Teuchos::ArrayView<const size_t> LengthsFrom = D.
getLengthsFrom();
580 for (
size_t i = 0, j = 0; i < NumReceives; ++i) {
581 const int pid = ProcsFrom[i];
582 for (
size_t k = 0; k < LengthsFrom[i]; ++k) {
588 out <<
"distor.NumRecvs : "<<NumReceives<<endl
589 <<
"distor.ProcsFrom : "<<toString(ProcsFrom)<<endl
590 <<
"distor.LengthsFrom: "<<toString(LengthsFrom)<<endl;
593 <<
"distor.ProcsTo : "<<toString(D.
getProcsTo())<<endl
594 <<
"distor.LengthsTo : "<<toString(D.
getLengthsTo())<<endl;
598 out <<
"permuteFromLIDs: " << toString (getPermuteFromLIDs ()) << endl
599 <<
"permuteToLIDs: " << toString (getPermuteToLIDs ()) << endl
600 <<
"remoteLIDs: " << toString (getRemoteLIDs ()) << endl
601 <<
"remoteGIDs: " << toString (RemoteGIDs ()) << endl
602 <<
"remotePIDs: " << toString (RemotePIDs ()) << endl
603 <<
"exportLIDs: " << toString (getExportLIDs ()) << endl
604 <<
"exportGIDs: " << toString (ExportGIDs ()) << endl
605 <<
"exportPIDs: " << toString (getExportPIDs ()) << endl;
609 return outString->str ();
614 template <
class LO,
class GO,
class NT>
616 expertSetRemoteLIDsContiguous(Transfer<LO, GO, NT> transfer,
bool contig) {
617 transfer.TransferData_->remoteLIDsContiguous_ = contig;
621 template <
class LO,
class GO,
class NT>
623 expertSetExportLIDsContiguous(Transfer<LO, GO, NT> transfer,
bool contig) {
624 transfer.TransferData_->exportLIDsContiguous_ = contig;
631 #endif // TPETRA_DETAILS_TRANSFER_DEF_HPP
Kokkos::DualView< const LO *, device_type > getRemoteLIDs_dv() const
List of entries in the target Map to receive from other processes, as a const DualView (that is sync'...
void describeImpl(Teuchos::FancyOStream &out, const std::string &className, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Implementation of describe() for subclasses (Tpetra::Import and Tpetra::Export).
Common base class of Import and Export.
size_t getNumReceives() const
The number of processes from which we will receive data.
Teuchos::FancyOStream & verboseOutputStream() const
Valid (nonnull) output stream for verbose output.
bool isLocallyFitted() const
Are source and target map locally fitted?
Declaration of a function that prints strings from each process.
Teuchos::ArrayView< const int > getExportPIDs() const
List of processes to which entries will be sent.
Teuchos::ArrayView< const LO > getPermuteToLIDs() const
List of local IDs in the target Map that are permuted.
Teuchos::RCP< const map_type > getTargetMap() const
The target Map used to construct this Export or Import.
size_t getNumSameIDs() const
Number of initial identical IDs.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
size_t getNumRemoteIDs() const
Number of entries not on the calling process.
Kokkos::DualView< const LO *, device_type > getPermuteFromLIDs_dv() const
List of local IDs in the source Map that are permuted, as a const DualView (that is sync'd to both ho...
Teuchos::ArrayView< const LO > getRemoteLIDs() const
List of entries in the target Map to receive from other processes.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
Kokkos::DualView< const LO *, device_type > getPermuteToLIDs_dv() const
List of local IDs in the target Map that are permuted, as a const DualView (that is sync'd to both ho...
size_t getNumPermuteIDs() const
Number of IDs to permute but not to communicate.
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
Implementation detail of Import and Export.
Sets up and executes a communication plan for a Tpetra DistObject.
static bool verbose()
Whether Tpetra is in verbose mode.
Teuchos::RCP< ImportExportData< LO, GO, NT > > TransferData_
All the data needed for executing the Export communication plan.
Kokkos::DualView< const LO *, device_type > getExportLIDs_dv() const
List of entries in the source Map that will be sent to other processes, as a const DualView (that is ...
Teuchos::RCP< const map_type > getSourceMap() const
The source Map used to construct this Export or Import.
size_t getNumExportIDs() const
Number of entries that must be sent by the calling process to other processes.
bool isLocallyComplete() const
Is this Export or Import locally complete?
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
Teuchos::ArrayView< const LO > getExportLIDs() const
List of entries in the source Map that will be sent to other processes.
Teuchos::ArrayView< const LO > getPermuteFromLIDs() const
List of local IDs in the source Map that are permuted.
::Tpetra::Distributor & getDistributor() const
The Distributor that this Export or Import object uses to move data.
virtual void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
size_t getNumSends() const
The number of processes to which we will send data.
bool verbose() const
Whether to print verbose debugging output.