42 #ifndef TPETRA_DETAILS_TRANSFER_DEF_HPP
43 #define TPETRA_DETAILS_TRANSFER_DEF_HPP
45 #include "Tpetra_Details_gathervPrint.hpp"
46 #include "Tpetra_Distributor.hpp"
47 #include "Tpetra_ImportExportData.hpp"
48 #include "Tpetra_Map.hpp"
49 #include "Teuchos_CommHelpers.hpp"
50 #include "Teuchos_TypeNameTraits.hpp"
56 template<
class ElementType,
class DeviceType>
57 Teuchos::ArrayView<const ElementType>
58 makeConstArrayViewFromDualView (
const Kokkos::DualView<ElementType*, DeviceType>& dv)
60 TEUCHOS_ASSERT( ! dv.need_sync_host () );
61 auto hostView = dv.view_host ();
62 const auto size = hostView.extent (0);
63 return Teuchos::ArrayView<const ElementType> (size == 0 ?
nullptr : hostView.data (), size);
71 template <
class LO,
class GO,
class NT>
72 Transfer<LO, GO, NT>::
73 Transfer (
const Teuchos::RCP<const map_type>& source,
74 const Teuchos::RCP<const map_type>& target,
75 const Teuchos::RCP<Teuchos::FancyOStream>& out,
76 const Teuchos::RCP<Teuchos::ParameterList>& plist,
77 const std::string& className) :
78 TransferData_ (new
ImportExportData<LO, GO, NT> (source, target, out, plist))
81 this->setParameterList (plist, className);
84 template <
class LO,
class GO,
class NT>
90 TEUCHOS_ASSERT( ! this->TransferData_->out_.is_null () );
93 template <
class LO,
class GO,
class NT>
97 const std::string& className)
99 using ::Tpetra::Details::Behavior;
104 bool verboseParam =
false;
105 if (! plist.is_null ()) {
108 if (plist->isType<
bool> (
"Verbose")) {
109 verboseParam = plist->get<
bool> (
"Verbose");
111 else if (plist->isType<
bool> (
"Debug")) {
112 verboseParam = plist->get<
bool> (
"Debug");
115 this->TransferData_->verbose_ = verboseEnv || verboseParam;
118 template <
class LO,
class GO,
class NT>
122 return TransferData_->numSameIDs_;
125 template <
class LO,
class GO,
class NT>
129 return static_cast<size_t> (TransferData_->permuteFromLIDs_.extent (0));
132 template <
class LO,
class GO,
class NT>
133 Kokkos::DualView<const LO*, typename Transfer<LO, GO, NT>::device_type>
136 const auto& dv = TransferData_->permuteFromLIDs_;
137 TEUCHOS_TEST_FOR_EXCEPTION
138 (dv.need_sync_device (), std::logic_error,
139 "Tpetra::Details::Transfer::getPermuteFromLIDs_dv: "
140 "DualView needs sync to device" );
141 TEUCHOS_TEST_FOR_EXCEPTION
142 (dv.need_sync_host (), std::logic_error,
143 "Tpetra::Details::Transfer::getPermuteFromLIDs_dv: "
144 "DualView needs sync to host" );
148 template <
class LO,
class GO,
class NT>
149 Teuchos::ArrayView<const LO>
152 return makeConstArrayViewFromDualView (TransferData_->permuteFromLIDs_);
155 template <
class LO,
class GO,
class NT>
156 Kokkos::DualView<const LO*, typename Transfer<LO, GO, NT>::device_type>
159 const auto& dv = TransferData_->permuteToLIDs_;
160 TEUCHOS_TEST_FOR_EXCEPTION
161 (dv.need_sync_device (), std::logic_error,
162 "Tpetra::Details::Transfer::getPermuteToLIDs_dv: "
163 "DualView needs sync to device" );
164 TEUCHOS_TEST_FOR_EXCEPTION
165 (dv.need_sync_host (), std::logic_error,
166 "Tpetra::Details::Transfer::getPermuteToLIDs_dv: "
167 "DualView needs sync to host" );
171 template <
class LO,
class GO,
class NT>
172 Teuchos::ArrayView<const LO>
175 return makeConstArrayViewFromDualView (TransferData_->permuteToLIDs_);
178 template <
class LO,
class GO,
class NT>
182 return static_cast<size_t> (TransferData_->remoteLIDs_.extent (0));
185 template <
class LO,
class GO,
class NT>
186 Kokkos::DualView<const LO*, typename Transfer<LO, GO, NT>::device_type>
189 const auto& dv = TransferData_->remoteLIDs_;
190 TEUCHOS_TEST_FOR_EXCEPTION
191 (dv.need_sync_device (), std::logic_error,
192 "Tpetra::Details::Transfer::getRemoteLIDs_dv: "
193 "DualView needs sync to device" );
194 TEUCHOS_TEST_FOR_EXCEPTION
195 (dv.need_sync_host (), std::logic_error,
196 "Tpetra::Details::Transfer::getRemoteLIDs_dv: "
197 "DualView needs sync to host" );
201 template <
class LO,
class GO,
class NT>
202 Teuchos::ArrayView<const LO>
205 return makeConstArrayViewFromDualView (TransferData_->remoteLIDs_);
208 template <
class LO,
class GO,
class NT>
212 return static_cast<size_t> (TransferData_->exportLIDs_.extent (0));
215 template <
class LO,
class GO,
class NT>
216 Kokkos::DualView<const LO*, typename Transfer<LO, GO, NT>::device_type>
219 const auto& dv = TransferData_->exportLIDs_;
220 TEUCHOS_TEST_FOR_EXCEPTION
221 (dv.need_sync_device (), std::logic_error,
222 "Tpetra::Details::Transfer::getExportLIDs_dv: "
223 "DualView needs sync to device" );
224 TEUCHOS_TEST_FOR_EXCEPTION
225 (dv.need_sync_host (), std::logic_error,
226 "Tpetra::Details::Transfer::getExportLIDs_dv: "
227 "DualView needs sync to host" );
231 template <
class LO,
class GO,
class NT>
232 Teuchos::ArrayView<const LO>
235 return makeConstArrayViewFromDualView (TransferData_->exportLIDs_);
238 template <
class LO,
class GO,
class NT>
239 Teuchos::ArrayView<const int>
242 return TransferData_->exportPIDs_ ();
245 template <
class LO,
class GO,
class NT>
246 Teuchos::RCP<const typename Transfer<LO, GO, NT>::map_type>
249 return TransferData_->source_;
252 template <
class LO,
class GO,
class NT>
253 Teuchos::RCP<const typename Transfer<LO, GO, NT>::map_type>
256 return TransferData_->target_;
259 template <
class LO,
class GO,
class NT>
263 return TransferData_->distributor_;
266 template <
class LO,
class GO,
class NT>
270 return TransferData_->isLocallyComplete_;
273 template <
class LO,
class GO,
class NT>
277 const Teuchos::EVerbosityLevel verbLevel)
const
279 this->describeImpl (out,
"Tpetra::Details::Transfer", verbLevel);
282 template<
class LO,
class GO,
class NT>
283 Teuchos::FancyOStream&
287 Teuchos::FancyOStream* outPtr = TransferData_->out_.getRawPtr ();
288 TEUCHOS_ASSERT( outPtr !=
nullptr );
292 template<
class LO,
class GO,
class NT>
296 return TransferData_->verbose_;
299 template<
class LO,
class GO,
class NT>
303 const std::string& className,
304 const Teuchos::EVerbosityLevel verbLevel)
const
306 using Teuchos::TypeNameTraits;
307 using Teuchos::VERB_DEFAULT;
308 using Teuchos::VERB_NONE;
309 using Teuchos::VERB_LOW;
311 const Teuchos::EVerbosityLevel vl =
312 (verbLevel == VERB_DEFAULT) ? VERB_LOW : verbLevel;
314 if (vl == VERB_NONE) {
321 auto srcMap = this->getSourceMap ();
322 if (srcMap.is_null ()) {
325 auto comm = srcMap->getComm ();
326 if (comm.is_null ()) {
329 if (this->getTargetMap ().is_null () ||
330 this->getTargetMap ()->getComm ().is_null ()) {
334 const int myRank = comm->getRank ();
335 const int numProcs = comm->getSize ();
342 Teuchos::RCP<Teuchos::OSTab> tab0, tab1;
348 tab0 = Teuchos::rcp (
new Teuchos::OSTab (out));
350 out <<
"\"" << className <<
"\":" << endl;
351 tab1 = Teuchos::rcp (
new Teuchos::OSTab (out));
354 out <<
"Template parameters:" << endl;
355 Teuchos::OSTab tab2 (out);
356 out <<
"LocalOrdinal: " << TypeNameTraits<LO>::name () << endl
357 <<
"GlobalOrdinal: " << TypeNameTraits<GO>::name () << endl
358 <<
"Node: " << TypeNameTraits<NT>::name () << endl;
361 const std::string label = this->getObjectLabel ();
363 out <<
"Label: " << label << endl;
365 out <<
"Number of processes: " << numProcs << endl;
372 this->globalDescribe (out, vl);
384 out <<
"Source Map:" << endl;
387 this->getSourceMap ()->describe (out, vl);
390 out <<
"Target Map:" << endl;
393 this->getTargetMap ()->describe (out, vl);
396 out <<
"Distributor:" << endl;
398 this->getDistributor ().describe (out, vl);
401 template<
class LO,
class GO,
class NT>
405 const Teuchos::EVerbosityLevel vl)
const
408 using Teuchos::OSTab;
410 using Teuchos::toString;
417 auto srcMap = this->getSourceMap ();
418 if (srcMap.is_null ()) {
421 RCP<const Teuchos::Comm<int> > comm = srcMap->getComm ();
422 if (comm.is_null ()) {
426 const std::string myStr = localDescribeToString (vl);
427 ::Tpetra::Details::gathervPrint (out, myStr, *comm);
430 template<
class LO,
class GO,
class NT>
432 Transfer<LO, GO, NT>::
433 localDescribeToString (
const Teuchos::EVerbosityLevel vl)
const
435 using Teuchos::OSTab;
439 RCP<std::ostringstream> outString (
new std::ostringstream);
440 RCP<Teuchos::FancyOStream> outp = Teuchos::getFancyOStream (outString);
441 Teuchos::FancyOStream& out = *outp;
443 RCP<const Teuchos::Comm<int> > comm = this->getSourceMap ()->getComm ();
444 if (this->getSourceMap ().is_null () ||
445 this->getSourceMap ()->getComm ().is_null ()) {
450 return std::string (
"");
453 const int myRank = comm->getRank ();
454 const int numProcs = comm->getSize ();
456 out <<
"Process " << myRank <<
" of " << numProcs <<
":" << endl;
459 out <<
"numSameIDs: " << getNumSameIDs () << endl;
460 out <<
"numPermuteIDs: " << getNumPermuteIDs () << endl;
461 out <<
"numRemoteIDs: " << getNumRemoteIDs () << endl;
462 out <<
"numExportIDs: " << getNumExportIDs () << endl;
466 if (vl <= Teuchos::VERB_MEDIUM) {
467 out <<
"permuteFromLIDs count: " << getPermuteFromLIDs ().size () << endl
468 <<
"permuteToLIDs count: " << getPermuteToLIDs ().size () << endl
469 <<
"remoteLIDs count: " << getRemoteLIDs ().size () << endl
470 <<
"exportLIDs count: " << getExportLIDs ().size () << endl
471 <<
"exportPIDs count: " << getExportPIDs () << endl;
475 RCP<const Map<LO,GO,NT> > tmap = getTargetMap();
476 RCP<const Map<LO,GO,NT> > smap = getSourceMap();
477 Teuchos::Array<GO> RemoteGIDs(getRemoteLIDs().size());
478 Teuchos::Array<int> RemotePIDs(getRemoteLIDs().size());
479 for(
size_t i=0; i<(size_t)getRemoteLIDs().size(); i++)
480 RemoteGIDs[i] = tmap->getGlobalElement(getRemoteLIDs()[i]);
482 Teuchos::Array<int> ExportGIDs(getExportLIDs().size());
483 for(
size_t i=0; i<(size_t)getExportLIDs().size(); i++)
484 ExportGIDs[i] = smap->getGlobalElement(getExportLIDs()[i]);
489 Teuchos::ArrayView<const int> ProcsFrom = D.
getProcsFrom();
490 Teuchos::ArrayView<const size_t> LengthsFrom = D.
getLengthsFrom();
491 for (
size_t i = 0, j = 0; i < NumReceives; ++i) {
492 const int pid = ProcsFrom[i];
493 for (
size_t k = 0; k < LengthsFrom[i]; ++k) {
499 out <<
"distor.NumRecvs : "<<NumReceives<<endl
500 <<
"distor.ProcsFrom : "<<toString(ProcsFrom)<<endl
501 <<
"distor.LengthsFrom: "<<toString(LengthsFrom)<<endl;
504 <<
"distor.ProcsTo : "<<toString(D.
getProcsTo())<<endl
505 <<
"distor.LengthsTo : "<<toString(D.
getLengthsTo())<<endl;
509 out <<
"permuteFromLIDs: " << toString (getPermuteFromLIDs ()) << endl
510 <<
"permuteToLIDs: " << toString (getPermuteToLIDs ()) << endl
511 <<
"remoteLIDs: " << toString (getRemoteLIDs ()) << endl
512 <<
"remoteGIDs: " << toString (RemoteGIDs ()) << endl
513 <<
"remotePIDs: " << toString (RemotePIDs ()) << endl
514 <<
"exportLIDs: " << toString (getExportLIDs ()) << endl
515 <<
"exportGIDs: " << toString (ExportGIDs ()) << endl
516 <<
"exportPIDs: " << toString (getExportPIDs ()) << endl;
520 return outString->str ();
527 #endif // TPETRA_DETAILS_TRANSFER_DEF_HPP
Kokkos::DualView< const LO *, device_type > getRemoteLIDs_dv() const
List of entries in the target Map to receive from other processes, as a const DualView (that is sync'...
void describeImpl(Teuchos::FancyOStream &out, const std::string &className, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Implementation of describe() for subclasses (Tpetra::Import and Tpetra::Export).
Common base class of Import and Export.
size_t getNumReceives() const
The number of processes from which we will receive data.
Teuchos::FancyOStream & verboseOutputStream() const
Valid (nonnull) output stream for verbose output.
Teuchos::ArrayView< const int > getExportPIDs() const
List of processes to which entries will be sent.
Teuchos::ArrayView< const LO > getPermuteToLIDs() const
List of local IDs in the target Map that are permuted.
Teuchos::RCP< const map_type > getTargetMap() const
The target Map used to construct this Export or Import.
size_t getNumSameIDs() const
Number of initial identical IDs.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
size_t getNumRemoteIDs() const
Number of entries not on the calling process.
Kokkos::DualView< const LO *, device_type > getPermuteFromLIDs_dv() const
List of local IDs in the source Map that are permuted, as a const DualView (that is sync'd to both ho...
Teuchos::ArrayView< const LO > getRemoteLIDs() const
List of entries in the target Map to receive from other processes.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
Kokkos::DualView< const LO *, device_type > getPermuteToLIDs_dv() const
List of local IDs in the target Map that are permuted, as a const DualView (that is sync'd to both ho...
size_t getNumPermuteIDs() const
Number of IDs to permute but not to communicate.
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
Implementation detail of Import and Export.
Sets up and executes a communication plan for a Tpetra DistObject.
static bool verbose()
Whether Tpetra is in verbose mode.
Teuchos::RCP< ImportExportData< LO, GO, NT > > TransferData_
All the data needed for executing the Export communication plan.
Kokkos::DualView< const LO *, device_type > getExportLIDs_dv() const
List of entries in the source Map that will be sent to other processes, as a const DualView (that is ...
Teuchos::RCP< const map_type > getSourceMap() const
The source Map used to construct this Export or Import.
size_t getNumExportIDs() const
Number of entries that must be sent by the calling process to other processes.
bool isLocallyComplete() const
Is this Export or Import locally complete?
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
Teuchos::ArrayView< const LO > getExportLIDs() const
List of entries in the source Map that will be sent to other processes.
Teuchos::ArrayView< const LO > getPermuteFromLIDs() const
List of local IDs in the source Map that are permuted.
::Tpetra::Distributor & getDistributor() const
The Distributor that this Export or Import object uses to move data.
virtual void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
size_t getNumSends() const
The number of processes to which we will send data.
bool verbose() const
Whether to print verbose debugging output.