Tpetra parallel linear algebra  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Tpetra_Distributor.hpp
1 // @HEADER
2 // *****************************************************************************
3 // Tpetra: Templated Linear Algebra Services Package
4 //
5 // Copyright 2008 NTESS and the Tpetra contributors.
6 // SPDX-License-Identifier: BSD-3-Clause
7 // *****************************************************************************
8 // @HEADER
9 
10 #ifndef TPETRA_DISTRIBUTOR_HPP
11 #define TPETRA_DISTRIBUTOR_HPP
12 
13 #include "Tpetra_Details_DistributorActor.hpp"
15 
16 #include "Tpetra_Util.hpp"
17 #include "Teuchos_as.hpp"
18 #include "Teuchos_Describable.hpp"
19 #include "Teuchos_ParameterListAcceptorDefaultBase.hpp"
20 #include "Teuchos_VerboseObject.hpp"
22 
23 #include "KokkosCompat_View.hpp"
24 #include "Kokkos_Core.hpp"
25 #include "Kokkos_TeuchosCommAdapters.hpp"
26 #include <memory>
27 #include <sstream>
28 #include <type_traits>
29 
30 namespace Tpetra {
31 
32 #if defined (TPETRA_ENABLE_DEPRECATED_CODE)
33  [[deprecated]] Teuchos::Array<std::string> distributorSendTypes ();
40 #endif
41 
104  class Distributor :
105  public Teuchos::Describable,
106  public Teuchos::ParameterListAcceptorDefaultBase {
107  public:
109 
110 
119  explicit Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm);
120 
132  Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
133  const Teuchos::RCP<Teuchos::FancyOStream>& out);
134 
148  Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
149  const Teuchos::RCP<Teuchos::ParameterList>& plist);
150 
167  Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
168  const Teuchos::RCP<Teuchos::FancyOStream>& out,
169  const Teuchos::RCP<Teuchos::ParameterList>& plist);
170 
172  Distributor (const Distributor& distributor);
173 
178  virtual ~Distributor () = default;
179 
185  void swap (Distributor& rhs);
186 
188 
190 
195  void setParameterList (const Teuchos::RCP<Teuchos::ParameterList>& plist);
196 
201  Teuchos::RCP<const Teuchos::ParameterList> getValidParameters () const;
202 
204 
206 
226  size_t createFromSends (const Teuchos::ArrayView<const int>& exportProcIDs);
227 
261  template <class Ordinal>
262  void
263  createFromRecvs (const Teuchos::ArrayView<const Ordinal>& remoteIDs,
264  const Teuchos::ArrayView<const int>& remoteProcIDs,
265  Teuchos::Array<Ordinal>& exportIDs,
266  Teuchos::Array<int>& exportProcIDs);
267 
275  void
276  createFromSendsAndRecvs (const Teuchos::ArrayView<const int>& exportProcIDs,
277  const Teuchos::ArrayView<const int>& remoteProcIDs);
278 
280 
282 
286  size_t getNumReceives() const;
287 
291  size_t getNumSends() const;
292 
294  bool hasSelfMessage() const;
295 
297  size_t getMaxSendLength() const;
298 
300  size_t getTotalReceiveLength() const;
301 
306  Teuchos::ArrayView<const int> getProcsFrom() const;
307 
312  Teuchos::ArrayView<const int> getProcsTo() const;
313 
321  Teuchos::ArrayView<const size_t> getLengthsFrom() const;
322 
330  Teuchos::ArrayView<const size_t> getLengthsTo() const;
331 
337  return plan_.howInitialized();
338  }
339 
341 
343 
354  Teuchos::RCP<Distributor> getReverse(bool create=true) const;
355 
357 
359 
366  void doWaits ();
367 
374  void doReverseWaits ();
375 
396  template <class ExpView, class ImpView>
397  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
399  const ExpView &exports,
400  size_t numPackets,
401  const ImpView &imports);
402 
424  template <class ExpView, class ImpView>
425  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
426  doPostsAndWaits (const ExpView &exports,
427  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
428  const ImpView &imports,
429  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
430 
455  template <class ExpView, class ImpView>
456  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
457  doPosts (const ExpView &exports,
458  size_t numPackets,
459  const ImpView &imports);
460 
479  template <class ExpView, class ImpView>
480  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
481  doPosts (const ExpView &exports,
482  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
483  const ImpView &imports,
484  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
485 
490  template <class ExpView, class ImpView>
491  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
492  doReversePostsAndWaits (const ExpView &exports,
493  size_t numPackets,
494  const ImpView &imports);
495 
500  template <class ExpView, class ImpView>
501  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
502  doReversePostsAndWaits (const ExpView &exports,
503  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
504  const ImpView &imports,
505  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
506 
511  template <class ExpView, class ImpView>
512  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
513  doReversePosts (const ExpView &exports,
514  size_t numPackets,
515  const ImpView &imports);
516 
521  template <class ExpView, class ImpView>
522  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
523  doReversePosts (const ExpView &exports,
524  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
525  const ImpView &imports,
526  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
527 
529 
531 
533  std::string description() const;
534 
556  void
557  describe (Teuchos::FancyOStream& out,
558  const Teuchos::EVerbosityLevel verbLevel =
559  Teuchos::Describable::verbLevel_default) const;
561 
566  const Details::DistributorPlan& getPlan() const { return plan_; }
567  private:
569  Details::DistributorActor actor_;
570 
572 
573 
575  static bool getVerbose();
576 
581  std::unique_ptr<std::string>
582  createPrefix(const char methodName[]) const;
583 
585  bool verbose_ = getVerbose();
587 
592  mutable Teuchos::RCP<Distributor> reverseDistributor_;
593 
606  template <class Ordinal>
607  void computeSends (const Teuchos::ArrayView<const Ordinal> &remoteGIDs,
608  const Teuchos::ArrayView<const int> &remoteProcIDs,
609  Teuchos::Array<Ordinal> &exportGIDs,
610  Teuchos::Array<int> &exportProcIDs);
611 
613  void createReverseDistributor() const;
614 
615 
620  std::string
621  localDescribeToString (const Teuchos::EVerbosityLevel vl) const;
622  }; // class Distributor
623 
624  template <class ExpView, class ImpView>
625  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
626  Distributor::
627  doPostsAndWaits (const ExpView& exports,
628  size_t numPackets,
629  const ImpView& imports)
630  {
631  actor_.doPostsAndWaits(plan_, exports, numPackets, imports);
632  }
633 
634  template <class ExpView, class ImpView>
635  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
636  Distributor::
637  doPostsAndWaits(const ExpView& exports,
638  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
639  const ImpView& imports,
640  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
641  {
642  actor_.doPostsAndWaits(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
643  }
644 
645 
646  template <class ExpView, class ImpView>
647  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
648  Distributor::
649  doPosts (const ExpView &exports,
650  size_t numPackets,
651  const ImpView &imports)
652  {
653  actor_.doPosts(plan_, exports, numPackets, imports);
654  }
655 
656  template <class ExpView, class ImpView>
657  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
658  Distributor::
659  doPosts (const ExpView &exports,
660  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
661  const ImpView &imports,
662  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
663  {
664  actor_.doPosts(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
665  }
666 
667  template <class ExpView, class ImpView>
668  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
669  Distributor::
670  doReversePostsAndWaits (const ExpView& exports,
671  size_t numPackets,
672  const ImpView& imports)
673  {
674  doReversePosts (exports, numPackets, imports);
675  doReverseWaits ();
676  }
677 
678  template <class ExpView, class ImpView>
679  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
680  Distributor::
681  doReversePostsAndWaits (const ExpView& exports,
682  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
683  const ImpView& imports,
684  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
685  {
686  doReversePosts (exports, numExportPacketsPerLID, imports,
687  numImportPacketsPerLID);
688  doReverseWaits ();
689  }
690 
691  template <class ExpView, class ImpView>
692  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
693  Distributor::
694  doReversePosts (const ExpView &exports,
695  size_t numPackets,
696  const ImpView &imports)
697  {
698  // FIXME (mfh 29 Mar 2012) WHY?
699  TEUCHOS_TEST_FOR_EXCEPTION(
700  ! plan_.getIndicesTo().is_null(), std::runtime_error,
701  "Tpetra::Distributor::doReversePosts(3 args): Can only do "
702  "reverse communication when original data are blocked by process.");
703  if (reverseDistributor_.is_null ()) {
704  createReverseDistributor ();
705  }
706  reverseDistributor_->doPosts (exports, numPackets, imports);
707  }
708 
709  template <class ExpView, class ImpView>
710  typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
711  Distributor::
712  doReversePosts (const ExpView &exports,
713  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
714  const ImpView &imports,
715  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
716  {
717  // FIXME (mfh 29 Mar 2012) WHY?
718  TEUCHOS_TEST_FOR_EXCEPTION(
719  ! plan_.getIndicesTo().is_null(), std::runtime_error,
720  "Tpetra::Distributor::doReversePosts(3 args): Can only do "
721  "reverse communication when original data are blocked by process.");
722  if (reverseDistributor_.is_null ()) {
723  createReverseDistributor ();
724  }
725  reverseDistributor_->doPosts (exports, numExportPacketsPerLID,
726  imports, numImportPacketsPerLID);
727  }
728 
729  template <class OrdinalType>
730  void Distributor::
731  computeSends(const Teuchos::ArrayView<const OrdinalType>& importGIDs,
732  const Teuchos::ArrayView<const int>& importProcIDs,
733  Teuchos::Array<OrdinalType>& exportGIDs,
734  Teuchos::Array<int>& exportProcIDs)
735  {
736  // NOTE (mfh 19 Apr 2012): There was a note on this code saying:
737  // "assumes that size_t >= Ordinal". The code certainly does
738  // assume that sizeof(size_t) >= sizeof(OrdinalType) as well as
739  // sizeof(size_t) >= sizeof(int). This is because it casts the
740  // OrdinalType elements of importGIDs (along with their
741  // corresponding process IDs, as int) to size_t, and does a
742  // doPostsAndWaits<size_t>() to send the packed data.
743  using Teuchos::ArrayView;
744  using std::endl;
745  using size_type = typename ArrayView<const OrdinalType>::size_type;
746  const char errPrefix[] = "Tpetra::Distributor::computeSends: ";
747  const char suffix[] =
748  " Please report this bug to the Tpetra developers.";
749 
750  const int myRank = plan_.getComm()->getRank ();
751 
752  TEUCHOS_TEST_FOR_EXCEPTION
753  (importGIDs.size () != importProcIDs.size (),
754  std::invalid_argument, errPrefix << "On Process " << myRank
755  << ": importProcIDs.size()=" << importProcIDs.size()
756  << " != importGIDs.size()=" << importGIDs.size() << ".");
757 
758  const size_type numImports = importProcIDs.size();
759  Kokkos::View<size_t*, Kokkos::HostSpace> importObjs("importObjs", 2*numImports);
760  // Pack pairs (importGIDs[i], my process ID) to send into importObjs.
761  for (size_type i = 0; i < numImports; ++i) {
762  importObjs[2*i] = static_cast<size_t>(importGIDs[i]);
763  importObjs[2*i+1] = static_cast<size_t>(myRank);
764  }
765  //
766  // Use a temporary Distributor to send the (importGIDs[i], myRank)
767  // pairs to importProcIDs[i].
768  //
769  Distributor tempPlan(plan_.getComm());
770  // mfh 20 Mar 2014: An extra-cautious cast from unsigned to
771  // signed, in order to forestall any possible causes for Bug 6069.
772  const size_t numExportsAsSizeT =
773  tempPlan.createFromSends(importProcIDs);
774  const size_type numExports =
775  static_cast<size_type>(numExportsAsSizeT);
776  TEUCHOS_TEST_FOR_EXCEPTION
777  (numExports < 0, std::logic_error, errPrefix <<
778  "tempPlan.createFromSends() returned numExports="
779  << numExportsAsSizeT << " as a size_t, which overflows to "
780  << numExports << " when cast to " <<
781  Teuchos::TypeNameTraits<size_type>::name () << "." << suffix);
782  TEUCHOS_TEST_FOR_EXCEPTION
783  (size_type(tempPlan.getTotalReceiveLength()) != numExports,
784  std::logic_error, errPrefix << "tempPlan.getTotalReceiveLength()="
785  << tempPlan.getTotalReceiveLength () << " != numExports="
786  << numExports << "." << suffix);
787 
788  if (numExports > 0) {
789  exportGIDs.resize(numExports);
790  exportProcIDs.resize(numExports);
791  }
792 
793  // exportObjs: Packed receive buffer. (exportObjs[2*i],
794  // exportObjs[2*i+1]) will give the (GID, PID) pair for export i,
795  // after tempPlan.doPostsAndWaits(...) finishes below.
796  //
797  // FIXME (mfh 19 Mar 2014) This only works if OrdinalType fits in
798  // size_t. This issue might come up, for example, on a 32-bit
799  // machine using 64-bit global indices. I will add a check here
800  // for that case.
801  static_assert(sizeof(size_t) >= sizeof(OrdinalType),
802  "Tpetra::Distributor::computeSends: "
803  "sizeof(size_t) < sizeof(OrdinalType).");
804 
805  TEUCHOS_TEST_FOR_EXCEPTION
806  (tempPlan.getTotalReceiveLength () < size_t(numExports),
807  std::logic_error,
808  errPrefix << "tempPlan.getTotalReceiveLength()="
809  << tempPlan.getTotalReceiveLength() << " < numExports="
810  << numExports << "." << suffix);
811 
812  Kokkos::View<size_t*, Kokkos::HostSpace> exportObjs("exportObjs", tempPlan.getTotalReceiveLength() * 2);
813  tempPlan.doPostsAndWaits(importObjs, 2, exportObjs);
814 
815  // Unpack received (GID, PID) pairs into exportIDs resp. exportProcIDs.
816  for (size_type i = 0; i < numExports; ++i) {
817  exportGIDs[i] = static_cast<OrdinalType> (exportObjs[2*i]);
818  exportProcIDs[i] = static_cast<int> (exportObjs[2*i+1]);
819  }
820 
821  // Store tempPlan instead of recreating it later
822  plan_ = tempPlan.plan_;
823  }
824 
825  template <class OrdinalType>
826  void Distributor::
827  createFromRecvs (const Teuchos::ArrayView<const OrdinalType> &remoteGIDs,
828  const Teuchos::ArrayView<const int> &remoteProcIDs,
829  Teuchos::Array<OrdinalType> &exportGIDs,
830  Teuchos::Array<int> &exportProcIDs)
831  {
832  using std::endl;
833  const char errPrefix[] = "Tpetra::Distributor::createFromRecvs: ";
834  const int myRank = plan_.getComm()->getRank();
835 
836  std::unique_ptr<std::string> prefix;
837  if (verbose_) {
838  prefix = createPrefix("createFromRecvs");
839  std::ostringstream os;
840  os << *prefix << "Start" << endl;
841  std::cerr << os.str();
842  }
843 
844  const bool debug = Details::Behavior::debug("Distributor");
845  if (debug) {
846  using Teuchos::outArg;
847  using Teuchos::REDUCE_MAX;
848  using Teuchos::reduceAll;
849  // In debug mode, first test locally, then do an all-reduce to
850  // make sure that all processes passed.
851  const int errProc =
852  (remoteGIDs.size () != remoteProcIDs.size ()) ? myRank : -1;
853  int maxErrProc = -1;
854  reduceAll(*plan_.getComm(), REDUCE_MAX, errProc, outArg(maxErrProc));
855  TEUCHOS_TEST_FOR_EXCEPTION
856  (maxErrProc != -1, std::runtime_error, errPrefix << "Lists "
857  "of remote IDs and remote process IDs must have the same "
858  "size on all participating processes. Maximum process ID "
859  "with error: " << maxErrProc << ".");
860  }
861  else { // in non-debug mode, just test locally
862  // NOTE (mfh 13 Feb 2020) This needs to throw std::runtime_error
863  // in order to make an existing Distributor unit test pass.
864  TEUCHOS_TEST_FOR_EXCEPTION
865  (remoteGIDs.size() != remoteProcIDs.size(), std::runtime_error,
866  errPrefix << "On Process " << myRank << ": "
867  "remoteGIDs.size()=" << remoteGIDs.size() <<
868  " != remoteProcIDs.size()=" << remoteProcIDs.size() << ".");
869  }
870 
871  computeSends(remoteGIDs, remoteProcIDs, exportGIDs, exportProcIDs);
872 
873  plan_.createFromRecvs(remoteProcIDs);
874 
875  if (verbose_) {
876  std::ostringstream os;
877  os << *prefix << "Done" << endl;
878  std::cerr << os.str();
879  }
880  }
881 
882 } // namespace Tpetra
883 
884 #endif // TPETRA_DISTRIBUTOR_HPP
const Details::DistributorPlan & getPlan() const
Get this Distributor&#39;s DistributorPlan.
size_t getNumReceives() const
The number of processes from which we will receive data.
std::string description() const
Return a one-line description of this object.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the reverse communication plan.
Teuchos::RCP< Distributor > getReverse(bool create=true) const
A reverse communication plan Distributor.
EDistributorHowInitialized
Enum indicating how and whether a Distributor was initialized.
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const
List of valid Distributor parameters.
virtual ~Distributor()=default
Destructor (virtual for memory safety).
static bool debug()
Whether Tpetra is in debug mode.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a forward plan, but do not execute the waits yet.
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
Details::EDistributorHowInitialized howInitialized() const
Return an enum indicating whether and how a Distributor was initialized.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a reverse plan, but do not execute the waits yet.
Sets up and executes a communication plan for a Tpetra DistObject.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters.
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the (forward) communication plan.
Stand-alone utility functions and macros.
Teuchos::Array< std::string > distributorSendTypes()
Valid string values for Distributor&#39;s &quot;Send type&quot; parameter.
size_t getNumSends() const
The number of processes to which we will send data.
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
size_t getMaxSendLength() const
Maximum number of values this process will send to another single process.
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
Distributor(const Teuchos::RCP< const Teuchos::Comm< int > > &comm)
Construct using the specified communicator and default parameters.
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra&#39;s behavior.