Tpetra parallel linear algebra  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Tpetra_DistObject_decl.hpp
Go to the documentation of this file.
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Tpetra: Templated Linear Algebra Services Package
5 // Copyright (2008) Sandia Corporation
6 //
7 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8 // the U.S. Government retains certain rights in this software.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // ************************************************************************
38 // @HEADER
39 
40 #ifndef TPETRA_DISTOBJECT_DECL_HPP
41 #define TPETRA_DISTOBJECT_DECL_HPP
42 
45 
46 #include "Tpetra_Details_DistributorActor.hpp"
47 #include "Tpetra_Map.hpp"
48 #include "Tpetra_Import.hpp"
49 #include "Tpetra_Export.hpp"
50 #include "Tpetra_SrcDistObject.hpp"
52 #include "Kokkos_ArithTraits.hpp"
53 #include <memory>
54 #include <type_traits>
55 
56 // #ifndef HAVE_TPETRA_TRANSFER_TIMERS
57 // # define HAVE_TPETRA_TRANSFER_TIMERS 1
58 // #endif // HAVE_TPETRA_TRANSFER_TIMERS
59 
60 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
61 # undef HAVE_TPETRA_TRANSFER_TIMERS
62 #endif // HAVE_TPETRA_TRANSFER_TIMERS
63 
64 namespace KokkosClassic {
69  enum ReadWriteOption {
70  ReadWrite = 0,
71  OverwriteAll = 1
72  };
73 } // namespace KokkosClassic
74 
75 namespace Tpetra {
76 
159  template<class DistObjectType>
160  void
161  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input,
162  const Teuchos::RCP<const Map<typename DistObjectType::local_ordinal_type,
163  typename DistObjectType::global_ordinal_type,
164  typename DistObjectType::node_type> >& newMap);
165 
202  template<class DistObjectType>
203  void
204  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input);
205 
317  template <class Packet,
318  class LocalOrdinal,
319  class GlobalOrdinal,
320  class Node>
321  class DistObject :
322  virtual public SrcDistObject,
323  virtual public Teuchos::Describable
324  {
325  public:
327 
328 
333  using packet_type = typename ::Kokkos::Details::ArithTraits<Packet>::val_type;
335  using local_ordinal_type = LocalOrdinal;
337  using global_ordinal_type = GlobalOrdinal;
339  using node_type = Node;
340 
342  using device_type = typename Node::device_type;
344  using execution_space = typename device_type::execution_space;
345 
348 
350 
352 
356  explicit DistObject (const Teuchos::RCP<const map_type>& map);
357 
360 
363 
366 
369 
379  virtual ~DistObject () = default;
380 
382 
384 
412  void
413  doImport (const SrcDistObject& source,
415  const CombineMode CM,
416  const bool restrictedMode = false);
417 
445  void
446  doExport (const SrcDistObject& source,
448  const CombineMode CM,
449  const bool restrictedMode = false);
450 
479  void
480  doImport (const SrcDistObject& source,
482  const CombineMode CM,
483  const bool restrictedMode = false);
484 
513  void
514  doExport (const SrcDistObject& source,
516  const CombineMode CM,
517  const bool restrictedMode = false);
518 
519  void
520  beginImport(const SrcDistObject& source,
522  const CombineMode CM,
523  const bool restrictedMode = false);
524 
525  void
526  beginExport(const SrcDistObject& source,
528  const CombineMode CM,
529  const bool restrictedMode = false);
530 
531  void
532  beginImport(const SrcDistObject& source,
534  const CombineMode CM,
535  const bool restrictedMode = false);
536 
537  void
538  beginExport(const SrcDistObject& source,
540  const CombineMode CM,
541  const bool restrictedMode = false);
542 
543  void
544  endImport(const SrcDistObject& source,
546  const CombineMode CM,
547  const bool restrictedMode = false);
548 
549  void
550  endExport(const SrcDistObject& source,
552  const CombineMode CM,
553  const bool restrictedMode = false);
554 
555  void
556  endImport(const SrcDistObject& source,
558  const CombineMode CM,
559  const bool restrictedMode = false);
560 
561  void
562  endExport(const SrcDistObject& source,
564  const CombineMode CM,
565  const bool restrictedMode = false);
566 
569  bool transferArrived() const;
570 
572 
574 
580  bool isDistributed () const;
581 
588  virtual Teuchos::RCP<const map_type> getMap () const { return map_; }
589 
591 
593 
598  void print (std::ostream& os) const;
599 
601 
603 
608  virtual std::string description () const;
609 
614  virtual void
615  describe (Teuchos::FancyOStream &out,
616  const Teuchos::EVerbosityLevel verbLevel =
617  Teuchos::Describable::verbLevel_default) const;
618 
620 
622 
669  virtual void
670  removeEmptyProcessesInPlace (const Teuchos::RCP<const map_type>& newMap);
671 
673 
674  protected:
684  DoForward, //*!< Perform the transfer in forward mode.
685  DoReverse //*!< Perform the transfer in reverse mode.
686  };
687 
704  virtual size_t constantNumberOfPackets () const;
705 
725  virtual void
726  doTransfer (const SrcDistObject& src,
727  const ::Tpetra::Details::Transfer<local_ordinal_type, global_ordinal_type, node_type>& transfer,
728  const char modeString[],
729  const ReverseOption revOp,
730  const CombineMode CM,
731  const bool restrictedMode);
732 
747  virtual bool
748  reallocArraysForNumPacketsPerLid (const size_t numExportLIDs,
749  const size_t numImportLIDs);
750 
751 
754  using buffer_memory_space =
755  ::Tpetra::Details::DefaultTypes::comm_buffer_memory_space<device_type>;
756 
757  public:
768  using buffer_device_type =
769  Kokkos::Device<typename device_type::execution_space,
771  protected:
777  void beginTransfer(const SrcDistObject& src,
778  const ::Tpetra::Details::Transfer<local_ordinal_type, global_ordinal_type, node_type>& transfer,
779  const char modeString[],
780  const ReverseOption revOp,
781  const CombineMode CM,
782  const bool restrictedMode);
783 
784  void endTransfer(const SrcDistObject& src,
785  const ::Tpetra::Details::Transfer<local_ordinal_type, global_ordinal_type, node_type>& transfer,
786  const char modeString[],
787  const ReverseOption revOp,
788  const CombineMode CM,
789  const bool restrictedMode);
790 
791  void doPosts(const Details::DistributorPlan& distributorPlan,
792  size_t constantNumPackets,
793  bool commOnHost,
794  std::shared_ptr<std::string> prefix,
795  const bool canTryAliasing,
796  const CombineMode CM);
797 
798  void doPackAndPrepare(const SrcDistObject& src,
799  const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& exportLIDs,
800  size_t& constantNumPackets);
801 
802  void doUnpackAndCombine(const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& remoteLIDs,
803  size_t constantNumPackets,
804  CombineMode CM);
805 
816 
817 
821  virtual bool
822  checkSizes (const SrcDistObject& source) = 0;
823 
853  virtual void
854  copyAndPermute (const SrcDistObject& source,
855  const size_t numSameIDs,
856  const Kokkos::DualView<const local_ordinal_type*,
857  buffer_device_type>& permuteToLIDs,
858  const Kokkos::DualView<const local_ordinal_type*,
859  buffer_device_type>& permuteFromLIDs,
860  const CombineMode CM);
861 
899  virtual void
900  packAndPrepare (const SrcDistObject& source,
901  const Kokkos::DualView<const local_ordinal_type*,
902  buffer_device_type>& exportLIDs,
903  Kokkos::DualView<packet_type*,
904  buffer_device_type>& exports,
905  Kokkos::DualView<size_t*,
906  buffer_device_type> numPacketsPerLID,
907  size_t& constantNumPackets);
908 
948  virtual void
949  unpackAndCombine (const Kokkos::DualView<const local_ordinal_type*,
950  buffer_device_type>& importLIDs,
951  Kokkos::DualView<packet_type*,
952  buffer_device_type> imports,
953  Kokkos::DualView<size_t*,
954  buffer_device_type> numPacketsPerLID,
955  const size_t constantNumPackets,
956  const CombineMode combineMode);
957 
958 
960  Teuchos::RCP<const map_type> map_;
961 
962  protected:
963  std::unique_ptr<std::string>
964  createPrefix(const char className[],
965  const char methodName[]) const;
966 
973  Kokkos::DualView<packet_type*, buffer_device_type> imports_;
974 
993  virtual bool
994  reallocImportsIfNeeded (const size_t newSize,
995  const bool verbose,
996  const std::string* prefix,
997  const bool remoteLIDsContiguous=false,
998  const CombineMode CM=INSERT);
999 
1013  Kokkos::DualView<size_t*, buffer_device_type> numImportPacketsPerLID_;
1014 
1020  Kokkos::DualView<packet_type*, buffer_device_type> exports_;
1021 
1035  Kokkos::DualView<size_t*, buffer_device_type> numExportPacketsPerLID_;
1036 
1037  private:
1039 
1040  Details::DistributorActor distributorActor_;
1041 
1042 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
1043  Teuchos::RCP<Teuchos::Time> doXferTimer_;
1044  Teuchos::RCP<Teuchos::Time> copyAndPermuteTimer_;
1045  Teuchos::RCP<Teuchos::Time> packAndPrepareTimer_;
1046  Teuchos::RCP<Teuchos::Time> doPostsAndWaitsTimer_;
1047  Teuchos::RCP<Teuchos::Time> unpackAndCombineTimer_;
1048 #endif // HAVE_TPETRA_TRANSFER_TIMERS
1049  }; // class DistObject
1050 } // namespace Tpetra
1051 
1052 #endif // TPETRA_DISTOBJECT_DECL_HPP
Communication plan for data redistribution from a uniquely-owned to a (possibly) multiply-owned distr...
Node node_type
The Node type. If you don&#39;t know what this is, don&#39;t use it.
Kokkos::DualView< size_t *, buffer_device_type > numExportPacketsPerLID_
Number of packets to send for each send operation.
virtual void copyAndPermute(const SrcDistObject &source, const size_t numSameIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &permuteToLIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &permuteFromLIDs, const CombineMode CM)
Perform copies and permutations that are local to the calling (MPI) process.
void doImport(const SrcDistObject &source, const Import< LocalOrdinal, GlobalOrdinal, Node > &importer, const CombineMode CM, const bool restrictedMode=false)
Import data into this object using an Import object (&quot;forward mode&quot;).
void print(std::ostream &os) const
Print this object to the given output stream.
virtual bool reallocArraysForNumPacketsPerLid(const size_t numExportLIDs, const size_t numImportLIDs)
Reallocate numExportPacketsPerLID_ and/or numImportPacketsPerLID_, if necessary.
Abstract base class for sources of an Import or Export.
bool isDistributed() const
Whether this is a globally distributed object.
Forward declaration of Tpetra::DistObject.
void removeEmptyProcessesInPlace(Teuchos::RCP< DistObjectType > &input, const Teuchos::RCP< const Map< typename DistObjectType::local_ordinal_type, typename DistObjectType::global_ordinal_type, typename DistObjectType::node_type > > &newMap)
Remove processes which contain no elements in this object&#39;s Map.
virtual void unpackAndCombine(const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &importLIDs, Kokkos::DualView< packet_type *, buffer_device_type > imports, Kokkos::DualView< size_t *, buffer_device_type > numPacketsPerLID, const size_t constantNumPackets, const CombineMode combineMode)
Perform any unpacking and combining after communication.
virtual void doTransfer(const SrcDistObject &src, const ::Tpetra::Details::Transfer< local_ordinal_type, global_ordinal_type, node_type > &transfer, const char modeString[], const ReverseOption revOp, const CombineMode CM, const bool restrictedMode)
Redistribute data across (MPI) processes.
void beginTransfer(const SrcDistObject &src, const ::Tpetra::Details::Transfer< local_ordinal_type, global_ordinal_type, node_type > &transfer, const char modeString[], const ReverseOption revOp, const CombineMode CM, const bool restrictedMode)
Implementation detail of doTransfer.
virtual ~DistObject()=default
Destructor (virtual for memory safety of derived classes).
typename::Kokkos::Details::ArithTraits< GlobalOrdinal >::val_type packet_type
The type of each datum being sent or received in an Import or Export.
Kokkos::DualView< size_t *, buffer_device_type > numImportPacketsPerLID_
Number of packets to receive for each receive operation.
Insert new values that don&#39;t currently exist.
typename device_type::execution_space execution_space
The Kokkos execution space.
Kokkos::DualView< packet_type *, buffer_device_type > imports_
Buffer into which packed data are imported (received from other processes).
Kokkos::DualView< packet_type *, buffer_device_type > exports_
Buffer from which packed data are exported (sent to other processes).
::Tpetra::Details::DefaultTypes::comm_buffer_memory_space< device_type > buffer_memory_space
Kokkos memory space for communication buffers.
Communication plan for data redistribution from a (possibly) multiply-owned to a uniquely-owned distr...
virtual void packAndPrepare(const SrcDistObject &source, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &exportLIDs, Kokkos::DualView< packet_type *, buffer_device_type > &exports, Kokkos::DualView< size_t *, buffer_device_type > numPacketsPerLID, size_t &constantNumPackets)
Pack data and metadata for communication (sends).
CombineMode
Rule for combining data in an Import or Export.
Teuchos::RCP< const map_type > map_
The Map over which this object is distributed.
Abstract base class for objects that can be the source of an Import or Export operation.
DistObject & operator=(const DistObject< Packet, LocalOrdinal, GlobalOrdinal, Node > &)=default
Assignment operator (default).
DistObject(const Teuchos::RCP< const map_type > &map)
Constructor.
virtual std::string description() const
One-line descriptiion of this object.
bool transferArrived() const
Whether the data from an import/export operation has arrived, and is ready for the unpack and combine...
virtual size_t constantNumberOfPackets() const
Whether the implementation&#39;s instance promises always to have a constant number of packets per LID (l...
virtual bool reallocImportsIfNeeded(const size_t newSize, const bool verbose, const std::string *prefix, const bool remoteLIDsContiguous=false, const CombineMode CM=INSERT)
Reallocate imports_ if needed.
A parallel distribution of indices over processes.
void doExport(const SrcDistObject &source, const Export< LocalOrdinal, GlobalOrdinal, Node > &exporter, const CombineMode CM, const bool restrictedMode=false)
Export data into this object using an Export object (&quot;forward mode&quot;).
virtual Teuchos::RCP< const map_type > getMap() const
The Map describing the parallel distribution of this object.
virtual void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Print a descriptiion of this object to the given output stream.
Kokkos::Device< typename device_type::execution_space, buffer_memory_space > buffer_device_type
Base class for distributed Tpetra objects that support data redistribution.
virtual void removeEmptyProcessesInPlace(const Teuchos::RCP< const map_type > &newMap)
Remove processes which contain no entries in this object&#39;s Map.
virtual bool checkSizes(const SrcDistObject &source)=0
Compare the source and target (this) objects for compatibility.