Tpetra parallel linear algebra  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Tpetra_DistObject_decl.hpp
Go to the documentation of this file.
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Tpetra: Templated Linear Algebra Services Package
5 // Copyright (2008) Sandia Corporation
6 //
7 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8 // the U.S. Government retains certain rights in this software.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // ************************************************************************
38 // @HEADER
39 
40 #ifndef TPETRA_DISTOBJECT_DECL_HPP
41 #define TPETRA_DISTOBJECT_DECL_HPP
42 
50 
51 #include "Tpetra_Map.hpp"
52 #include "Tpetra_Import.hpp"
53 #include "Tpetra_Export.hpp"
54 #include "Tpetra_SrcDistObject.hpp"
56 #include "Kokkos_ArithTraits.hpp"
57 #include <memory>
58 #include <type_traits>
59 
60 // #ifndef HAVE_TPETRA_TRANSFER_TIMERS
61 // # define HAVE_TPETRA_TRANSFER_TIMERS 1
62 // #endif // HAVE_TPETRA_TRANSFER_TIMERS
63 
64 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
65 # undef HAVE_TPETRA_TRANSFER_TIMERS
66 #endif // HAVE_TPETRA_TRANSFER_TIMERS
67 
68 namespace KokkosClassic {
73  enum ReadWriteOption {
74  ReadWrite = 0,
75  WriteOnly = 1
76  };
77 } // namespace KokkosClassic
78 
79 namespace Tpetra {
80 
163  template<class DistObjectType>
164  void
165  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input,
166  const Teuchos::RCP<const Map<typename DistObjectType::local_ordinal_type,
167  typename DistObjectType::global_ordinal_type,
168  typename DistObjectType::node_type> >& newMap);
169 
206  template<class DistObjectType>
207  void
208  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input);
209 
321  template <class Packet,
322  class LocalOrdinal,
323  class GlobalOrdinal,
324  class Node>
325  class DistObject :
326  virtual public SrcDistObject,
327  virtual public Teuchos::Describable
328  {
329  public:
331 
332 
337  using packet_type = typename ::Kokkos::Details::ArithTraits<Packet>::val_type;
339  using local_ordinal_type = LocalOrdinal;
341  using global_ordinal_type = GlobalOrdinal;
343  using node_type = Node;
344 
346  using device_type = typename Node::device_type;
348  using execution_space = typename device_type::execution_space;
349 
352 
354 
356 
360  explicit DistObject (const Teuchos::RCP<const map_type>& map);
361 
364 
367 
370 
373 
383  virtual ~DistObject () = default;
384 
386 
388 
416  void
417  doImport (const SrcDistObject& source,
419  const CombineMode CM,
420  const bool restrictedMode = false);
421 
449  void
450  doExport (const SrcDistObject& source,
452  const CombineMode CM,
453  const bool restrictedMode = false);
454 
483  void
484  doImport (const SrcDistObject& source,
486  const CombineMode CM,
487  const bool restrictedMode = false);
488 
517  void
518  doExport (const SrcDistObject& source,
520  const CombineMode CM,
521  const bool restrictedMode = false);
522 
524 
526 
532  bool isDistributed () const;
533 
540  virtual Teuchos::RCP<const map_type> getMap () const { return map_; }
541 
543 
545 
550  void print (std::ostream& os) const;
551 
553 
555 
560  virtual std::string description () const;
561 
566  virtual void
567  describe (Teuchos::FancyOStream &out,
568  const Teuchos::EVerbosityLevel verbLevel =
569  Teuchos::Describable::verbLevel_default) const;
570 
572 
574 
621  virtual void
622  removeEmptyProcessesInPlace (const Teuchos::RCP<const map_type>& newMap);
623 
625 
626  protected:
636  DoForward, //*!< Perform the transfer in forward mode.
637  DoReverse //*!< Perform the transfer in reverse mode.
638  };
639 
656  virtual size_t constantNumberOfPackets () const;
657 
677  virtual void
678  doTransfer (const SrcDistObject& src,
679  const ::Tpetra::Details::Transfer<local_ordinal_type, global_ordinal_type, node_type>& transfer,
680  const char modeString[],
681  const ReverseOption revOp,
682  const CombineMode CM,
683  const bool restrictedMode);
684 
699  virtual bool
700  reallocArraysForNumPacketsPerLid (const size_t numExportLIDs,
701  const size_t numImportLIDs);
702 
703 
706  using buffer_memory_space =
708  typename device_type::execution_space>;
709 
710  public:
721  using buffer_device_type =
722  Kokkos::Device<typename device_type::execution_space,
724  protected:
730  virtual void
731  doTransferNew (const SrcDistObject& src,
732  const CombineMode CM,
733  const size_t numSameIDs,
734  const Kokkos::DualView<const local_ordinal_type*,
735  buffer_device_type>& permuteToLIDs,
736  const Kokkos::DualView<const local_ordinal_type*,
737  buffer_device_type>& permuteFromLIDs,
738  const Kokkos::DualView<const local_ordinal_type*,
739  buffer_device_type>& remoteLIDs,
740  const Kokkos::DualView<const local_ordinal_type*,
741  buffer_device_type>& exportLIDs,
742  Distributor& distor,
743  const ReverseOption revOp,
744  const bool commOnHost,
745  const bool restrictedMode);
746 
757 
758 
762  virtual bool
763  checkSizes (const SrcDistObject& source) = 0;
764 
791  virtual void
792  copyAndPermute (const SrcDistObject& source,
793  const size_t numSameIDs,
794  const Kokkos::DualView<const local_ordinal_type*,
795  buffer_device_type>& permuteToLIDs,
796  const Kokkos::DualView<const local_ordinal_type*,
797  buffer_device_type>& permuteFromLIDs);
798 
839  virtual void
840  packAndPrepare (const SrcDistObject& source,
841  const Kokkos::DualView<const local_ordinal_type*,
842  buffer_device_type>& exportLIDs,
843  Kokkos::DualView<packet_type*,
844  buffer_device_type>& exports,
845  Kokkos::DualView<size_t*,
846  buffer_device_type> numPacketsPerLID,
847  size_t& constantNumPackets,
848  Distributor& distor);
849 
892  virtual void
893  unpackAndCombine (const Kokkos::DualView<const local_ordinal_type*,
894  buffer_device_type>& importLIDs,
895  Kokkos::DualView<packet_type*,
896  buffer_device_type> imports,
897  Kokkos::DualView<size_t*,
898  buffer_device_type> numPacketsPerLID,
899  const size_t constantNumPackets,
900  Distributor& distor,
901  const CombineMode combineMode);
902 
903 
905  Teuchos::RCP<const map_type> map_;
906 
907  protected:
908  std::unique_ptr<std::string>
909  createPrefix(const char className[],
910  const char methodName[]) const;
911 
918  Kokkos::DualView<packet_type*, buffer_device_type> imports_;
919 
938  bool
939  reallocImportsIfNeeded (const size_t newSize,
940  const bool verbose,
941  const std::string* prefix);
942 
956  Kokkos::DualView<size_t*, buffer_device_type> numImportPacketsPerLID_;
957 
963  Kokkos::DualView<packet_type*, buffer_device_type> exports_;
964 
978  Kokkos::DualView<size_t*, buffer_device_type> numExportPacketsPerLID_;
979 
980  private:
982 
983 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
984  Teuchos::RCP<Teuchos::Time> doXferTimer_;
985  Teuchos::RCP<Teuchos::Time> copyAndPermuteTimer_;
986  Teuchos::RCP<Teuchos::Time> packAndPrepareTimer_;
987  Teuchos::RCP<Teuchos::Time> doPostsAndWaitsTimer_;
988  Teuchos::RCP<Teuchos::Time> unpackAndCombineTimer_;
989 #endif // HAVE_TPETRA_TRANSFER_TIMERS
990  }; // class DistObject
991 } // namespace Tpetra
992 
993 #endif // TPETRA_DISTOBJECT_DECL_HPP
Communication plan for data redistribution from a uniquely-owned to a (possibly) multiply-owned distr...
Node node_type
The Node type. If you don&#39;t know what this is, don&#39;t use it.
Kokkos::DualView< size_t *, buffer_device_type > numExportPacketsPerLID_
Number of packets to send for each send operation.
typename ExecutionSpace::memory_space comm_buffer_memory_space
Memory space used for MPI communication buffers.
void doImport(const SrcDistObject &source, const Import< LocalOrdinal, GlobalOrdinal, Node > &importer, const CombineMode CM, const bool restrictedMode=false)
Import data into this object using an Import object (&quot;forward mode&quot;).
virtual void doTransferNew(const SrcDistObject &src, const CombineMode CM, const size_t numSameIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &permuteToLIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &permuteFromLIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &remoteLIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &exportLIDs, Distributor &distor, const ReverseOption revOp, const bool commOnHost, const bool restrictedMode)
Implementation detail of doTransfer.
void print(std::ostream &os) const
Print this object to the given output stream.
virtual bool reallocArraysForNumPacketsPerLid(const size_t numExportLIDs, const size_t numImportLIDs)
Reallocate numExportPacketsPerLID_ and/or numImportPacketsPerLID_, if necessary.
Abstract base class for sources of an Import or Export.
bool isDistributed() const
Whether this is a globally distributed object.
Forward declaration of Tpetra::DistObject.
void removeEmptyProcessesInPlace(Teuchos::RCP< DistObjectType > &input, const Teuchos::RCP< const Map< typename DistObjectType::local_ordinal_type, typename DistObjectType::global_ordinal_type, typename DistObjectType::node_type > > &newMap)
Remove processes which contain no elements in this object&#39;s Map.
virtual void doTransfer(const SrcDistObject &src, const ::Tpetra::Details::Transfer< local_ordinal_type, global_ordinal_type, node_type > &transfer, const char modeString[], const ReverseOption revOp, const CombineMode CM, const bool restrictedMode)
Redistribute data across (MPI) processes.
virtual ~DistObject()=default
Destructor (virtual for memory safety of derived classes).
typename::Kokkos::Details::ArithTraits< GlobalOrdinal >::val_type packet_type
The type of each datum being sent or received in an Import or Export.
Kokkos::DualView< size_t *, buffer_device_type > numImportPacketsPerLID_
Number of packets to receive for each receive operation.
typename device_type::execution_space execution_space
The Kokkos execution space.
Kokkos::DualView< packet_type *, buffer_device_type > imports_
Buffer into which packed data are imported (received from other processes).
Kokkos::DualView< packet_type *, buffer_device_type > exports_
Buffer from which packed data are exported (sent to other processes).
::Tpetra::Details::DefaultTypes::comm_buffer_memory_space< typename device_type::execution_space > buffer_memory_space
Kokkos memory space for communication buffers.
Communication plan for data redistribution from a (possibly) multiply-owned to a uniquely-owned distr...
Sets up and executes a communication plan for a Tpetra DistObject.
CombineMode
Rule for combining data in an Import or Export.
Teuchos::RCP< const map_type > map_
The Map over which this object is distributed.
virtual void packAndPrepare(const SrcDistObject &source, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &exportLIDs, Kokkos::DualView< packet_type *, buffer_device_type > &exports, Kokkos::DualView< size_t *, buffer_device_type > numPacketsPerLID, size_t &constantNumPackets, Distributor &distor)
Pack data and metadata for communication (sends).
Abstract base class for objects that can be the source of an Import or Export operation.
bool reallocImportsIfNeeded(const size_t newSize, const bool verbose, const std::string *prefix)
Reallocate imports_ if needed.
DistObject & operator=(const DistObject< Packet, LocalOrdinal, GlobalOrdinal, Node > &)=default
Assignment operator (default).
DistObject(const Teuchos::RCP< const map_type > &map)
Constructor.
virtual std::string description() const
One-line descriptiion of this object.
virtual size_t constantNumberOfPackets() const
Whether the implementation&#39;s instance promises always to have a constant number of packets per LID (l...
A parallel distribution of indices over processes.
void doExport(const SrcDistObject &source, const Export< LocalOrdinal, GlobalOrdinal, Node > &exporter, const CombineMode CM, const bool restrictedMode=false)
Export data into this object using an Export object (&quot;forward mode&quot;).
virtual void copyAndPermute(const SrcDistObject &source, const size_t numSameIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &permuteToLIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &permuteFromLIDs)
Perform copies and permutations that are local to the calling (MPI) process.
virtual void unpackAndCombine(const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &importLIDs, Kokkos::DualView< packet_type *, buffer_device_type > imports, Kokkos::DualView< size_t *, buffer_device_type > numPacketsPerLID, const size_t constantNumPackets, Distributor &distor, const CombineMode combineMode)
Perform any unpacking and combining after communication.
virtual Teuchos::RCP< const map_type > getMap() const
The Map describing the parallel distribution of this object.
virtual void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Print a descriptiion of this object to the given output stream.
Kokkos::Device< typename device_type::execution_space, buffer_memory_space > buffer_device_type
Base class for distributed Tpetra objects that support data redistribution.
virtual void removeEmptyProcessesInPlace(const Teuchos::RCP< const map_type > &newMap)
Remove processes which contain no entries in this object&#39;s Map.
virtual bool checkSizes(const SrcDistObject &source)=0
Compare the source and target (this) objects for compatibility.