Tpetra parallel linear algebra  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Tpetra_Distributor.cpp
1 // @HEADER
2 // *****************************************************************************
3 // Tpetra: Templated Linear Algebra Services Package
4 //
5 // Copyright 2008 NTESS and the Tpetra contributors.
6 // SPDX-License-Identifier: BSD-3-Clause
7 // *****************************************************************************
8 // @HEADER
9 
10 #include "Tpetra_Distributor.hpp"
13 #include "Tpetra_Util.hpp"
14 #include "Tpetra_Details_makeValidVerboseStream.hpp"
15 #include "Teuchos_StandardParameterEntryValidators.hpp"
16 #include "Teuchos_VerboseObjectParameterListHelpers.hpp"
17 #include <numeric>
18 
19 namespace Tpetra {
20  // We set default values of Distributor's Boolean parameters here,
21  // in this one place. That way, if we want to change the default
22  // value of a parameter, we don't have to search the whole file to
23  // ensure a consistent setting.
24  namespace {
25  // Default value of the "Debug" parameter.
26  const bool tpetraDistributorDebugDefault = false;
27  } // namespace (anonymous)
28 
29 #if defined (TPETRA_ENABLE_DEPRECATED_CODE)
30  Teuchos::Array<std::string>
32  {
34  }
35 #endif
36 
38  Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
39  const Teuchos::RCP<Teuchos::FancyOStream>& /* out */,
40  const Teuchos::RCP<Teuchos::ParameterList>& plist)
41  : plan_(comm)
42  {
43  this->setParameterList(plist);
44  }
45 
47  Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm)
48  : Distributor (comm, Teuchos::null, Teuchos::null)
49  {}
50 
52  Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
53  const Teuchos::RCP<Teuchos::FancyOStream>& out)
54  : Distributor (comm, out, Teuchos::null)
55  {}
56 
58  Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
59  const Teuchos::RCP<Teuchos::ParameterList>& plist)
60  : Distributor (comm, Teuchos::null, plist)
61  {}
62 
64  Distributor (const Distributor& distributor)
65  : plan_(distributor.plan_)
66  , actor_(distributor.actor_)
67  , verbose_ (distributor.verbose_)
68  , reverseDistributor_ (distributor.reverseDistributor_)
69  {
70  using Teuchos::ParameterList;
71  using Teuchos::RCP;
72  using Teuchos::rcp;
73 
74  RCP<const ParameterList> rhsList = distributor.getParameterList ();
75  RCP<ParameterList> newList = rhsList.is_null () ? Teuchos::null :
76  Teuchos::parameterList (*rhsList);
77  this->setParameterList (newList);
78  }
79 
81  using Teuchos::ParameterList;
82  using Teuchos::parameterList;
83  using Teuchos::RCP;
84 
85  std::swap (plan_, rhs.plan_);
86  std::swap (actor_, rhs.actor_);
87  std::swap (verbose_, rhs.verbose_);
88  std::swap (reverseDistributor_, rhs.reverseDistributor_);
89 
90  // Swap parameter lists. If they are the same object, make a deep
91  // copy first, so that modifying one won't modify the other one.
92  RCP<ParameterList> lhsList = this->getNonconstParameterList ();
93  RCP<ParameterList> rhsList = rhs.getNonconstParameterList ();
94  if (lhsList.getRawPtr () == rhsList.getRawPtr () && ! rhsList.is_null ()) {
95  rhsList = parameterList (*rhsList);
96  }
97  if (! rhsList.is_null ()) {
98  this->setMyParamList (rhsList);
99  }
100  if (! lhsList.is_null ()) {
101  rhs.setMyParamList (lhsList);
102  }
103 
104  // We don't need to swap timers, because all instances of
105  // Distributor use the same timers.
106  }
107 
108  bool
109  Distributor::getVerbose()
110  {
111  return Details::Behavior::verbose("Distributor") ||
112  Details::Behavior::verbose("Tpetra::Distributor");
113  }
114 
115  std::unique_ptr<std::string>
116  Distributor::
117  createPrefix(const char methodName[]) const
118  {
119  return Details::createPrefix(
120  plan_.getComm().getRawPtr(), "Distributor", methodName);
121  }
122 
123  void
125  setParameterList (const Teuchos::RCP<Teuchos::ParameterList>& plist)
126  {
127  using ::Tpetra::Details::Behavior;
128  using Teuchos::FancyOStream;
129  using Teuchos::getIntegralValue;
130  using Teuchos::includesVerbLevel;
131  using Teuchos::ParameterList;
132  using Teuchos::parameterList;
133  using Teuchos::RCP;
134  using std::endl;
135 
136  if (! plist.is_null()) {
137  RCP<const ParameterList> validParams = getValidParameters ();
138  plist->validateParametersAndSetDefaults (*validParams);
139 
140  // ParameterListAcceptor semantics require pointer identity of the
141  // sublist passed to setParameterList(), so we save the pointer.
142  this->setMyParamList (plist);
143 
144  RCP<ParameterList> planParams(plist);
145  planParams->remove("Debug", false);
146  planParams->remove("VerboseObject", false);
147  plan_.setParameterList(planParams);
148  }
149  }
150 
151  Teuchos::RCP<const Teuchos::ParameterList>
153  {
154  using Teuchos::Array;
155  using Teuchos::ParameterList;
156  using Teuchos::parameterList;
157  using Teuchos::RCP;
158  using Teuchos::setStringToIntegralParameter;
159 
160  const bool debug = tpetraDistributorDebugDefault;
161 
162  Array<std::string> sendTypes = Details::distributorSendTypes ();
163  const Array<Details::EDistributorSendType> sendTypeEnums = Details::distributorSendTypeEnums ();
164 
165  RCP<ParameterList> plist = parameterList ("Tpetra::Distributor");
166  setStringToIntegralParameter<Details::EDistributorSendType> ("Send type",
167  Details::Behavior::defaultSendType(), "When using MPI, the variant of send to use in "
168  "do[Reverse]Posts()", sendTypes(), sendTypeEnums(), plist.getRawPtr());
169  plist->set ("Debug", debug, "Whether to print copious debugging output on "
170  "all processes.");
171  plist->set ("Timer Label","","Label for Time Monitor output");
172 
173  // mfh 24 Dec 2015: Tpetra no longer inherits from
174  // Teuchos::VerboseObject, so it doesn't need the "VerboseObject"
175  // sublist. However, we retain the "VerboseObject" sublist
176  // anyway, for backwards compatibility (otherwise the above
177  // validation would fail with an invalid parameter name, should
178  // the user still want to provide this list).
179  Teuchos::setupVerboseObjectSublist (&*plist);
180  return Teuchos::rcp_const_cast<const ParameterList> (plist);
181  }
182 
183 
185  { return plan_.getTotalReceiveLength(); }
186 
188  { return plan_.getNumReceives(); }
189 
191  { return plan_.hasSelfMessage(); }
192 
194  { return plan_.getNumSends(); }
195 
197  { return plan_.getMaxSendLength(); }
198 
199  Teuchos::ArrayView<const int> Distributor::getProcsFrom() const
200  { return plan_.getProcsFrom(); }
201 
202  Teuchos::ArrayView<const size_t> Distributor::getLengthsFrom() const
203  { return plan_.getLengthsFrom(); }
204 
205  Teuchos::ArrayView<const int> Distributor::getProcsTo() const
206  { return plan_.getProcsTo(); }
207 
208  Teuchos::ArrayView<const size_t> Distributor::getLengthsTo() const
209  { return plan_.getLengthsTo(); }
210 
211  Teuchos::RCP<Distributor>
212  Distributor::getReverse(bool create) const {
213  if (reverseDistributor_.is_null () && create) {
214  createReverseDistributor ();
215  }
216  TEUCHOS_TEST_FOR_EXCEPTION
217  (reverseDistributor_.is_null () && create, std::logic_error, "The reverse "
218  "Distributor is null after createReverseDistributor returned. "
219  "Please report this bug to the Tpetra developers.");
220  return reverseDistributor_;
221  }
222 
223 
224  void
225  Distributor::createReverseDistributor() const
226  {
227  reverseDistributor_ = Teuchos::rcp(new Distributor(plan_.getComm()));
228  reverseDistributor_->plan_ = *plan_.getReversePlan();
229  reverseDistributor_->verbose_ = verbose_;
230 
231  // requests_: Allocated on demand.
232  // reverseDistributor_: See note below
233 
234  // I am my reverse Distributor's reverse Distributor.
235  // Thus, it would be legit to do the following:
236  //
237  // reverseDistributor_->reverseDistributor_ = Teuchos::rcp (this, false);
238  //
239  // (Note use of a "weak reference" to avoid a circular RCP
240  // dependency.) The only issue is that if users hold on to the
241  // reverse Distributor but let go of the forward one, this
242  // reference won't be valid anymore. However, the reverse
243  // Distributor is really an implementation detail of Distributor
244  // and not meant to be used directly, so we don't need to do this.
245  reverseDistributor_->reverseDistributor_ = Teuchos::null;
246  }
247 
248  void
250  {
251  actor_.doWaits(plan_);
252  }
253 
255  // call doWaits() on the reverse Distributor, if it exists
256  if (! reverseDistributor_.is_null()) {
257  reverseDistributor_->doWaits();
258  }
259  }
260 
261  std::string Distributor::description () const {
262  std::ostringstream out;
263 
264  out << "\"Tpetra::Distributor\": {";
265  const std::string label = this->getObjectLabel ();
266  if (label != "") {
267  out << "Label: " << label << ", ";
268  }
269  out << "How initialized: "
270  << Details::DistributorHowInitializedEnumToString (plan_.howInitialized())
271  << ", Parameters: {"
272  << "Send type: "
273  << DistributorSendTypeEnumToString (plan_.getSendType())
274  << ", Debug: " << (verbose_ ? "true" : "false")
275  << "}}";
276  return out.str ();
277  }
278 
279  std::string
280  Distributor::
281  localDescribeToString (const Teuchos::EVerbosityLevel vl) const
282  {
283  using Teuchos::toString;
284  using Teuchos::VERB_HIGH;
285  using Teuchos::VERB_EXTREME;
286  using std::endl;
287 
288  // This preserves current behavior of Distributor.
289  if (vl <= Teuchos::VERB_LOW || plan_.getComm().is_null ()) {
290  return std::string ();
291  }
292 
293  auto outStringP = Teuchos::rcp (new std::ostringstream ());
294  auto outp = Teuchos::getFancyOStream (outStringP); // returns RCP
295  Teuchos::FancyOStream& out = *outp;
296 
297  const int myRank = plan_.getComm()->getRank ();
298  const int numProcs = plan_.getComm()->getSize ();
299  out << "Process " << myRank << " of " << numProcs << ":" << endl;
300  Teuchos::OSTab tab1 (out);
301 
302  out << "selfMessage: " << hasSelfMessage() << endl;
303  out << "numSends: " << getNumSends() << endl;
304  if (vl == VERB_HIGH || vl == VERB_EXTREME) {
305  out << "procsTo: " << toString (plan_.getProcsTo()) << endl;
306  out << "lengthsTo: " << toString (plan_.getLengthsTo()) << endl;
307  out << "maxSendLength: " << getMaxSendLength() << endl;
308  }
309  if (vl == VERB_EXTREME) {
310  out << "startsTo: " << toString (plan_.getStartsTo()) << endl;
311  out << "indicesTo: " << toString (plan_.getIndicesTo()) << endl;
312  }
313  if (vl == VERB_HIGH || vl == VERB_EXTREME) {
314  out << "numReceives: " << getNumReceives() << endl;
315  out << "totalReceiveLength: " << getTotalReceiveLength() << endl;
316  out << "lengthsFrom: " << toString (plan_.getLengthsFrom()) << endl;
317  out << "procsFrom: " << toString (plan_.getProcsFrom()) << endl;
318  }
319 
320  out.flush (); // make sure the ostringstream got everything
321  return outStringP->str ();
322  }
323 
324  void
326  describe (Teuchos::FancyOStream& out,
327  const Teuchos::EVerbosityLevel verbLevel) const
328  {
329  using std::endl;
330  using Teuchos::VERB_DEFAULT;
331  using Teuchos::VERB_NONE;
332  using Teuchos::VERB_LOW;
333  using Teuchos::VERB_MEDIUM;
334  using Teuchos::VERB_HIGH;
335  using Teuchos::VERB_EXTREME;
336  const Teuchos::EVerbosityLevel vl =
337  (verbLevel == VERB_DEFAULT) ? VERB_LOW : verbLevel;
338 
339  if (vl == VERB_NONE) {
340  return; // don't print anything
341  }
342  // If this Distributor's Comm is null, then the the calling
343  // process does not participate in Distributor-related collective
344  // operations with the other processes. In that case, it is not
345  // even legal to call this method. The reasonable thing to do in
346  // that case is nothing.
347  if (plan_.getComm().is_null ()) {
348  return;
349  }
350  const int myRank = plan_.getComm()->getRank ();
351  const int numProcs = plan_.getComm()->getSize ();
352 
353  // Only Process 0 should touch the output stream, but this method
354  // in general may need to do communication. Thus, we may need to
355  // preserve the current tab level across multiple "if (myRank ==
356  // 0) { ... }" inner scopes. This is why we sometimes create
357  // OSTab instances by pointer, instead of by value. We only need
358  // to create them by pointer if the tab level must persist through
359  // multiple inner scopes.
360  Teuchos::RCP<Teuchos::OSTab> tab0, tab1;
361 
362  if (myRank == 0) {
363  // At every verbosity level but VERB_NONE, Process 0 prints.
364  // By convention, describe() always begins with a tab before
365  // printing.
366  tab0 = Teuchos::rcp (new Teuchos::OSTab (out));
367  // We quote the class name because it contains colons.
368  // This makes the output valid YAML.
369  out << "\"Tpetra::Distributor\":" << endl;
370  tab1 = Teuchos::rcp (new Teuchos::OSTab (out));
371 
372  const std::string label = this->getObjectLabel ();
373  if (label != "") {
374  out << "Label: " << label << endl;
375  }
376  out << "Number of processes: " << numProcs << endl
377  << "How initialized: "
378  << Details::DistributorHowInitializedEnumToString (plan_.howInitialized())
379  << endl;
380  {
381  out << "Parameters: " << endl;
382  Teuchos::OSTab tab2 (out);
383  out << "\"Send type\": "
384  << DistributorSendTypeEnumToString (plan_.getSendType()) << endl
385  << "\"Debug\": " << (verbose_ ? "true" : "false") << endl;
386  }
387  } // if myRank == 0
388 
389  // This is collective over the Map's communicator.
390  if (vl > VERB_LOW) {
391  const std::string lclStr = this->localDescribeToString (vl);
392  Tpetra::Details::gathervPrint (out, lclStr, *plan_.getComm());
393  }
394 
395  out << "Reverse Distributor:";
396  if (reverseDistributor_.is_null ()) {
397  out << " null" << endl;
398  }
399  else {
400  out << endl;
401  reverseDistributor_->describe (out, vl);
402  }
403  }
404 
405  size_t
407  createFromSends(const Teuchos::ArrayView<const int>& exportProcIDs)
408  {
409  return plan_.createFromSends(exportProcIDs);
410  }
411 
412  void
414  createFromSendsAndRecvs (const Teuchos::ArrayView<const int>& exportProcIDs,
415  const Teuchos::ArrayView<const int>& remoteProcIDs)
416  {
417  plan_.createFromSendsAndRecvs(exportProcIDs, remoteProcIDs);
418  }
419 
420 } // namespace Tpetra
size_t getNumReceives() const
The number of processes from which we will receive data.
std::string description() const
Return a one-line description of this object.
Teuchos::RCP< Distributor > getReverse(bool create=true) const
A reverse communication plan Distributor.
Declaration of a function that prints strings from each process.
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const
List of valid Distributor parameters.
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
void gathervPrint(std::ostream &out, const std::string &s, const Teuchos::Comm< int > &comm)
On Process 0 in the given communicator, print strings from each process in that communicator, in rank order.
Teuchos::Array< EDistributorSendType > distributorSendTypeEnums()
Valid enum values of distributor send types.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
Sets up and executes a communication plan for a Tpetra DistObject.
static bool verbose()
Whether Tpetra is in verbose mode.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters.
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
std::string DistributorHowInitializedEnumToString(EDistributorHowInitialized how)
Convert an EDistributorHowInitialized enum value to a string.
Stand-alone utility functions and macros.
Teuchos::Array< std::string > distributorSendTypes()
Valid string values for Distributor&#39;s &quot;Send type&quot; parameter.
size_t getNumSends() const
The number of processes to which we will send data.
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
size_t getMaxSendLength() const
Maximum number of values this process will send to another single process.
static std::string defaultSendType()
Default send type.
std::unique_ptr< std::string > createPrefix(const int myRank, const char prefix[])
Create string prefix for each line of verbose output.
Definition: Tpetra_Util.cpp:71
Distributor(const Teuchos::RCP< const Teuchos::Comm< int > > &comm)
Construct using the specified communicator and default parameters.
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra&#39;s behavior.