39 #include "Tpetra_Distributor.hpp"
43 #include "Tpetra_Details_makeValidVerboseStream.hpp"
44 #include "Teuchos_StandardParameterEntryValidators.hpp"
45 #include "Teuchos_VerboseObjectParameterListHelpers.hpp"
55 const bool tpetraDistributorDebugDefault =
false;
58 Teuchos::Array<std::string>
61 Teuchos::Array<std::string> sendTypes;
62 sendTypes.push_back (
"Isend");
63 sendTypes.push_back (
"Send");
64 sendTypes.push_back (
"Alltoall");
65 #if defined(HAVE_TPETRACORE_MPI_ADVANCE)
66 sendTypes.push_back (
"MpiAdvanceAlltoall");
67 sendTypes.push_back (
"MpiAdvanceNbralltoallv");
73 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
74 const Teuchos::RCP<Teuchos::FancyOStream>& ,
75 const Teuchos::RCP<Teuchos::ParameterList>& plist)
82 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm)
87 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
88 const Teuchos::RCP<Teuchos::FancyOStream>& out)
93 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
94 const Teuchos::RCP<Teuchos::ParameterList>& plist)
100 : plan_(distributor.plan_)
101 , actor_(distributor.actor_)
102 , verbose_ (distributor.verbose_)
103 , reverseDistributor_ (distributor.reverseDistributor_)
105 using Teuchos::ParameterList;
109 RCP<const ParameterList> rhsList = distributor.getParameterList ();
110 RCP<ParameterList> newList = rhsList.is_null () ? Teuchos::null :
111 Teuchos::parameterList (*rhsList);
116 using Teuchos::ParameterList;
117 using Teuchos::parameterList;
120 std::swap (plan_, rhs.plan_);
121 std::swap (actor_, rhs.actor_);
122 std::swap (verbose_, rhs.verbose_);
123 std::swap (reverseDistributor_, rhs.reverseDistributor_);
127 RCP<ParameterList> lhsList = this->getNonconstParameterList ();
128 RCP<ParameterList> rhsList = rhs.getNonconstParameterList ();
129 if (lhsList.getRawPtr () == rhsList.getRawPtr () && ! rhsList.is_null ()) {
130 rhsList = parameterList (*rhsList);
132 if (! rhsList.is_null ()) {
133 this->setMyParamList (rhsList);
135 if (! lhsList.is_null ()) {
136 rhs.setMyParamList (lhsList);
144 Distributor::getVerbose()
150 std::unique_ptr<std::string>
152 createPrefix(
const char methodName[])
const
155 plan_.getComm().getRawPtr(),
"Distributor", methodName);
162 using ::Tpetra::Details::Behavior;
163 using Teuchos::FancyOStream;
164 using Teuchos::getIntegralValue;
165 using Teuchos::includesVerbLevel;
166 using Teuchos::ParameterList;
167 using Teuchos::parameterList;
171 if (! plist.is_null()) {
173 plist->validateParametersAndSetDefaults (*validParams);
177 this->setMyParamList (plist);
179 RCP<ParameterList> planParams(plist);
180 planParams->remove(
"Debug",
false);
181 planParams->remove(
"VerboseObject",
false);
182 plan_.setParameterList(planParams);
186 Teuchos::RCP<const Teuchos::ParameterList>
189 using Teuchos::Array;
190 using Teuchos::ParameterList;
191 using Teuchos::parameterList;
193 using Teuchos::setStringToIntegralParameter;
195 const bool debug = tpetraDistributorDebugDefault;
198 const std::string defaultSendType (
"Send");
199 Array<Details::EDistributorSendType> sendTypeEnums;
200 sendTypeEnums.push_back (Details::DISTRIBUTOR_ISEND);
201 sendTypeEnums.push_back (Details::DISTRIBUTOR_SEND);
202 sendTypeEnums.push_back (Details::DISTRIBUTOR_ALLTOALL);
203 #if defined(HAVE_TPETRACORE_MPI_ADVANCE)
204 sendTypeEnums.push_back(Details::DISTRIBUTOR_MPIADVANCE_ALLTOALL);
205 sendTypeEnums.push_back(Details::DISTRIBUTOR_MPIADVANCE_NBRALLTOALLV);
208 RCP<ParameterList> plist = parameterList (
"Tpetra::Distributor");
209 setStringToIntegralParameter<Details::EDistributorSendType> (
"Send type",
210 defaultSendType,
"When using MPI, the variant of send to use in "
211 "do[Reverse]Posts()", sendTypes(), sendTypeEnums(), plist.getRawPtr());
212 plist->set (
"Debug", debug,
"Whether to print copious debugging output on "
214 plist->set (
"Timer Label",
"",
"Label for Time Monitor output");
222 Teuchos::setupVerboseObjectSublist (&*plist);
223 return Teuchos::rcp_const_cast<
const ParameterList> (plist);
228 {
return plan_.getTotalReceiveLength(); }
231 {
return plan_.getNumReceives(); }
234 {
return plan_.hasSelfMessage(); }
237 {
return plan_.getNumSends(); }
240 {
return plan_.getMaxSendLength(); }
243 {
return plan_.getProcsFrom(); }
246 {
return plan_.getLengthsFrom(); }
249 {
return plan_.getProcsTo(); }
252 {
return plan_.getLengthsTo(); }
254 Teuchos::RCP<Distributor>
256 if (reverseDistributor_.is_null () && create) {
257 createReverseDistributor ();
259 TEUCHOS_TEST_FOR_EXCEPTION
260 (reverseDistributor_.is_null () && create, std::logic_error,
"The reverse "
261 "Distributor is null after createReverseDistributor returned. "
262 "Please report this bug to the Tpetra developers.");
263 return reverseDistributor_;
268 Distributor::createReverseDistributor()
const
270 reverseDistributor_ = Teuchos::rcp(
new Distributor(plan_.getComm()));
271 reverseDistributor_->plan_ = *plan_.getReversePlan();
272 reverseDistributor_->verbose_ = verbose_;
288 reverseDistributor_->reverseDistributor_ = Teuchos::null;
294 actor_.doWaits(plan_);
299 if (! reverseDistributor_.is_null()) {
300 reverseDistributor_->doWaits();
305 std::ostringstream out;
307 out <<
"\"Tpetra::Distributor\": {";
308 const std::string label = this->getObjectLabel ();
310 out <<
"Label: " << label <<
", ";
312 out <<
"How initialized: "
316 << DistributorSendTypeEnumToString (plan_.getSendType())
317 <<
", Debug: " << (verbose_ ?
"true" :
"false")
324 localDescribeToString (
const Teuchos::EVerbosityLevel vl)
const
326 using Teuchos::toString;
327 using Teuchos::VERB_HIGH;
328 using Teuchos::VERB_EXTREME;
332 if (vl <= Teuchos::VERB_LOW || plan_.getComm().is_null ()) {
333 return std::string ();
336 auto outStringP = Teuchos::rcp (
new std::ostringstream ());
337 auto outp = Teuchos::getFancyOStream (outStringP);
338 Teuchos::FancyOStream& out = *outp;
340 const int myRank = plan_.getComm()->getRank ();
341 const int numProcs = plan_.getComm()->getSize ();
342 out <<
"Process " << myRank <<
" of " << numProcs <<
":" << endl;
343 Teuchos::OSTab tab1 (out);
347 if (vl == VERB_HIGH || vl == VERB_EXTREME) {
348 out <<
"procsTo: " << toString (plan_.getProcsTo()) << endl;
349 out <<
"lengthsTo: " << toString (plan_.getLengthsTo()) << endl;
352 if (vl == VERB_EXTREME) {
353 out <<
"startsTo: " << toString (plan_.getStartsTo()) << endl;
354 out <<
"indicesTo: " << toString (plan_.getIndicesTo()) << endl;
356 if (vl == VERB_HIGH || vl == VERB_EXTREME) {
359 out <<
"lengthsFrom: " << toString (plan_.getLengthsFrom()) << endl;
360 out <<
"procsFrom: " << toString (plan_.getProcsFrom()) << endl;
364 return outStringP->str ();
370 const Teuchos::EVerbosityLevel verbLevel)
const
373 using Teuchos::VERB_DEFAULT;
374 using Teuchos::VERB_NONE;
375 using Teuchos::VERB_LOW;
376 using Teuchos::VERB_MEDIUM;
377 using Teuchos::VERB_HIGH;
378 using Teuchos::VERB_EXTREME;
379 const Teuchos::EVerbosityLevel vl =
380 (verbLevel == VERB_DEFAULT) ? VERB_LOW : verbLevel;
382 if (vl == VERB_NONE) {
390 if (plan_.getComm().is_null ()) {
393 const int myRank = plan_.getComm()->getRank ();
394 const int numProcs = plan_.getComm()->getSize ();
403 Teuchos::RCP<Teuchos::OSTab> tab0, tab1;
409 tab0 = Teuchos::rcp (
new Teuchos::OSTab (out));
412 out <<
"\"Tpetra::Distributor\":" << endl;
413 tab1 = Teuchos::rcp (
new Teuchos::OSTab (out));
415 const std::string label = this->getObjectLabel ();
417 out <<
"Label: " << label << endl;
419 out <<
"Number of processes: " << numProcs << endl
420 <<
"How initialized: "
424 out <<
"Parameters: " << endl;
425 Teuchos::OSTab tab2 (out);
426 out <<
"\"Send type\": "
427 << DistributorSendTypeEnumToString (plan_.getSendType()) << endl
428 <<
"\"Debug\": " << (verbose_ ?
"true" :
"false") << endl;
434 const std::string lclStr = this->localDescribeToString (vl);
438 out <<
"Reverse Distributor:";
439 if (reverseDistributor_.is_null ()) {
440 out <<
" null" << endl;
444 reverseDistributor_->describe (out, vl);
452 return plan_.createFromSends(exportProcIDs);
458 const Teuchos::ArrayView<const int>& remoteProcIDs)
460 plan_.createFromSendsAndRecvs(exportProcIDs, remoteProcIDs);
size_t getNumReceives() const
The number of processes from which we will receive data.
std::string description() const
Return a one-line description of this object.
Teuchos::RCP< Distributor > getReverse(bool create=true) const
A reverse communication plan Distributor.
Declaration of a function that prints strings from each process.
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const
List of valid Distributor parameters.
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
void gathervPrint(std::ostream &out, const std::string &s, const Teuchos::Comm< int > &comm)
On Process 0 in the given communicator, print strings from each process in that communicator, in rank order.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
Sets up and executes a communication plan for a Tpetra DistObject.
static bool verbose()
Whether Tpetra is in verbose mode.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters.
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
Teuchos::Array< std::string > distributorSendTypes()
Valid values for Distributor's "Send type" parameter.
std::string DistributorHowInitializedEnumToString(EDistributorHowInitialized how)
Convert an EDistributorHowInitialized enum value to a string.
Stand-alone utility functions and macros.
size_t getNumSends() const
The number of processes to which we will send data.
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
size_t getMaxSendLength() const
Maximum number of values this process will send to another single process.
std::unique_ptr< std::string > createPrefix(const int myRank, const char prefix[])
Create string prefix for each line of verbose output.
Distributor(const Teuchos::RCP< const Teuchos::Comm< int > > &comm)
Construct using the specified communicator and default parameters.
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.