Teuchos Package Browser (Single Doxygen Collection)  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Teuchos_CommHelpers.cpp
Go to the documentation of this file.
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Teuchos: Common Tools Package
5 // Copyright (2004) Sandia Corporation
6 //
7 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8 // license for use of this work by or on behalf of the U.S. Government.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ***********************************************************************
40 // @HEADER
41 
42 #include "Teuchos_CommHelpers.hpp"
43 #ifdef HAVE_TEUCHOS_MPI
46 #endif // HAVE_TEUCHOS_MPI
47 #ifdef HAVE_TEUCHOSCORE_CXX11
48 # include <memory>
49 #endif
50 
51 namespace Teuchos {
52 
53 #ifdef HAVE_TEUCHOS_MPI
54 namespace Details {
55 
56 std::string getMpiErrorString (const int errCode) {
57  // Space for storing the error string returned by MPI.
58  // Leave room for null termination, since I don't know if MPI does this.
59  char errString [MPI_MAX_ERROR_STRING+1];
60  int errStringLen = MPI_MAX_ERROR_STRING; // output argument
61  (void) MPI_Error_string (errCode, errString, &errStringLen);
62  // errStringLen on output is the number of characters written.
63  // I'm not sure (the MPI 3.0 Standard doesn't say) if this
64  // includes the '\0', so I'll make sure. We reserved space for
65  // the extra '\0' if needed.
66  if (errString[errStringLen-1] != '\0') {
67  errString[errStringLen] = '\0';
68  }
69  return std::string (errString); // This copies the original string.
70 }
71 
72 } // namespace Details
73 #endif // HAVE_TEUCHOS_MPI
74 
75 namespace { // (anonymous)
76 
84 template<class T>
85 void
86 reduceAllImpl (const Comm<int>& comm,
87  const EReductionType reductType,
88  const int count,
89  const T sendBuffer[],
90  T globalReducts[])
91 {
92 #ifdef HAVE_TEUCHOS_MPI
93  using Teuchos::Details::MpiTypeTraits;
94 
95  // mfh 17 Oct 2012: Even in an MPI build, Comm might be either a
96  // SerialComm or an MpiComm. If it's something else, we fall back
97  // to the most general implementation.
98  const MpiComm<int>* mpiComm = dynamic_cast<const MpiComm<int>* > (&comm);
99  if (mpiComm == NULL) {
100  // Is it a SerialComm?
101  const SerialComm<int>* serialComm = dynamic_cast<const SerialComm<int>* > (&comm);
102  if (serialComm == NULL) {
103  // We don't know what kind of Comm we have, so fall back to the
104  // most general implementation.
105 #ifdef HAVE_TEUCHOSCORE_CXX11
106  std::unique_ptr<ValueTypeReductionOp<int, T> >
107 #else
108  std::auto_ptr<ValueTypeReductionOp<int, T> >
109 #endif
110  reductOp (createOp<int, T> (reductType));
111  reduceAll (comm, *reductOp, count, sendBuffer, globalReducts);
112  }
113  else { // It's a SerialComm; there is only 1 process, so just copy.
114  std::copy (sendBuffer, sendBuffer + count, globalReducts);
115  }
116  } else { // It's an MpiComm. Invoke MPI directly.
117  MPI_Op rawMpiOp = ::Teuchos::Details::getMpiOpForEReductionType (reductType);
118  MPI_Comm rawMpiComm = * (mpiComm->getRawMpiComm ());
119  T t;
120  MPI_Datatype rawMpiType = MpiTypeTraits<T>::getType (t);
121 
122  int err = MPI_SUCCESS;
123  if (sendBuffer == globalReducts) {
124  // NOTE (mfh 31 May 2017) This is only safe if the communicator
125  // is NOT an intercomm. The usual case is that communicators
126  // are intracomms.
127  err = MPI_Allreduce (MPI_IN_PLACE, globalReducts,
128  count, rawMpiType, rawMpiOp, rawMpiComm);
129  }
130  else {
131  err = MPI_Allreduce (const_cast<T*> (sendBuffer), globalReducts,
132  count, rawMpiType, rawMpiOp, rawMpiComm);
133  }
135  err != MPI_SUCCESS,
136  std::runtime_error,
137  "MPI_Allreduce failed with the following error: "
138  << ::Teuchos::Details::getMpiErrorString (err));
139  }
140 #else
141  // We've built without MPI, so just assume it's a SerialComm and copy the data.
142  std::copy (sendBuffer, sendBuffer + count, globalReducts);
143 #endif // HAVE_TEUCHOS_MPI
144 }
145 
146 
154 template<class T>
155 void
156 gatherImpl (const T sendBuf[],
157  const int sendCount,
158  T recvBuf[],
159  const int recvCount,
160  const int root,
161  const Comm<int>& comm)
162 {
163 #ifdef HAVE_TEUCHOS_MPI
164  using Teuchos::Details::MpiTypeTraits;
165 
166  // mfh 17 Oct 2012: Even in an MPI build, Comm might be either a
167  // SerialComm or an MpiComm. If it's something else, we fall back
168  // to the most general implementation.
169  const MpiComm<int>* mpiComm = dynamic_cast<const MpiComm<int>* > (&comm);
170  if (mpiComm == NULL) {
171  // Is it a SerialComm?
172  const SerialComm<int>* serialComm = dynamic_cast<const SerialComm<int>* > (&comm);
173  if (serialComm == NULL) {
174  // We don't know what kind of Comm we have, so fall back to the
175  // most general implementation.
176  gather<int, T> (sendBuf, sendCount, recvBuf, recvCount, root, comm);
177  }
178  else { // It's a SerialComm; there is only 1 process, so just copy.
179  std::copy (sendBuf, sendBuf + sendCount, recvBuf);
180  }
181  } else { // It's an MpiComm. Invoke MPI directly.
182  MPI_Comm rawMpiComm = * (mpiComm->getRawMpiComm ());
183  T t;
184  MPI_Datatype rawMpiType = MpiTypeTraits<T>::getType (t);
185  const int err = MPI_Gather (const_cast<T*> (sendBuf), sendCount, rawMpiType,
186  recvBuf, recvCount, rawMpiType,
187  root, rawMpiComm);
189  err != MPI_SUCCESS,
190  std::runtime_error,
191  "MPI_Gather failed with the following error: "
192  << ::Teuchos::Details::getMpiErrorString (err));
193  }
194 #else
195  // We've built without MPI, so just assume it's a SerialComm and copy the data.
196  std::copy (sendBuf, sendBuf + sendCount, recvBuf);
197 #endif // HAVE_TEUCHOS_MPI
198 }
199 
200 
208 template<class T>
209 void
210 scatterImpl (const T sendBuf[],
211  const int sendCount,
212  T recvBuf[],
213  const int recvCount,
214  const int root,
215  const Comm<int>& comm)
216 {
217 #ifdef HAVE_TEUCHOS_MPI
218  using Teuchos::Details::MpiTypeTraits;
219 
220  // mfh 17 Oct 2012: Even in an MPI build, Comm might be either a
221  // SerialComm or an MpiComm. If it's something else, we fall back
222  // to the most general implementation.
223  const MpiComm<int>* mpiComm = dynamic_cast<const MpiComm<int>* > (&comm);
224  if (mpiComm == NULL) {
225  // Is it a SerialComm?
226  const SerialComm<int>* serialComm = dynamic_cast<const SerialComm<int>* > (&comm);
227  if (serialComm == NULL) {
228  // We don't know what kind of Comm we have, so fall back to the
229  // most general implementation.
230  scatter<int, T> (sendBuf, sendCount, recvBuf, recvCount, root, comm);
231  }
232  else { // It's a SerialComm; there is only 1 process, so just copy.
233  std::copy (sendBuf, sendBuf + sendCount, recvBuf);
234  }
235  } else { // It's an MpiComm. Invoke MPI directly.
236  MPI_Comm rawMpiComm = * (mpiComm->getRawMpiComm ());
237  T t;
238  MPI_Datatype rawMpiType = MpiTypeTraits<T>::getType (t);
239  const int err =
240  MPI_Scatter (const_cast<T*> (sendBuf), sendCount, rawMpiType,
241  recvBuf, recvCount, rawMpiType,
242  root, rawMpiComm);
244  (err != MPI_SUCCESS, std::runtime_error,
245  "MPI_Scatter failed with the following error: "
246  << ::Teuchos::Details::getMpiErrorString (err));
247  }
248 #else
249  // We've built without MPI, so just assume it's a SerialComm and
250  // copy the data.
251  std::copy (sendBuf, sendBuf + sendCount, recvBuf);
252 #endif // HAVE_TEUCHOS_MPI
253 }
254 
255 
263 template<class T>
264 void
265 reduceImpl (const T sendBuf[],
266  T recvBuf[],
267  const int count,
268  const EReductionType reductType,
269  const int root,
270  const Comm<int>& comm)
271 {
272 #ifdef HAVE_TEUCHOS_MPI
273  using Teuchos::Details::MpiTypeTraits;
274 
275  // mfh 17 Oct 2012: Even in an MPI build, Comm might be either a
276  // SerialComm or an MpiComm. If it's something else, we fall back
277  // to the most general implementation.
278  const MpiComm<int>* mpiComm = dynamic_cast<const MpiComm<int>* > (&comm);
279  if (mpiComm == NULL) {
280  // Is it a SerialComm?
281  const SerialComm<int>* serialComm = dynamic_cast<const SerialComm<int>* > (&comm);
282  if (serialComm == NULL) {
283  // We don't know what kind of Comm we have, so fall back to the
284  // most general implementation.
285  reduce<int, T> (sendBuf, recvBuf, count, reductType, root, comm);
286  }
287  else { // It's a SerialComm; there is only 1 process, so just copy.
288  std::copy (sendBuf, sendBuf + count, recvBuf);
289  }
290  } else { // It's an MpiComm. Invoke MPI directly.
291  MPI_Op rawMpiOp = ::Teuchos::Details::getMpiOpForEReductionType (reductType);
292  MPI_Comm rawMpiComm = * (mpiComm->getRawMpiComm ());
293  T t;
294  MPI_Datatype rawMpiType = MpiTypeTraits<T>::getType (t);
295  const int err = MPI_Reduce (const_cast<T*> (sendBuf), recvBuf, count,
296  rawMpiType, rawMpiOp, root, rawMpiComm);
298  (err != MPI_SUCCESS, std::runtime_error, "MPI_Reduce failed with the "
299  "following error: " << ::Teuchos::Details::getMpiErrorString (err));
300  }
301 #else
302  // We've built without MPI, so just assume it's a SerialComm and copy the data.
303  std::copy (sendBuf, sendBuf + count, recvBuf);
304 #endif // HAVE_TEUCHOS_MPI
305 }
306 
307 
315 template<class T>
316 void
317 gathervImpl (const T sendBuf[],
318  const int sendCount,
319  T recvBuf[],
320  const int recvCounts[],
321  const int displs[],
322  const int root,
323  const Comm<int>& comm)
324 {
325 #ifdef HAVE_TEUCHOS_MPI
326  using Teuchos::Details::MpiTypeTraits;
327 
328  // mfh 17 Oct 2012: Even in an MPI build, Comm might be either a
329  // SerialComm or an MpiComm. If it's something else, we fall back
330  // to the most general implementation.
331  const MpiComm<int>* mpiComm = dynamic_cast<const MpiComm<int>* > (&comm);
332  if (mpiComm == NULL) {
333  // Is it a SerialComm?
334  const SerialComm<int>* serialComm = dynamic_cast<const SerialComm<int>* > (&comm);
335  if (serialComm == NULL) {
336  // We don't know what kind of Comm we have, so fall back to the
337  // most general implementation.
338  gatherv<int, T> (sendBuf, sendCount, recvBuf, recvCounts, displs, root, comm);
339  }
340  else { // It's a SerialComm; there is only 1 process, so just copy.
342  recvCounts[0] > sendCount, std::invalid_argument,
343  "Teuchos::gatherv: If the input communicator contains only one "
344  "process, then you cannot receive more entries than you send. "
345  "You aim to receive " << recvCounts[0] << " entries, but to send "
346  << sendCount << " entries.");
347  // Serial communicator case: just copy. recvCounts[0] is the
348  // amount to receive, so it's the amount to copy. Start writing
349  // to recvbuf at the offset displs[0].
350  std::copy (sendBuf, sendBuf + recvCounts[0], recvBuf + displs[0]);
351  }
352  } else { // It's an MpiComm. Invoke MPI directly.
353  MPI_Comm rawMpiComm = * (mpiComm->getRawMpiComm ());
354  T t;
355  MPI_Datatype rawMpiType = MpiTypeTraits<T>::getType (t);
356  const int err = MPI_Gatherv (const_cast<T*> (sendBuf),
357  sendCount,
358  rawMpiType,
359  recvBuf,
360  const_cast<int*> (recvCounts),
361  const_cast<int*> (displs),
362  rawMpiType,
363  root,
364  rawMpiComm);
366  err != MPI_SUCCESS,
367  std::runtime_error,
368  "MPI_Gatherv failed with the following error: "
369  << ::Teuchos::Details::getMpiErrorString (err));
370  }
371 #else
372  // We've built without MPI, so just assume it's a SerialComm and copy the data.
374  recvCounts[0] > sendCount, std::invalid_argument,
375  "Teuchos::gatherv: If the input communicator contains only one "
376  "process, then you cannot receive more entries than you send. "
377  "You aim to receive " << recvCounts[0] << " entries, but to send "
378  << sendCount << " entries.");
379  // Serial communicator case: just copy. recvCounts[0] is the
380  // amount to receive, so it's the amount to copy. Start writing
381  // to recvbuf at the offset displs[0].
382  std::copy (sendBuf, sendBuf + recvCounts[0], recvBuf + displs[0]);
383 #endif // HAVE_TEUCHOS_MPI
384 }
385 
391 template<typename Packet>
392 RCP<Teuchos::CommRequest<int> >
393 ireceiveGeneral(const Comm<int>& comm,
394  const ArrayRCP<Packet> &recvBuffer,
395  const int sourceRank)
396 {
398  "Teuchos::ireceive<int, " << "," << TypeNameTraits<Packet>::name ()
399  << "> ( value type )"
400  );
401  ValueTypeSerializationBuffer<int, Packet>
402  charRecvBuffer (recvBuffer.size (), recvBuffer.getRawPtr ());
403  RCP<CommRequest<int> > commRequest =
404  comm.ireceive (charRecvBuffer.getCharBufferView (), sourceRank);
405  set_extra_data (recvBuffer, "buffer", inOutArg (commRequest));
406  return commRequest;
407 }
408 
411 template<typename Packet>
412 RCP<Teuchos::CommRequest<int> >
413 ireceiveGeneral (const ArrayRCP<Packet> &recvBuffer,
414  const int sourceRank,
415  const int tag,
416  const Comm<int>& comm)
417 {
419  "Teuchos::ireceive<int, " << "," << TypeNameTraits<Packet>::name ()
420  << "> ( value type )"
421  );
422  ValueTypeSerializationBuffer<int, Packet>
423  charRecvBuffer (recvBuffer.size (), recvBuffer.getRawPtr ());
424  RCP<CommRequest<int> > commRequest =
425  comm.ireceive (charRecvBuffer.getCharBufferView (), sourceRank, tag);
426  set_extra_data (recvBuffer, "buffer", inOutArg (commRequest));
427  return commRequest;
428 }
429 
442 template<class T>
443 RCP<CommRequest<int> >
444 ireceiveImpl (const Comm<int>& comm,
445  const ArrayRCP<T>& recvBuffer,
446  const int sourceRank)
447 {
448 #ifdef HAVE_TEUCHOS_MPI
449  using Teuchos::Details::MpiTypeTraits;
450 
451  // Even in an MPI build, Comm might be either a SerialComm or an
452  // MpiComm. If it's something else, we fall back to the most
453  // general implementation.
454  const MpiComm<int>* mpiComm = dynamic_cast<const MpiComm<int>* > (&comm);
455  if (mpiComm == NULL) {
456  // Is it a SerialComm?
457  const SerialComm<int>* serialComm = dynamic_cast<const SerialComm<int>* > (&comm);
458  if (serialComm == NULL) {
459  // We don't know what kind of Comm we have, so fall back to the
460  // most general implementation.
461  return ireceiveGeneral<T> (comm, recvBuffer, sourceRank);
462  }
463  else { // SerialComm doesn't implement ireceive anyway.
465  true,
466  std::logic_error,
467  "ireceiveImpl: Not implemented for a serial communicator.");
468  }
469  }
470  else { // It's an MpiComm. Invoke MPI directly.
471  MPI_Comm rawComm = * (mpiComm->getRawMpiComm ());
472  T t;
473  MPI_Datatype rawType = MpiTypeTraits<T>::getType (t);
474  T* rawRecvBuf = recvBuffer.getRawPtr ();
475  const int count = as<int> (recvBuffer.size ());
476  const int tag = mpiComm->getTag ();
477  MPI_Request rawRequest = MPI_REQUEST_NULL;
478  const int err = MPI_Irecv (rawRecvBuf, count, rawType, sourceRank, tag,
479  rawComm, &rawRequest);
481  err != MPI_SUCCESS, std::runtime_error,
482  "MPI_Irecv failed with the following error: "
483  << ::Teuchos::Details::getMpiErrorString (err));
484 
485  ArrayRCP<const char> buf =
486  arcp_const_cast<const char> (arcp_reinterpret_cast<char> (recvBuffer));
487  RCP<Details::MpiCommRequest> req (new Details::MpiCommRequest (rawRequest, buf));
488  return rcp_implicit_cast<CommRequest<int> > (req);
489  }
490 #else
492  true,
493  std::logic_error,
494  "ireceiveImpl: Not implemented for a serial communicator.");
495 
496  // NOTE (mfh 15 Sep 2014): Most compilers have figured out that the
497  // return statement below is unreachable. Some older compilers
498  // might not realize this. That's why the return statement was put
499  // there, so that those compilers don't warn that this function
500  // doesn't return a value. If it's a choice between one warning and
501  // another, I would prefer the choice that produces less code and
502  // doesn't have unreachable code (which never gets tested).
503 
504  //return null; // Guard to avoid compiler warning about not returning a value.
505 #endif // HAVE_TEUCHOS_MPI
506 }
507 
510 template<class T>
511 RCP<CommRequest<int> >
512 ireceiveImpl (const ArrayRCP<T>& recvBuffer,
513  const int sourceRank,
514  const int tag,
515  const Comm<int>& comm)
516 {
517 #ifdef HAVE_TEUCHOS_MPI
518  using Teuchos::Details::MpiTypeTraits;
519 
520  // Even in an MPI build, Comm might be either a SerialComm or an
521  // MpiComm. If it's something else, we fall back to the most
522  // general implementation.
523  const MpiComm<int>* mpiComm = dynamic_cast<const MpiComm<int>* > (&comm);
524  if (mpiComm == NULL) {
525  // Is it a SerialComm?
526  const SerialComm<int>* serialComm = dynamic_cast<const SerialComm<int>* > (&comm);
527  if (serialComm == NULL) {
528  // We don't know what kind of Comm we have, so fall back to the
529  // most general implementation.
530  return ireceiveGeneral<T> (recvBuffer, sourceRank, tag, comm);
531  }
532  else { // SerialComm doesn't implement ireceive anyway.
534  true,
535  std::logic_error,
536  "ireceiveImpl: Not implemented for a serial communicator.");
537  }
538  }
539  else { // It's an MpiComm. Invoke MPI directly.
540  MPI_Comm rawComm = * (mpiComm->getRawMpiComm ());
541  T t;
542  MPI_Datatype rawType = MpiTypeTraits<T>::getType (t);
543  T* rawRecvBuf = recvBuffer.getRawPtr ();
544  const int count = as<int> (recvBuffer.size ());
545  MPI_Request rawRequest = MPI_REQUEST_NULL;
546  const int err = MPI_Irecv (rawRecvBuf, count, rawType, sourceRank, tag,
547  rawComm, &rawRequest);
549  err != MPI_SUCCESS, std::runtime_error,
550  "MPI_Irecv failed with the following error: "
551  << ::Teuchos::Details::getMpiErrorString (err));
552 
553  ArrayRCP<const char> buf =
554  arcp_const_cast<const char> (arcp_reinterpret_cast<char> (recvBuffer));
555  RCP<Details::MpiCommRequest> req (new Details::MpiCommRequest (rawRequest, buf));
556  return rcp_implicit_cast<CommRequest<int> > (req);
557  }
558 #else
560  true,
561  std::logic_error,
562  "ireceiveImpl: Not implemented for a serial communicator.");
563 
564  return null; // Guard to avoid compiler warning about not returning a value.
565 #endif // HAVE_TEUCHOS_MPI
566 }
567 
573 template<class T>
574 void
575 sendGeneral (const Comm<int>& comm,
576  const int count,
577  const T sendBuffer[],
578  const int destRank)
579 {
581  "Teuchos::send<int, " << TypeNameTraits<T>::name () << ">");
582  ConstValueTypeSerializationBuffer<int,T> charSendBuffer (count, sendBuffer);
583  comm.send (charSendBuffer.getBytes (),
584  charSendBuffer.getCharBuffer (),
585  destRank);
586 }
587 
590 template<class T>
591 void
592 sendGeneral (const T sendBuffer[],
593  const int count,
594  const int destRank,
595  const int tag,
596  const Comm<int>& comm)
597 {
599  "Teuchos::send<int, " << TypeNameTraits<T>::name () << ">");
600  ConstValueTypeSerializationBuffer<int,T> charSendBuffer (count, sendBuffer);
601  comm.send (charSendBuffer.getBytes (),
602  charSendBuffer.getCharBuffer (),
603  destRank, tag);
604 }
605 
618 template<class T>
619 void
620 sendImpl (const Comm<int>& comm,
621  const int count,
622  const T sendBuffer[],
623  const int destRank)
624 {
625 #ifdef HAVE_TEUCHOS_MPI
626  using Teuchos::Details::MpiTypeTraits;
627 
628  // Even in an MPI build, Comm might be either a SerialComm or an
629  // MpiComm. If it's something else, we fall back to the most
630  // general implementation.
631  const MpiComm<int>* mpiComm = dynamic_cast<const MpiComm<int>* > (&comm);
632  if (mpiComm == NULL) {
633  // Is it a SerialComm?
634  const SerialComm<int>* serialComm = dynamic_cast<const SerialComm<int>* > (&comm);
635  if (serialComm == NULL) {
636  // We don't know what kind of Comm we have, so fall back to the
637  // most general implementation.
638  sendGeneral<T> (comm, count, sendBuffer, destRank);
639  }
640  else { // SerialComm doesn't implement send correctly anyway.
642  true,
643  std::logic_error,
644  "sendImpl: Not implemented for a serial communicator.");
645  }
646  }
647  else { // It's an MpiComm. Invoke MPI directly.
648  MPI_Comm rawComm = * (mpiComm->getRawMpiComm ());
649  T t;
650  MPI_Datatype rawType = MpiTypeTraits<T>::getType (t);
651  T* rawBuf = const_cast<T*> (sendBuffer);
652  const int tag = mpiComm->getTag ();
653  const int err = MPI_Send (rawBuf, count, rawType, destRank, tag, rawComm);
655  err != MPI_SUCCESS,
656  std::runtime_error,
657  "MPI_Send failed with the following error: "
658  << ::Teuchos::Details::getMpiErrorString (err));
659  }
660 #else
662  true,
663  std::logic_error,
664  "sendImpl: Not implemented for a serial communicator.");
665 #endif // HAVE_TEUCHOS_MPI
666 }
667 
670 template<class T>
671 void
672 sendImpl (const T sendBuffer[],
673  const int count,
674  const int destRank,
675  const int tag,
676  const Comm<int>& comm)
677 {
678 #ifdef HAVE_TEUCHOS_MPI
679  using Teuchos::Details::MpiTypeTraits;
680 
681  // Even in an MPI build, Comm might be either a SerialComm or an
682  // MpiComm. If it's something else, we fall back to the most
683  // general implementation.
684  const MpiComm<int>* mpiComm = dynamic_cast<const MpiComm<int>* > (&comm);
685  if (mpiComm == NULL) {
686  // Is it a SerialComm?
687  const SerialComm<int>* serialComm = dynamic_cast<const SerialComm<int>* > (&comm);
688  if (serialComm == NULL) {
689  // We don't know what kind of Comm we have, so fall back to the
690  // most general implementation.
691  sendGeneral<T> (sendBuffer, count, destRank, tag, comm);
692  }
693  else { // SerialComm doesn't implement send correctly anyway.
695  true,
696  std::logic_error,
697  "sendImpl: Not implemented for a serial communicator.");
698  }
699  }
700  else { // It's an MpiComm. Invoke MPI directly.
701  MPI_Comm rawComm = * (mpiComm->getRawMpiComm ());
702  T t;
703  MPI_Datatype rawType = MpiTypeTraits<T>::getType (t);
704  T* rawBuf = const_cast<T*> (sendBuffer);
705  const int err = MPI_Send (rawBuf, count, rawType, destRank, tag, rawComm);
707  err != MPI_SUCCESS,
708  std::runtime_error,
709  "MPI_Send failed with the following error: "
710  << ::Teuchos::Details::getMpiErrorString (err));
711  }
712 #else
714  true,
715  std::logic_error,
716  "sendImpl: Not implemented for a serial communicator.");
717 #endif // HAVE_TEUCHOS_MPI
718 }
719 
725 template<class T>
726 RCP<CommRequest<int> >
727 isendGeneral (const Comm<int>& comm,
728  const ArrayRCP<const T>& sendBuffer,
729  const int destRank)
730 {
732  "Teuchos::isend<int," << TypeNameTraits<T>::name () << ">");
733  ConstValueTypeSerializationBuffer<int, T>
734  charSendBuffer (sendBuffer.size (), sendBuffer.getRawPtr ());
735  RCP<CommRequest<int> > commRequest =
736  comm.isend (charSendBuffer.getCharBufferView (), destRank);
737  set_extra_data (sendBuffer, "buffer", inOutArg (commRequest));
738  return commRequest;
739 }
740 
747 template<class T>
748 RCP<CommRequest<int> >
749 isendGeneral (const ArrayRCP<const T>& sendBuffer,
750  const int destRank,
751  const int tag,
752  const Comm<int>& comm)
753 {
755  "Teuchos::isend<int," << TypeNameTraits<T>::name () << ">");
756  ConstValueTypeSerializationBuffer<int, T>
757  charSendBuffer (sendBuffer.size (), sendBuffer.getRawPtr ());
758  RCP<CommRequest<int> > commRequest =
759  comm.isend (charSendBuffer.getCharBufferView (), destRank, tag);
760  set_extra_data (sendBuffer, "buffer", inOutArg (commRequest));
761  return commRequest;
762 }
763 
766 template<class T>
767 RCP<Teuchos::CommRequest<int> >
768 isendImpl (const ArrayRCP<const T>& sendBuffer,
769  const int destRank,
770  const int tag,
771  const Comm<int>& comm)
772 {
773 #ifdef HAVE_TEUCHOS_MPI
774  using Teuchos::Details::MpiTypeTraits;
775 
776  // Even in an MPI build, Comm might be either a SerialComm or an
777  // MpiComm. If it's something else, we fall back to the most
778  // general implementation.
779  const MpiComm<int>* mpiComm = dynamic_cast<const MpiComm<int>* > (&comm);
780  if (mpiComm == NULL) {
781  // Is it a SerialComm?
782  const SerialComm<int>* serialComm = dynamic_cast<const SerialComm<int>* > (&comm);
783  if (serialComm == NULL) {
784  // We don't know what kind of Comm we have, so fall back to the
785  // most general implementation.
786  return isendGeneral<T> (sendBuffer, destRank, tag, comm);
787  }
788  else { // SerialComm doesn't implement send correctly anyway.
790  true, std::logic_error,
791  "isendImpl: Not implemented for a serial communicator.");
792  }
793  }
794  else { // It's an MpiComm. Invoke MPI directly.
795  MPI_Comm rawComm = * (mpiComm->getRawMpiComm ());
796  T t;
797  MPI_Datatype rawType = MpiTypeTraits<T>::getType (t);
798  // MPI promises not to modify the send buffer; the const_cast
799  // merely ensures compatibilty with C89, which does not have a
800  // "const" keyword.
801  T* rawSendBuf = const_cast<T*> (sendBuffer.getRawPtr ());
802  const int count = as<int> (sendBuffer.size ());
803  MPI_Request rawRequest = MPI_REQUEST_NULL;
804  const int err = MPI_Isend (rawSendBuf, count, rawType, destRank, tag,
805  rawComm, &rawRequest);
807  err != MPI_SUCCESS,
808  std::runtime_error,
809  "MPI_Isend failed with the following error: "
810  << ::Teuchos::Details::getMpiErrorString (err));
811 
812  ArrayRCP<const char> buf = arcp_reinterpret_cast<const char> (sendBuffer);
813  RCP<Details::MpiCommRequest> req (new Details::MpiCommRequest (rawRequest, buf));
814  return rcp_implicit_cast<CommRequest<int> > (req);
815  }
816 #else
818  true,
819  std::logic_error,
820  "isendImpl: Not implemented for a serial communicator.");
821 #endif // HAVE_TEUCHOS_MPI
822 }
823 
824 } // namespace (anonymous)
825 
826 
827 // mfh 18 Oct 2012: Note on full template specializations
828 //
829 // To make Windows builds happy, declarations of full template
830 // specializations (as found in Teuchos_CommHelpers.hpp) must use the
831 // TEUCHOSCOMM_LIB_DLL_EXPORT macro. However, _definitions_ of the
832 // specializations (as found in this file) must _not_ use the macro.
833 // That's why we don't use that macro here.
834 
835 // amb See note in .hpp file.
836 #if 0
837 #ifdef HAVE_TEUCHOS_COMPLEX
838 // Specialization for Ordinal=int and Packet=std::complex<double>.
839 template<>
840 void
841 reduceAll<int, std::complex<double> > (const Comm<int>& comm,
842  const EReductionType reductType,
843  const int count,
844  const std::complex<double> sendBuffer[],
845  std::complex<double> globalReducts[])
846 {
848  "Teuchos::reduceAll<int, std::complex<double> > (" << count << ", "
849  << toString (reductType) << ")"
850  );
851  reduceAllImpl<std::complex<double> > (comm, reductType, count, sendBuffer, globalReducts);
852 }
853 
854 template<>
855 RCP<Teuchos::CommRequest<int> >
856 ireceive<int, std::complex<double> > (const Comm<int>& comm,
857  const ArrayRCP<std::complex<double> >& recvBuffer,
858  const int sourceRank)
859 {
860  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, std::complex<double> >");
861  return ireceiveImpl<std::complex<double> > (comm, recvBuffer, sourceRank);
862 }
863 
864 template<>
865 RCP<Teuchos::CommRequest<int> >
866 ireceive<int, std::complex<double> > (const ArrayRCP<std::complex<double> >& recvBuffer,
867  const int sourceRank,
868  const int tag,
869  const Comm<int>& comm)
870 {
871  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, std::complex<double> >");
872  return ireceiveImpl<std::complex<double> > (recvBuffer, sourceRank, tag, comm);
873 }
874 
875 template<>
876 void
877 send<int, std::complex<double> > (const Comm<int>& comm,
878  const int count,
879  const std::complex<double> sendBuffer[],
880  const int destRank)
881 {
882  sendImpl<std::complex<double> > (comm, count, sendBuffer, destRank);
883 }
884 
885 template<>
886 void
887 send<int, std::complex<double> > (const std::complex<double> sendBuffer[],
888  const int count,
889  const int destRank,
890  const int tag,
891  const Comm<int>& comm)
892 {
893  sendImpl<std::complex<double> > (sendBuffer, count, destRank, tag, comm);
894 }
895 
896 template<>
897 RCP<Teuchos::CommRequest<int> >
898 isend (const ArrayRCP<const std::complex<double> >& sendBuffer,
899  const int destRank,
900  const int tag,
901  const Comm<int>& comm)
902 {
903  return isendImpl<std::complex<double> > (sendBuffer, destRank, tag, comm);
904 }
905 
906 // Specialization for Ordinal=int and Packet=std::complex<float>.
907 template<>
908 void
909 reduceAll<int, std::complex<float> > (const Comm<int>& comm,
910  const EReductionType reductType,
911  const int count,
912  const std::complex<float> sendBuffer[],
913  std::complex<float> globalReducts[])
914 {
916  "Teuchos::reduceAll<int, std::complex<float> > (" << count << ", "
917  << toString (reductType) << ")"
918  );
919  reduceAllImpl<std::complex<float> > (comm, reductType, count, sendBuffer, globalReducts);
920 }
921 
922 template<>
923 RCP<Teuchos::CommRequest<int> >
924 ireceive<int, std::complex<float> > (const Comm<int>& comm,
925  const ArrayRCP<std::complex<float> >& recvBuffer,
926  const int sourceRank)
927 {
928  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, std::complex<float> >");
929  return ireceiveImpl<std::complex<float> > (comm, recvBuffer, sourceRank);
930 }
931 
932 template<>
933 RCP<Teuchos::CommRequest<int> >
934 ireceive<int, std::complex<float> > (const ArrayRCP<std::complex<float> >& recvBuffer,
935  const int sourceRank,
936  const int tag,
937  const Comm<int>& comm)
938 {
939  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, std::complex<float> >");
940  return ireceiveImpl<std::complex<float> > (recvBuffer, sourceRank, tag, comm);
941 }
942 
943 template<>
944 void
945 send<int, std::complex<float> > (const Comm<int>& comm,
946  const int count,
947  const std::complex<float> sendBuffer[],
948  const int destRank)
949 {
950  return sendImpl<std::complex<float> > (comm, count, sendBuffer, destRank);
951 }
952 
953 template<>
954 void
955 send<int, std::complex<float> > (const std::complex<float> sendBuffer[],
956  const int count,
957  const int destRank,
958  const int tag,
959  const Comm<int>& comm)
960 {
961  return sendImpl<std::complex<float> > (sendBuffer, count, destRank, tag, comm);
962 }
963 
964 template<>
965 RCP<Teuchos::CommRequest<int> >
966 isend (const ArrayRCP<const std::complex<float> >& sendBuffer,
967  const int destRank,
968  const int tag,
969  const Comm<int>& comm)
970 {
971  return isendImpl<std::complex<float> > (sendBuffer, destRank, tag, comm);
972 }
973 #endif // HAVE_TEUCHOS_COMPLEX
974 #endif // if 0
975 
976 
977 // Specialization for Ordinal=int and Packet=double.
978 template<>
979 void
981  const EReductionType reductType,
982  const int count,
983  const double sendBuffer[],
984  double globalReducts[])
985 {
987  "Teuchos::reduceAll<int, double> (" << count << ", "
988  << toString (reductType) << ")"
989  );
990  reduceAllImpl<double> (comm, reductType, count, sendBuffer, globalReducts);
991 }
992 
993 template<>
994 RCP<Teuchos::CommRequest<int> >
996  const ArrayRCP<double>& recvBuffer,
997  const int sourceRank)
998 {
999  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, double>");
1000  return ireceiveImpl<double> (comm, recvBuffer, sourceRank);
1001 }
1002 
1003 template<>
1004 RCP<Teuchos::CommRequest<int> >
1006  const int sourceRank,
1007  const int tag,
1008  const Comm<int>& comm)
1009 {
1010  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, double>");
1011  return ireceiveImpl<double> (recvBuffer, sourceRank, tag, comm);
1012 }
1013 
1014 template<>
1015 void
1017  const int count,
1018  const double sendBuffer[],
1019  const int destRank)
1020 {
1021  return sendImpl<double> (comm, count, sendBuffer, destRank);
1022 }
1023 
1024 template<>
1025 void
1026 send<int, double> (const double sendBuffer[],
1027  const int count,
1028  const int destRank,
1029  const int tag,
1030  const Comm<int>& comm)
1031 {
1032  return sendImpl<double> (sendBuffer, count, destRank, tag, comm);
1033 }
1034 
1035 template<>
1036 RCP<Teuchos::CommRequest<int> >
1037 isend (const ArrayRCP<const double>& sendBuffer,
1038  const int destRank,
1039  const int tag,
1040  const Comm<int>& comm)
1041 {
1042  return isendImpl<double> (sendBuffer, destRank, tag, comm);
1043 }
1044 
1045 // Specialization for Ordinal=int and Packet=float.
1046 template<>
1047 void
1049  const EReductionType reductType,
1050  const int count,
1051  const float sendBuffer[],
1052  float globalReducts[])
1053 {
1055  "Teuchos::reduceAll<int, float> (" << count << ", "
1056  << toString (reductType) << ")"
1057  );
1058  reduceAllImpl<float> (comm, reductType, count, sendBuffer, globalReducts);
1059 }
1060 
1061 template<>
1062 RCP<Teuchos::CommRequest<int> >
1064  const ArrayRCP<float>& recvBuffer,
1065  const int sourceRank)
1066 {
1067  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, float>");
1068  return ireceiveImpl<float> (comm, recvBuffer, sourceRank);
1069 }
1070 
1071 template<>
1072 RCP<Teuchos::CommRequest<int> >
1074  const int sourceRank,
1075  const int tag,
1076  const Comm<int>& comm)
1077 {
1078  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, float>");
1079  return ireceiveImpl<float> (recvBuffer, sourceRank, tag, comm);
1080 }
1081 
1082 template<>
1083 void
1085  const int count,
1086  const float sendBuffer[],
1087  const int destRank)
1088 {
1089  return sendImpl<float> (comm, count, sendBuffer, destRank);
1090 }
1091 
1092 template<>
1093 void
1094 send<int, float> (const float sendBuffer[],
1095  const int count,
1096  const int destRank,
1097  const int tag,
1098  const Comm<int>& comm)
1099 {
1100  return sendImpl<float> (sendBuffer, count, destRank, tag, comm);
1101 }
1102 
1103 template<>
1104 RCP<Teuchos::CommRequest<int> >
1105 isend (const ArrayRCP<const float>& sendBuffer,
1106  const int destRank,
1107  const int tag,
1108  const Comm<int>& comm)
1109 {
1110  return isendImpl<float> (sendBuffer, destRank, tag, comm);
1111 }
1112 
1113 
1114 // Specialization for Ordinal=int and Packet=long long.
1115 template<>
1116 void
1117 gather<int, long long> (const long long sendBuf[],
1118  const int sendCount,
1119  long long recvBuf[],
1120  const int recvCount,
1121  const int root,
1122  const Comm<int>& comm)
1123 {
1124  gatherImpl<long long> (sendBuf, sendCount, recvBuf, recvCount, root, comm);
1125 }
1126 
1127 template<>
1128 void
1129 gatherv<int, long long> (const long long sendBuf[],
1130  const int sendCount,
1131  long long recvBuf[],
1132  const int recvCounts[],
1133  const int displs[],
1134  const int root,
1135  const Comm<int>& comm)
1136 {
1137  gathervImpl<long long> (sendBuf, sendCount, recvBuf, recvCounts, displs, root, comm);
1138 }
1139 
1140 template<>
1141 void
1143  const EReductionType reductType,
1144  const int count,
1145  const long long sendBuffer[],
1146  long long globalReducts[])
1147 {
1149  "Teuchos::reduceAll<int, long long> (" << count << ", "
1150  << toString (reductType) << ")"
1151  );
1152  reduceAllImpl<long long> (comm, reductType, count, sendBuffer, globalReducts);
1153 }
1154 
1155 template<>
1156 RCP<Teuchos::CommRequest<int> >
1158  const ArrayRCP<long long>& recvBuffer,
1159  const int sourceRank)
1160 {
1161  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, long long>");
1162  return ireceiveImpl<long long> (comm, recvBuffer, sourceRank);
1163 }
1164 
1165 template<>
1166 RCP<Teuchos::CommRequest<int> >
1168  const int sourceRank,
1169  const int tag,
1170  const Comm<int>& comm)
1171 {
1172  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, long long>");
1173  return ireceiveImpl<long long> (recvBuffer, sourceRank, tag, comm);
1174 }
1175 
1176 template<>
1177 void
1179  const int count,
1180  const long long sendBuffer[],
1181  const int destRank)
1182 {
1183  return sendImpl<long long> (comm, count, sendBuffer, destRank);
1184 }
1185 
1186 template<>
1187 void
1188 send<int, long long> (const long long sendBuffer[],
1189  const int count,
1190  const int destRank,
1191  const int tag,
1192  const Comm<int>& comm)
1193 {
1194  return sendImpl<long long> (sendBuffer, count, destRank, tag, comm);
1195 }
1196 
1197 template<>
1198 RCP<Teuchos::CommRequest<int> >
1200  const int destRank,
1201  const int tag,
1202  const Comm<int>& comm)
1203 {
1204  return isendImpl<long long> (sendBuffer, destRank, tag, comm);
1205 }
1206 
1207 // Specialization for Ordinal=int and Packet=unsigned long long.
1208 template<>
1209 void
1210 gather<int, unsigned long long> (const unsigned long long sendBuf[],
1211  const int sendCount,
1212  unsigned long long recvBuf[],
1213  const int recvCount,
1214  const int root,
1215  const Comm<int>& comm)
1216 {
1217  gatherImpl<unsigned long long> (sendBuf, sendCount, recvBuf, recvCount, root, comm);
1218 }
1219 
1220 template<>
1221 void
1222 gatherv<int, unsigned long long> (const unsigned long long sendBuf[],
1223  const int sendCount,
1224  unsigned long long recvBuf[],
1225  const int recvCounts[],
1226  const int displs[],
1227  const int root,
1228  const Comm<int>& comm)
1229 {
1230  gathervImpl<unsigned long long> (sendBuf, sendCount, recvBuf, recvCounts, displs, root, comm);
1231 }
1232 
1233 template<>
1234 void
1236  const EReductionType reductType,
1237  const int count,
1238  const unsigned long long sendBuffer[],
1239  unsigned long long globalReducts[])
1240 {
1242  "Teuchos::reduceAll<int, unsigned long long> (" << count << ", "
1243  << toString (reductType) << ")"
1244  );
1245  reduceAllImpl<unsigned long long> (comm, reductType, count, sendBuffer, globalReducts);
1246 }
1247 
1248 template<>
1249 RCP<Teuchos::CommRequest<int> >
1251  const ArrayRCP<unsigned long long>& recvBuffer,
1252  const int sourceRank)
1253 {
1254  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, unsigned long long>");
1255  return ireceiveImpl<unsigned long long> (comm, recvBuffer, sourceRank);
1256 }
1257 
1258 template<>
1259 RCP<Teuchos::CommRequest<int> >
1261  const int sourceRank,
1262  const int tag,
1263  const Comm<int>& comm)
1264 {
1265  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, unsigned long long>");
1266  return ireceiveImpl<unsigned long long> (recvBuffer, sourceRank, tag, comm);
1267 }
1268 
1269 template<>
1270 void
1272  const int count,
1273  const unsigned long long sendBuffer[],
1274  const int destRank)
1275 {
1276  return sendImpl<unsigned long long> (comm, count, sendBuffer, destRank);
1277 }
1278 
1279 template<>
1280 void
1281 send<int, unsigned long long> (const unsigned long long sendBuffer[],
1282  const int count,
1283  const int destRank,
1284  const int tag,
1285  const Comm<int>& comm)
1286 {
1287  return sendImpl<unsigned long long> (sendBuffer, count, destRank, tag, comm);
1288 }
1289 
1290 template<>
1291 RCP<Teuchos::CommRequest<int> >
1293  const int destRank,
1294  const int tag,
1295  const Comm<int>& comm)
1296 {
1297  return isendImpl<unsigned long long> (sendBuffer, destRank, tag, comm);
1298 }
1299 
1300 
1301 // Specialization for Ordinal=int and Packet=long.
1302 template<>
1303 void
1304 gather<int, long> (const long sendBuf[],
1305  const int sendCount,
1306  long recvBuf[],
1307  const int recvCount,
1308  const int root,
1309  const Comm<int>& comm)
1310 {
1311  gatherImpl<long> (sendBuf, sendCount, recvBuf, recvCount, root, comm);
1312 }
1313 
1314 template<>
1315 void
1316 gatherv<int, long> (const long sendBuf[],
1317  const int sendCount,
1318  long recvBuf[],
1319  const int recvCounts[],
1320  const int displs[],
1321  const int root,
1322  const Comm<int>& comm)
1323 {
1324  gathervImpl<long> (sendBuf, sendCount, recvBuf, recvCounts, displs, root, comm);
1325 }
1326 
1327 template<>
1328 void
1330  const EReductionType reductType,
1331  const int count,
1332  const long sendBuffer[],
1333  long globalReducts[])
1334 {
1336  "Teuchos::reduceAll<int, long> (" << count << ", "
1337  << toString (reductType) << ")"
1338  );
1339  reduceAllImpl<long> (comm, reductType, count, sendBuffer, globalReducts);
1340 }
1341 
1342 template<>
1343 RCP<Teuchos::CommRequest<int> >
1345  const ArrayRCP<long>& recvBuffer,
1346  const int sourceRank)
1347 {
1348  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, long>");
1349  return ireceiveImpl<long> (comm, recvBuffer, sourceRank);
1350 }
1351 
1352 template<>
1353 RCP<Teuchos::CommRequest<int> >
1355  const int sourceRank,
1356  const int tag,
1357  const Comm<int>& comm)
1358 {
1359  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, long>");
1360  return ireceiveImpl<long> (recvBuffer, sourceRank, tag, comm);
1361 }
1362 
1363 template<>
1364 void
1366  const int count,
1367  const long sendBuffer[],
1368  const int destRank)
1369 {
1370  return sendImpl<long> (comm, count, sendBuffer, destRank);
1371 }
1372 
1373 template<>
1374 void
1375 send<int, long> (const long sendBuffer[],
1376  const int count,
1377  const int destRank,
1378  const int tag,
1379  const Comm<int>& comm)
1380 {
1381  return sendImpl<long> (sendBuffer, count, destRank, tag, comm);
1382 }
1383 
1384 template<>
1385 RCP<Teuchos::CommRequest<int> >
1386 isend (const ArrayRCP<const long>& sendBuffer,
1387  const int destRank,
1388  const int tag,
1389  const Comm<int>& comm)
1390 {
1391  return isendImpl<long> (sendBuffer, destRank, tag, comm);
1392 }
1393 
1394 
1395 // Specialization for Ordinal=int and Packet=unsigned long.
1396 template<>
1397 void
1398 gather<int, unsigned long> (const unsigned long sendBuf[],
1399  const int sendCount,
1400  unsigned long recvBuf[],
1401  const int recvCount,
1402  const int root,
1403  const Comm<int>& comm)
1404 {
1405  gatherImpl<unsigned long> (sendBuf, sendCount, recvBuf, recvCount, root, comm);
1406 }
1407 
1408 template<>
1409 void
1410 gatherv<int, unsigned long> (const unsigned long sendBuf[],
1411  const int sendCount,
1412  unsigned long recvBuf[],
1413  const int recvCounts[],
1414  const int displs[],
1415  const int root,
1416  const Comm<int>& comm)
1417 {
1418  gathervImpl<unsigned long> (sendBuf, sendCount, recvBuf, recvCounts, displs, root, comm);
1419 }
1420 
1421 template<>
1422 void
1424  const EReductionType reductType,
1425  const int count,
1426  const unsigned long sendBuffer[],
1427  unsigned long globalReducts[])
1428 {
1430  "Teuchos::reduceAll<int, unsigned long> (" << count << ", "
1431  << toString (reductType) << ")"
1432  );
1433  reduceAllImpl<unsigned long> (comm, reductType, count, sendBuffer, globalReducts);
1434 }
1435 
1436 template<>
1437 RCP<Teuchos::CommRequest<int> >
1439  const ArrayRCP<unsigned long>& recvBuffer,
1440  const int sourceRank)
1441 {
1442  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, unsigned long>");
1443  return ireceiveImpl<unsigned long> (comm, recvBuffer, sourceRank);
1444 }
1445 
1446 template<>
1447 RCP<Teuchos::CommRequest<int> >
1449  const int sourceRank,
1450  const int tag,
1451  const Comm<int>& comm)
1452 {
1453  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, unsigned long>");
1454  return ireceiveImpl<unsigned long> (recvBuffer, sourceRank, tag, comm);
1455 }
1456 
1457 template<>
1458 void
1460  const int count,
1461  const unsigned long sendBuffer[],
1462  const int destRank)
1463 {
1464  return sendImpl<unsigned long> (comm, count, sendBuffer, destRank);
1465 }
1466 
1467 template<>
1468 void
1469 send<int, unsigned long> (const unsigned long sendBuffer[],
1470  const int count,
1471  const int destRank,
1472  const int tag,
1473  const Comm<int>& comm)
1474 {
1475  return sendImpl<unsigned long> (sendBuffer, count, destRank, tag, comm);
1476 }
1477 
1478 template<>
1479 RCP<Teuchos::CommRequest<int> >
1481  const int destRank,
1482  const int tag,
1483  const Comm<int>& comm)
1484 {
1485  return isendImpl<unsigned long> (sendBuffer, destRank, tag, comm);
1486 }
1487 
1488 // Specialization for Ordinal=int and Packet=int.
1489 template<>
1490 void
1491 gather<int, int> (const int sendBuf[],
1492  const int sendCount,
1493  int recvBuf[],
1494  const int recvCount,
1495  const int root,
1496  const Comm<int>& comm)
1497 {
1498  gatherImpl<int> (sendBuf, sendCount, recvBuf, recvCount, root, comm);
1499 }
1500 
1501 template<>
1502 void
1503 gatherv<int, int> (const int sendBuf[],
1504  const int sendCount,
1505  int recvBuf[],
1506  const int recvCounts[],
1507  const int displs[],
1508  const int root,
1509  const Comm<int>& comm)
1510 {
1511  gathervImpl<int> (sendBuf, sendCount, recvBuf, recvCounts, displs, root, comm);
1512 }
1513 
1514 template<>
1515 void
1516 scatter<int, int> (const int sendBuf[],
1517  const int sendCount,
1518  int recvBuf[],
1519  const int recvCount,
1520  const int root,
1521  const Comm<int>& comm)
1522 {
1523  scatterImpl<int> (sendBuf, sendCount, recvBuf, recvCount, root, comm);
1524 }
1525 
1526 template<>
1527 void
1528 reduce<int, int> (const int sendBuf[],
1529  int recvBuf[],
1530  const int count,
1531  const EReductionType reductType,
1532  const int root,
1533  const Comm<int>& comm)
1534 {
1536  ("Teuchos::reduce<int, int> (" << count << ", " << toString (reductType)
1537  << ")");
1538  reduceImpl<int> (sendBuf, recvBuf, count, reductType, root, comm);
1539 }
1540 template<>
1541 void
1542 reduce<int, long> (const long sendBuf[],
1543  long recvBuf[],
1544  const int count,
1545  const EReductionType reductType,
1546  const int root,
1547  const Comm<int>& comm)
1548 {
1550  ("Teuchos::reduce<int, int> (" << count << ", " << toString (reductType)
1551  << ")");
1552  reduceImpl<long> (sendBuf, recvBuf, count, reductType, root, comm);
1553 }
1554 
1555 template<>
1556 void
1557 reduce<int, unsigned long> (const unsigned long sendBuf[],
1558  unsigned long recvBuf[],
1559  const int count,
1560  const EReductionType reductType,
1561  const int root,
1562  const Comm<int>& comm)
1563 {
1565  ("Teuchos::reduce<int, int> (" << count << ", " << toString (reductType)
1566  << ")");
1567  reduceImpl<unsigned long> (sendBuf, recvBuf, count, reductType, root, comm);
1568 }
1569 
1570 template<>
1571 void
1572 reduce<int, unsigned long long > (const unsigned long long sendBuf[],
1573  unsigned long long recvBuf[],
1574  const int count,
1575  const EReductionType reductType,
1576  const int root,
1577  const Comm<int>& comm)
1578 {
1580  ("Teuchos::reduce<int, int> (" << count << ", " << toString (reductType)
1581  << ")");
1582  reduceImpl<unsigned long long> (sendBuf, recvBuf, count, reductType, root, comm);
1583 }
1584 
1585 template<>
1586 void
1587 reduce<int, double> (const double sendBuf[],
1588  double recvBuf[],
1589  const int count,
1590  const EReductionType reductType,
1591  const int root,
1592  const Comm<int>& comm)
1593 {
1595  ("Teuchos::reduce<int, int> (" << count << ", " << toString (reductType)
1596  << ")");
1597  reduceImpl<double> (sendBuf, recvBuf, count, reductType, root, comm);
1598 }
1599 template<>
1600 void
1602  const EReductionType reductType,
1603  const int count,
1604  const int sendBuffer[],
1605  int globalReducts[])
1606 {
1608  "Teuchos::reduceAll<int, int> (" << count << ", "
1609  << toString (reductType) << ")"
1610  );
1611  reduceAllImpl<int> (comm, reductType, count, sendBuffer, globalReducts);
1612 }
1613 
1614 template<>
1615 RCP<Teuchos::CommRequest<int> >
1617  const ArrayRCP<int>& recvBuffer,
1618  const int sourceRank)
1619 {
1620  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, int>");
1621  return ireceiveImpl<int> (comm, recvBuffer, sourceRank);
1622 }
1623 
1624 template<>
1625 RCP<Teuchos::CommRequest<int> >
1627  const int sourceRank,
1628  const int tag,
1629  const Comm<int>& comm)
1630 {
1631  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, int>");
1632  return ireceiveImpl<int> (recvBuffer, sourceRank, tag, comm);
1633 }
1634 
1635 template<>
1636 void
1638  const int count,
1639  const int sendBuffer[],
1640  const int destRank)
1641 {
1642  return sendImpl<int> (comm, count, sendBuffer, destRank);
1643 }
1644 
1645 template<>
1646 void
1647 send<int, int> (const int sendBuffer[],
1648  const int count,
1649  const int destRank,
1650  const int tag,
1651  const Comm<int>& comm)
1652 {
1653  return sendImpl<int> (sendBuffer, count, destRank, tag, comm);
1654 }
1655 
1656 template<>
1657 RCP<Teuchos::CommRequest<int> >
1658 isend (const ArrayRCP<const int>& sendBuffer,
1659  const int destRank,
1660  const int tag,
1661  const Comm<int>& comm)
1662 {
1663  return isendImpl<int> (sendBuffer, destRank, tag, comm);
1664 }
1665 
1666 // Specialization for Ordinal=int and Packet=unsigned int.
1667 template<>
1668 void
1669 gather<int, unsigned int> (const unsigned int sendBuf[],
1670  const int sendCount,
1671  unsigned int recvBuf[],
1672  const int recvCount,
1673  const int root,
1674  const Comm<int>& comm)
1675 {
1676  gatherImpl<unsigned int> (sendBuf, sendCount, recvBuf, recvCount, root, comm);
1677 }
1678 
1679 template<>
1680 void
1681 gatherv<int, unsigned int> (const unsigned int sendBuf[],
1682  const int sendCount,
1683  unsigned int recvBuf[],
1684  const int recvCounts[],
1685  const int displs[],
1686  const int root,
1687  const Comm<int>& comm)
1688 {
1689  gathervImpl<unsigned int> (sendBuf, sendCount, recvBuf, recvCounts, displs, root, comm);
1690 }
1691 
1692 template<>
1693 void
1695  const EReductionType reductType,
1696  const int count,
1697  const unsigned int sendBuffer[],
1698  unsigned int globalReducts[])
1699 {
1701  "Teuchos::reduceAll<int, unsigned int> (" << count << ", "
1702  << toString (reductType) << ")"
1703  );
1704  reduceAllImpl<unsigned int> (comm, reductType, count, sendBuffer, globalReducts);
1705 }
1706 
1707 template<>
1708 RCP<Teuchos::CommRequest<int> >
1710  const ArrayRCP<unsigned int>& recvBuffer,
1711  const int sourceRank)
1712 {
1713  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, unsigned int>");
1714  return ireceiveImpl<unsigned int> (comm, recvBuffer, sourceRank);
1715 }
1716 
1717 template<>
1718 RCP<Teuchos::CommRequest<int> >
1720  const int sourceRank,
1721  const int tag,
1722  const Comm<int>& comm)
1723 {
1724  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, unsigned int>");
1725  return ireceiveImpl<unsigned int> (recvBuffer, sourceRank, tag, comm);
1726 }
1727 
1728 template<>
1729 void
1731  const int count,
1732  const unsigned int sendBuffer[],
1733  const int destRank)
1734 {
1735  return sendImpl<unsigned int> (comm, count, sendBuffer, destRank);
1736 }
1737 
1738 template<>
1739 void
1740 send<int, unsigned int> (const unsigned int sendBuffer[],
1741  const int count,
1742  const int destRank,
1743  const int tag,
1744  const Comm<int>& comm)
1745 {
1746  return sendImpl<unsigned int> (sendBuffer, count, destRank, tag, comm);
1747 }
1748 
1749 template<>
1750 RCP<Teuchos::CommRequest<int> >
1752  const int destRank,
1753  const int tag,
1754  const Comm<int>& comm)
1755 {
1756  return isendImpl<unsigned int> (sendBuffer, destRank, tag, comm);
1757 }
1758 
1759 
1760 // Specialization for Ordinal=int and Packet=short.
1761 template<>
1762 void
1763 gather<int, short> (const short sendBuf[],
1764  const int sendCount,
1765  short recvBuf[],
1766  const int recvCount,
1767  const int root,
1768  const Comm<int>& comm)
1769 {
1770  gatherImpl<short> (sendBuf, sendCount, recvBuf, recvCount, root, comm);
1771 }
1772 
1773 template<>
1774 void
1775 gatherv<int, short> (const short sendBuf[],
1776  const int sendCount,
1777  short recvBuf[],
1778  const int recvCounts[],
1779  const int displs[],
1780  const int root,
1781  const Comm<int>& comm)
1782 {
1783  gathervImpl<short> (sendBuf, sendCount, recvBuf, recvCounts, displs, root, comm);
1784 }
1785 
1786 template<>
1787 void
1789  const EReductionType reductType,
1790  const int count,
1791  const short sendBuffer[],
1792  short globalReducts[])
1793 {
1795  "Teuchos::reduceAll<int, short> (" << count << ", "
1796  << toString (reductType) << ")"
1797  );
1798  reduceAllImpl<short> (comm, reductType, count, sendBuffer, globalReducts);
1799 }
1800 
1801 template<>
1802 RCP<Teuchos::CommRequest<int> >
1804  const ArrayRCP<short>& recvBuffer,
1805  const int sourceRank)
1806 {
1807  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, short>");
1808  return ireceiveImpl<short> (comm, recvBuffer, sourceRank);
1809 }
1810 
1811 template<>
1812 RCP<Teuchos::CommRequest<int> >
1814  const int sourceRank,
1815  const int tag,
1816  const Comm<int>& comm)
1817 {
1818  TEUCHOS_COMM_TIME_MONITOR("ireceive<int, short>");
1819  return ireceiveImpl<short> (recvBuffer, sourceRank, tag, comm);
1820 }
1821 
1822 template<>
1823 void
1825  const int count,
1826  const short sendBuffer[],
1827  const int destRank)
1828 {
1829  return sendImpl<short> (comm, count, sendBuffer, destRank);
1830 }
1831 
1832 template<>
1833 void
1834 send<int, short> (const short sendBuffer[],
1835  const int count,
1836  const int destRank,
1837  const int tag,
1838  const Comm<int>& comm)
1839 {
1840  return sendImpl<short> (sendBuffer, count, destRank, tag, comm);
1841 }
1842 
1843 template<>
1844 RCP<Teuchos::CommRequest<int> >
1845 isend (const ArrayRCP<const short>& sendBuffer,
1846  const int destRank,
1847  const int tag,
1848  const Comm<int>& comm)
1849 {
1850  return isendImpl<short> (sendBuffer, destRank, tag, comm);
1851 }
1852 
1853 // mfh 18 Oct 2012: The specialization for Packet=char seems to be
1854 // causing problems such as the following:
1855 //
1856 // http://testing.sandia.gov/cdash/testDetails.php?test=9909246&build=747699
1857 //
1858 // I am disabling it for now. This should revert back to the old
1859 // behavior for Packet=char. That should fix the Tpetra errors, since
1860 // many Tpetra objects inherit from DistObject<char, ...>.
1861 #if 0
1862 // Specialization for Ordinal=int and Packet=char.
1863 template<>
1864 void
1865 reduceAll<int, char> (const Comm<int>& comm,
1866  const EReductionType reductType,
1867  const int count,
1868  const char sendBuffer[],
1869  char globalReducts[])
1870 {
1872  "Teuchos::reduceAll<int, char> (" << count << ", "
1873  << toString (reductType) << ")"
1874  );
1875  reduceAllImpl<char> (comm, reductType, count, sendBuffer, globalReducts);
1876 }
1877 #endif // 0
1878 
1879 } // namespace Teuchos
void reduce< int, unsigned long >(const unsigned long sendBuf[], unsigned long recvBuf[], const int count, const EReductionType reductType, const int root, const Comm< int > &comm)
void reduce< int, long >(const long sendBuf[], long recvBuf[], const int count, const EReductionType reductType, const int root, const Comm< int > &comm)
RCP< Teuchos::CommRequest< int > > ireceive< int, unsigned long long >(const Comm< int > &comm, const ArrayRCP< unsigned long long > &recvBuffer, const int sourceRank)
void send< int, float >(const Comm< int > &comm, const int count, const float sendBuffer[], const int destRank)
EReductionType
Predefined reduction operations that Teuchos::Comm understands.
RCP< Teuchos::CommRequest< int > > ireceive< int, float >(const Comm< int > &comm, const ArrayRCP< float > &recvBuffer, const int sourceRank)
void gatherv< int, short >(const short sendBuf[], const int sendCount, short recvBuf[], const int recvCounts[], const int displs[], const int root, const Comm< int > &comm)
void gather< int, unsigned long long >(const unsigned long long sendBuf[], const int sendCount, unsigned long long recvBuf[], const int recvCount, const int root, const Comm< int > &comm)
void reduceAll< int, long >(const Comm< int > &comm, const EReductionType reductType, const int count, const long sendBuffer[], long globalReducts[])
void send< int, unsigned long long >(const Comm< int > &comm, const int count, const unsigned long long sendBuffer[], const int destRank)
RCP< Teuchos::CommRequest< int > > ireceive< int, double >(const Comm< int > &comm, const ArrayRCP< double > &recvBuffer, const int sourceRank)
RCP< Teuchos::CommRequest< int > > ireceive< int, short >(const Comm< int > &comm, const ArrayRCP< short > &recvBuffer, const int sourceRank)
void send< int, long >(const Comm< int > &comm, const int count, const long sendBuffer[], const int destRank)
void gather< int, unsigned int >(const unsigned int sendBuf[], const int sendCount, unsigned int recvBuf[], const int recvCount, const int root, const Comm< int > &comm)
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
Macro for throwing an exception with breakpointing to ease debugging.
void reduce< int, double >(const double sendBuf[], double recvBuf[], const int count, const EReductionType reductType, const int root, const Comm< int > &comm)
void scatter< int, int >(const int sendBuf[], const int sendCount, int recvBuf[], const int recvCount, const int root, const Comm< int > &comm)
void gatherv< int, int >(const int sendBuf[], const int sendCount, int recvBuf[], const int recvCounts[], const int displs[], const int root, const Comm< int > &comm)
void gatherv< int, long long >(const long long sendBuf[], const int sendCount, long long recvBuf[], const int recvCounts[], const int displs[], const int root, const Comm< int > &comm)
void gather< int, long >(const long sendBuf[], const int sendCount, long recvBuf[], const int recvCount, const int root, const Comm< int > &comm)
RCP< Teuchos::CommRequest< int > > ireceive< int, long long >(const Comm< int > &comm, const ArrayRCP< long long > &recvBuffer, const int sourceRank)
void send< int, int >(const Comm< int > &comm, const int count, const int sendBuffer[], const int destRank)
void send< int, double >(const Comm< int > &comm, const int count, const double sendBuffer[], const int destRank)
void gather< int, int >(const int sendBuf[], const int sendCount, int recvBuf[], const int recvCount, const int root, const Comm< int > &comm)
void gather< int, long long >(const long long sendBuf[], const int sendCount, long long recvBuf[], const int recvCount, const int root, const Comm< int > &comm)
#define TEUCHOS_COMM_TIME_MONITOR(FUNCNAME)
void gatherv< int, unsigned long >(const unsigned long sendBuf[], const int sendCount, unsigned long recvBuf[], const int recvCounts[], const int displs[], const int root, const Comm< int > &comm)
void reduceAll< int, unsigned int >(const Comm< int > &comm, const EReductionType reductType, const int count, const unsigned int sendBuffer[], unsigned int globalReducts[])
void send< int, long long >(const Comm< int > &comm, const int count, const long long sendBuffer[], const int destRank)
void reduceAll< int, short >(const Comm< int > &comm, const EReductionType reductType, const int count, const short sendBuffer[], short globalReducts[])
void reduce< int, unsigned long long >(const unsigned long long sendBuf[], unsigned long long recvBuf[], const int count, const EReductionType reductType, const int root, const Comm< int > &comm)
void reduce< int, int >(const int sendBuf[], int recvBuf[], const int count, const EReductionType reductType, const int root, const Comm< int > &comm)
void reduceAll< int, double >(const Comm< int > &comm, const EReductionType reductType, const int count, const double sendBuffer[], double globalReducts[])
std::string toString(const HashSet< Key > &h)
void gatherv< int, unsigned long long >(const unsigned long long sendBuf[], const int sendCount, unsigned long long recvBuf[], const int recvCounts[], const int displs[], const int root, const Comm< int > &comm)
void gatherv< int, long >(const long sendBuf[], const int sendCount, long recvBuf[], const int recvCounts[], const int displs[], const int root, const Comm< int > &comm)
void reduceAll< int, int >(const Comm< int > &comm, const EReductionType reductType, const int count, const int sendBuffer[], int globalReducts[])
void reduceAll< int, unsigned long long >(const Comm< int > &comm, const EReductionType reductType, const int count, const unsigned long long sendBuffer[], unsigned long long globalReducts[])
void gatherv< int, unsigned int >(const unsigned int sendBuf[], const int sendCount, unsigned int recvBuf[], const int recvCounts[], const int displs[], const int root, const Comm< int > &comm)
RCP< Teuchos::CommRequest< int > > ireceive< int, unsigned int >(const Comm< int > &comm, const ArrayRCP< unsigned int > &recvBuffer, const int sourceRank)
void reduceAll< int, unsigned long >(const Comm< int > &comm, const EReductionType reductType, const int count, const unsigned long sendBuffer[], unsigned long globalReducts[])
RCP< Teuchos::CommRequest< int > > ireceive< int, int >(const Comm< int > &comm, const ArrayRCP< int > &recvBuffer, const int sourceRank)
RCP< Teuchos::CommRequest< int > > ireceive< int, unsigned long >(const Comm< int > &comm, const ArrayRCP< unsigned long > &recvBuffer, const int sourceRank)
RCP< Teuchos::CommRequest< int > > isend(const ArrayRCP< const double > &sendBuffer, const int destRank, const int tag, const Comm< int > &comm)
void reduceAll< int, float >(const Comm< int > &comm, const EReductionType reductType, const int count, const float sendBuffer[], float globalReducts[])
Declaration of Teuchos::Details::MpiTypeTraits (only if building with MPI)
RCP< Teuchos::CommRequest< int > > ireceive< int, long >(const Comm< int > &comm, const ArrayRCP< long > &recvBuffer, const int sourceRank)
void gather< int, short >(const short sendBuf[], const int sendCount, short recvBuf[], const int recvCount, const int root, const Comm< int > &comm)
void send< int, short >(const Comm< int > &comm, const int count, const short sendBuffer[], const int destRank)
void gather< int, unsigned long >(const unsigned long sendBuf[], const int sendCount, unsigned long recvBuf[], const int recvCount, const int root, const Comm< int > &comm)
void reduceAll< int, long long >(const Comm< int > &comm, const EReductionType reductType, const int count, const long long sendBuffer[], long long globalReducts[])
void send< int, unsigned long >(const Comm< int > &comm, const int count, const unsigned long sendBuffer[], const int destRank)
Reference-counted smart pointer for managing arrays.
void send< int, unsigned int >(const Comm< int > &comm, const int count, const unsigned int sendBuffer[], const int destRank)