Teuchos - Trilinos Tools Package  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Teuchos_DefaultComm.hpp
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Teuchos: Common Tools Package
5 // Copyright (2004) Sandia Corporation
6 //
7 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8 // license for use of this work by or on behalf of the U.S. Government.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ***********************************************************************
40 // @HEADER
41 
42 #ifndef TEUCHOS_DEFAULT_COMM_HPP
43 #define TEUCHOS_DEFAULT_COMM_HPP
44 
45 #include "Teuchos_RCP.hpp"
46 #include "Teuchos_DefaultSerialComm.hpp"
47 #ifdef HAVE_MPI
49 #endif
50 
51 namespace Teuchos {
52 
53 #ifdef HAVE_MPI
54 namespace Details {
55 
56 template<class OrdinalType>
57 int
58 mpiFreeDefaultComm (MPI_Comm, int, void*, void*);
59 
60 template<class OrdinalType>
61 int
62 mpiFreeDefaultSerialComm (MPI_Comm, int, void*, void*);
63 
64 } // namespace Details
65 #endif // HAVE_MPI
66 
93 template<typename OrdinalType>
94 class DefaultComm {
95 public:
109 
118 
119 private:
125  static const Comm<OrdinalType>* comm_;
126 
128  static const Comm<OrdinalType>* defaultSerialComm_;
129 
130 #ifdef HAVE_MPI
131  template<class OT>
133  friend int
134  Details::mpiFreeDefaultComm (MPI_Comm, int, void*, void*);
135 
137  template<class OT>
138  friend int
139  Details::mpiFreeDefaultSerialComm (MPI_Comm, int, void*, void*);
140 #endif // HAVE_MPI
141 
143  static void freeDefaultComm () {
144  if (comm_ != NULL) {
145  delete comm_;
146  comm_ = NULL;
147  }
148  }
149 
151  static void freeDefaultSerialComm () {
152  if (defaultSerialComm_ != NULL) {
153  delete defaultSerialComm_;
154  defaultSerialComm_ = NULL;
155  }
156  }
157 };
158 
159 #ifdef HAVE_MPI
160 namespace Details {
161 
162 template<class OrdinalType>
163 int
164 mpiFreeDefaultComm (MPI_Comm, int, void*, void*)
165 {
166  try {
168  } catch (...) {
169  // Destructors must not throw exceptions, so we must accept the
170  // possible memory leak and move on.
171  std::cerr << "Teuchos::DefaultComm: Failed to free default Comm! We can't "
172  "throw an exception here because this is a singleton destructor that "
173  "should only be called at MPI_Finalize or (if not building with MPI) at "
174  "exit from main()." << std::endl;
175  // FIXME (mfh 16 Nov 2014) There might be some way to create a
176  // custom return code with MPI error reporting. For now, we just
177  // pick some error code not equal to MPI_SUCCESS. It could
178  // perhaps overlap with some existing error code.
179  return (MPI_SUCCESS == 0) ? -1 : 0;
180  }
181  return MPI_SUCCESS;
182 }
183 
184 template<class OrdinalType>
185 int
186 mpiFreeDefaultSerialComm (MPI_Comm, int, void*, void*)
187 {
188  try {
190  } catch (...) {
191  // Destructors must not throw exceptions, so we must accept the
192  // possible memory leak and move on.
193  std::cerr << "Teuchos::DefaultComm: Failed to free default serial Comm! "
194  "We can't throw an exception here because this is a singleton destructor "
195  "that should only be called at MPI_Finalize or (if not building with MPI)"
196  " at exit from main()." << std::endl;
197  // FIXME (mfh 16 Nov 2014) There might be some way to create a
198  // custom return code with MPI error reporting. For now, we just
199  // pick some error code not equal to MPI_SUCCESS. It could
200  // perhaps overlap with some existing error code.
201  return (MPI_SUCCESS == 0) ? -1 : 0;
202  }
203  return MPI_SUCCESS;
204 }
205 
206 } // namespace Details
207 #endif // HAVE_MPI
208 
209 
210 template<typename OrdinalType>
213 {
214  if (comm_ == NULL) {
215 #ifdef HAVE_MPI
216 # if MPI_VERSION >= 2
217 
218  comm_ = new MpiComm<OrdinalType> (MPI_COMM_WORLD);
219 
220  // We want comm_ to be deallocated when MPI_Finalize is called.
221  // The standard idiom for this (look in the MPI standard) is to
222  // register an attribute ((key,value) pair) with MPI_COMM_SELF,
223  // with a custom "destructor" to be called at MPI_Finalize.
224 
225  // 'key' is an output argument of MPI_Comm_create_keyval.
226  int key = MPI_KEYVAL_INVALID;
227  int err =
228  MPI_Comm_create_keyval (MPI_COMM_NULL_COPY_FN,
229  Details::mpiFreeDefaultComm<OrdinalType>,
230  &key,
231  NULL); // no extra state
232  if (err != MPI_SUCCESS) {
233  if (comm_ != NULL) { // clean up if MPI call fails
234  delete comm_;
235  comm_ = NULL;
236  }
237  TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
238  "Teuchos::DefaultComm::getComm: MPI_Comm_create_keyval failed!");
239  }
240  int val = key; // doesn't matter
241 
242  // Attach the attribute to MPI_COMM_SELF.
243  err = MPI_Comm_set_attr (MPI_COMM_SELF, key, &val);
244  if (err != MPI_SUCCESS) {
245  // MPI (versions up to and including 3.0) doesn't promise
246  // correct behavior after any function returns something other
247  // than MPI_SUCCESS. Thus, it's not required to try to free the
248  // new key via MPI_Comm_free_keyval. Furthermore, if something
249  // went wrong with MPI_Comm_set_attr, it's likely that the
250  // attribute mechanism is broken. Thus, it would be unwise to
251  // call MPI_Comm_free_keyval. However, we can still clean up
252  // other data.
253  if (comm_ != NULL) { // clean up if MPI call fails
254  delete comm_;
255  comm_ = NULL;
256  }
257  TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
258  "Teuchos::DefaultComm::getComm: MPI_Comm_set_attr failed!");
259  }
260 
261  // It looks weird to "free" the key right away. However, this
262  // does not actually cause the "destructor" to be called. It only
263  // gets called at MPI_FINALIZE. See MPI 3.0 standard, Section
264  // 6.7.2, MPI_COMM_FREE_KEYVAL:
265  //
266  // "Note that it is not erroneous to free an attribute key that is
267  // in use, because the actual free does not transpire until after
268  // all references (in other communicators on the process) to the
269  // key have been freed. These references need to be explicitly
270  // freed by the program, either via calls to MPI_COMM_DELETE_ATTR
271  // that free one attribute instance, or by calls to MPI_COMM_FREE
272  // that free all attribute instances associated with the freed
273  // communicator."
274  //
275  // We rely here on the latter mechanism. MPI_FINALIZE calls
276  // MPI_COMM_FREE on MPI_COMM_SELF, so we do not need to call it
277  // explicitly.
278  //
279  // It's not clear what to do if the MPI_* calls above succeeded,
280  // but this call fails (i.e., returns != MPI_SUCCESS). We could
281  // throw; this would make sense to do, because MPI (versions up to
282  // and including 3.0) doesn't promise correct behavior after any
283  // MPI function returns something other than MPI_SUCCESS. We
284  // could also be optimistic and just ignore the return value,
285  // hoping that if the above calls succeeded, then the communicator
286  // will get freed at MPI_FINALIZE, even though the unfreed key may
287  // leak memory (see Bug 6338). I've chosen the latter.
288  (void) MPI_Comm_free_keyval (&key);
289 
290 # else // MPI_VERSION < 2
291 # error "Sorry, you need an MPI implementation that supports at least MPI 2.0 in order to build this code. MPI 2.0 came out in 1997. I wrote this comment in 2017. If you really _really_ want MPI 1.x support, please file a GitHub issue for this feature request at github.com/trilinos/trilinos/issues with an expression of its priority and we will get to it as soon as we can."
292 # endif // MPI_VERSION >= 2
293 
294 #else // NOT HAVE_MPI
295  comm_ = new SerialComm<OrdinalType> ();
296  // We want comm_ to be deallocated when main exits, so register
297  // its deallocation function as an atexit handler.
298  //
299  // The POSIX standard allows atexit to fail, in particular if it
300  // lacks space for registering more functions. "[T]he application
301  // should call sysconf() to obtain the value of {ATEXIT_MAX}, the
302  // [maximum] number of functions that can be registered. There is
303  // no way for an application to tell how many functions have
304  // already been registered with atexit()."
305  //
306  // We don't do this here. Instead, we just check atexit's return
307  // code. If it fails, we throw.
308  int err = atexit (freeDefaultComm);
309  if (err != 0) {
310  if (comm_ != NULL) { // clean up if atexit fails
311  delete comm_;
312  comm_ = NULL;
313  }
314  TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
315  "Teuchos::DefaultComm::getComm: atexit failed!");
316  }
317 #endif // HAVE_MPI
318  }
319 
321  (comm_ == NULL, std::logic_error, "Teuchos::DefaultComm::getComm: "
322  "comm_ == NULL before return. This should never happen. "
323  "Please report this bug to the Teuchos developers.");
324 
325  // Return a nonowning RCP, because we need to ensure that
326  // destruction happens at MPI_Finalize (or at exit of main(), if not
327  // building with MPI).
328  return rcp (comm_, false);
329 }
330 
331 template<typename OrdinalType>
335 {
336  if (! comm.is_null ()) {
337  return comm;
338  } else {
339  if (defaultSerialComm_ == NULL) {
340 #ifdef HAVE_MPI
341 # if MPI_VERSION >= 2
342  //defaultSerialComm_ = new MpiComm<OrdinalType> (MPI_COMM_SELF);
343  defaultSerialComm_ = new SerialComm<OrdinalType> ();
344 
345  // Register an MPI_Finalize hook to free defaultSerialComm_.
346  // (See getComm implementation above in this file for details.)
347 
348  int key = MPI_KEYVAL_INVALID;
349  int err =
350  MPI_Comm_create_keyval (MPI_COMM_NULL_COPY_FN,
351  Details::mpiFreeDefaultSerialComm<OrdinalType>,
352  &key,
353  NULL); // no extra state
354  if (err != MPI_SUCCESS) {
355  if (defaultSerialComm_ != NULL) { // clean up if MPI call fails
356  delete defaultSerialComm_;
357  defaultSerialComm_ = NULL;
358  }
360  true, std::runtime_error, "Teuchos::DefaultComm::getDefaultSerialComm"
361  ": MPI_Comm_create_keyval failed!");
362  }
363  int val = key; // doesn't matter
364 
365  // Attach the attribute to MPI_COMM_SELF.
366  err = MPI_Comm_set_attr (MPI_COMM_SELF, key, &val);
367  if (err != MPI_SUCCESS) {
368  // See comments in getComm implementation above to see why we
369  // don't call MPI_Comm_free_keyval here.
370  if (defaultSerialComm_ != NULL) { // clean up if MPI call fails
371  delete defaultSerialComm_;
372  defaultSerialComm_ = NULL;
373  }
375  true, std::runtime_error, "Teuchos::DefaultComm::getDefaultSerialComm"
376  ": MPI_Comm_set_attr failed!");
377  }
378 
379  // See comments in getComm implementation above to see why we
380  // _do_ call MPI_Comm_free_keyval here, and why we don't check
381  // the return code.
382  (void) MPI_Comm_free_keyval (&key);
383 
384 # else // MPI_VERSION < 2
385 # error "Sorry, you need an MPI implementation that supports at least MPI 2.0 in order to build this code. MPI 2.0 came out in 1997. I wrote this comment in 2017. If you really _really_ want MPI 1.x support, please file a GitHub issue for this feature request at github.com/trilinos/trilinos/issues with an expression of its priority and we will get to it as soon as we can."
386 # endif // MPI_VERSION >= 2
387 
388 #else // NOT HAVE_MPI
389  defaultSerialComm_ = new SerialComm<OrdinalType> ();
390  // We want defaultSerialComm_ to be deallocated when main exits,
391  // so register its deallocation function as an atexit handler.
392  //
393  // The POSIX standard allows atexit to fail, in particular if it
394  // lacks space for registering more functions. "[T]he
395  // application should call sysconf() to obtain the value of
396  // {ATEXIT_MAX}, the [maximum] number of functions that can be
397  // registered. There is no way for an application to tell how
398  // many functions have already been registered with atexit()."
399  //
400  // We don't do this here. Instead, we just check atexit's
401  // return code. If it fails, we throw.
402  int err = atexit (freeDefaultComm);
403  if (err != 0) {
404  if (defaultSerialComm_ != NULL) { // clean up if atexit fails
405  delete defaultSerialComm_;
406  defaultSerialComm_ = NULL;
407  }
408  TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
409  "Teuchos::DefaultComm::getDefaultSerialComm: atexit failed!");
410  }
411 #endif // HAVE_MPI
412  }
413 
415  (defaultSerialComm_ == NULL, std::logic_error, "Teuchos::DefaultComm::"
416  "getDefaultSerialComm: defaultSerialComm_ == NULL before return. This sh"
417  "ould never happen. Please report this bug to the Teuchos developers.");
418 
419  // Return a nonowning RCP, because we need to ensure that
420  // destruction happens at MPI_Finalize (or at exit of main(), if not
421  // building with MPI).
422  return rcp (defaultSerialComm_, false);
423  }
424 }
425 
426 template<typename OrdinalType>
429 
430 template<typename OrdinalType>
433 
434 } // namespace Teuchos
435 
436 #endif // TEUCHOS_DEFAULT_COMM_HPP
static Teuchos::RCP< const Comm< OrdinalType > > getDefaultSerialComm(const Teuchos::RCP< const Comm< OrdinalType > > &comm)
Return a serial Comm if the input Comm is null.
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
Macro for throwing an exception with breakpointing to ease debugging.
Concrete serial communicator subclass.
static Teuchos::RCP< const Comm< OrdinalType > > getComm()
Return the default global communicator.
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)
Deprecated.
Implementation of Teuchos wrappers for MPI.
Return a default global communicator appropriate for the build.
Abstract interface for distributed-memory communication.
Smart reference counting pointer class for automatic garbage collection.
Reference-counted pointer class and non-member templated function implementations.