Teuchos Package Browser (Single Doxygen Collection)  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Teuchos_GlobalMPISession.cpp
Go to the documentation of this file.
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Teuchos: Common Tools Package
5 // Copyright (2004) Sandia Corporation
6 //
7 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8 // license for use of this work by or on behalf of the U.S. Government.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ***********************************************************************
40 // @HEADER
41 
43 #include "Teuchos_Assert.hpp"
44 
45 // The header file does not at all depend on MPI routines or types,
46 // so we can defer inclusion of mpi.h to here. This also fixes Bug
47 // 5631: https://software.sandia.gov/bugzilla/show_bug.cgi?id=5631
48 #ifdef HAVE_MPI
49 # include "mpi.h"
50 #endif
51 
52 #ifdef HAVE_TEUCHOSCORE_KOKKOSCORE
53 # include "Kokkos_Core.hpp"
54 #endif // HAVE_TEUCHOSCORE_KOKKOSCORE
55 
56 
57 
58 namespace Teuchos {
59 
60 
65 
66 #ifdef HAVE_TEUCHOSCORE_KOKKOSCORE
67 
68 // We have to invoke the std::vector's constructor here,
69 // because it's a class (static) variable.
70 std::vector<std::string> GlobalMPISession::argvCopy_;
71 
72 #endif // HAVE_TEUCHOSCORE_KOKKOSCORE
73 
74 
75 GlobalMPISession::GlobalMPISession( int* argc, char*** argv, std::ostream *out )
76 {
77  std::ostringstream oss;
78 
79  // Above is used to create all output before sending to *out to avoid
80  // jumbled parallel output between processors
81 
82 #ifdef HAVE_MPI
83 
84  int mpierr = 0;
85 
86  // Assert that MPI is not already initialized
87  int mpiHasBeenStarted = 0;
88  MPI_Initialized(&mpiHasBeenStarted);
89  if (mpiHasBeenStarted) {
90  if (out) {
91  *out << "GlobalMPISession(): Error, MPI_Intialized() return true,"
92  << " calling std::terminate()!\n"
93  << std::flush;
94  }
95  std::terminate();
96  }
97 
98  // Initialize MPI
99  mpierr = ::MPI_Init(argc, (char ***) argv);
100  if (mpierr != 0) {
101  if (out) {
102  *out << "GlobalMPISession(): Error, MPI_Init() returned error code="
103  << mpierr << "!=0, calling std::terminate()!\n"
104  << std::flush;
105  }
106  std::terminate();
107  }
108 
109  initialize(out); // Get NProc_ and rank_
110 
111  int nameLen;
112  char procName[MPI_MAX_PROCESSOR_NAME];
113  mpierr = ::MPI_Get_processor_name(procName, &nameLen);
114  if (mpierr != 0) {
115  if (out) {
116  *out << "GlobalMPISession(): Error, MPI_Get_processor_name() error code="
117  << mpierr << "!=0, calling std::terminate()!\n"
118  << std::flush;
119  }
120  std::terminate();
121  }
122 
123  oss << "Teuchos::GlobalMPISession::GlobalMPISession(): started processor with name "
124  << procName << " and rank " << rank_ << "!" << std::endl;
125 
126 #else
127 
128  oss << "Teuchos::GlobalMPISession::GlobalMPISession(): started serial run"
129  << std::endl;
130 
131 #endif
132 
133 #ifndef TEUCHOS_SUPPRESS_PROC_STARTUP_BANNER
134 
135  // See if we should suppress the startup banner
136  bool printStartupBanner = true;
137  const std::string suppress_option("--teuchos-suppress-startup-banner");
138  for ( int opt_i = 0; opt_i < *argc; ++opt_i ) {
139  if ( suppress_option == (*argv)[opt_i] ) {
140  // We are suppressing the output!
141  printStartupBanner = false;
142  // Remove this option!
143  // Note that (*argv)[*argc]==0 but convention so we copy it too!
144  for( int i = opt_i; i < *argc; ++i )
145  (*argv)[i] = (*argv)[i+1];
146  --*argc;
147  }
148  }
149  if (out && printStartupBanner) {
150  *out << oss.str() << std::flush;
151  }
152 
153 #endif
154 
155 #ifdef HAVE_TEUCHOSCORE_KOKKOSCORE
156  // mfh 15/16 Apr 2016: This is the one chance we get to save the
157  // command-line arguments, so that we can (later) initialize Kokkos
158  // with the correct number of threads as specified by (e.g.,) the
159  // --kokkos-threads command-line argument. We won't attempt to
160  // initialize Kokkos now, because not all applications want Kokkos.
161  // Some applications may also prefer to initialize Kokkos with their
162  // own thread count.
163  //
164  // NOTE (mfh 15/16 Apr 2016): While static variables are not thread
165  // safe in general, and this is not thread safe in particular, it
166  // only makes sense to GlobalMPISession's constructor on a single
167  // thread per MPI process anyway, because MPI_Init has the same
168  // requirement.
169 
170  const int numArgs = *argc;
171  argvCopy_.resize (numArgs);
172  for (int c = 0; c < numArgs; ++c) {
173  argvCopy_[c] = std::string ((*argv)[c]); // deep copy
174  }
175 #endif // HAVE_TEUCHOSCORE_KOKKOSCORE
176 }
177 
178 
179 #ifdef HAVE_TEUCHOSCORE_KOKKOSCORE
180 std::vector<std::string> GlobalMPISession::getArgv ()
181 {
182  return argvCopy_;
183 }
184 #endif // HAVE_TEUCHOSCORE_KOKKOSCORE
185 
186 
188 {
189 
190 #ifdef HAVE_TEUCHOSCORE_KOKKOSCORE
191  try {
192  Kokkos::finalize_all();
193  }
194  catch (const std::runtime_error& e) {
195  std::cerr << "Kokkos::finalize_all failed:\n"
196  << e.what() << "\n";
197  }
198 #endif
199 
200  haveMPIState_ = false;
201 #ifdef HAVE_MPI
202  const int mpierr = ::MPI_Finalize();
203  mpiIsFinalized_ = (mpierr == 0);
204  if (mpierr != 0)
205  std::cerr << "Error code " << mpierr << " returned from MPI_Finalize()\n";
206 #else
207  mpiIsFinalized_ = true;
208 #endif
209 }
210 
213  #ifdef HAVE_MPI
214  MPI_Abort(MPI_COMM_WORLD, MPI_ERR_UNKNOWN);
215  #else
216  std::abort();
217  #endif
218 }
219 
222  return haveMPIState_;
223 }
224 
225 
227 {
228  return mpiIsFinalized_;
229 }
230 
231 
233 {
235  return rank_;
236 }
237 
238 
241  return nProc_;
242 }
243 
244 
246 {
248 #ifdef HAVE_MPI
249  MPI_Barrier(MPI_COMM_WORLD);
250 #endif
251 }
252 
253 
254 int GlobalMPISession::sum(int localVal)
255 {
257 #ifdef HAVE_MPI
258  int globalSum = -1;
259  MPI_Allreduce(&localVal, &globalSum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
260  return globalSum;
261 #else
262  return localVal;
263 #endif
264 }
265 
266 
267 void GlobalMPISession::allGather(int localVal, const ArrayView<int> &allVals)
268 {
270  TEUCHOS_ASSERT_EQUALITY(allVals.size(), getNProc());
271 #ifdef HAVE_MPI
272  MPI_Allgather( &localVal, 1, MPI_INT, allVals.getRawPtr(), 1, MPI_INT,
273  MPI_COMM_WORLD);
274 #else
275  allVals[0] = localVal;
276 #endif
277 }
278 
279 
280 // private
281 
282 
283 void GlobalMPISession::initialize( std::ostream *out )
284 {
285 #ifdef HAVE_MPI
286 
287  if(mpiIsFinalized_) {
288  // MPI has aleady been finalized so we have a serial machine again!
289  rank_ = 0;
290  nProc_ = 1;
291  return;
292  }
293 
294  if(haveMPIState_) {
295  return; // We already have what we need!
296  }
297 
298  // We don't have the state of MPI so the constructor for this class must not
299  // have been called. However, if MPI has been called in another way we
300  // can still get the state of MPI_COMM_WORLD here.
301 
302  int mpiHasBeenStarted = 0;
303  MPI_Initialized(&mpiHasBeenStarted);
304 
305  if(!mpiHasBeenStarted)
306  return; // We have to give up and just leave NProc_ and rank_ at the default values.
307 
308  // Get the state of MPI
309  // Don't throw exceptions here since this part of the code
310  // is used by TEUCHOS_STANDARD_CATCH_STATEMENTS().
311  // See bug #6192 <https://software.sandia.gov/bugzilla/show_bug.cgi?id=6192>.
312  int mpierr = 0;
313  mpierr = ::MPI_Comm_rank( MPI_COMM_WORLD, &rank_ );
314  if (mpierr != 0) {
315  *out << "Error code=" << mpierr << " detected in MPI_Comm_rank()"
316  << std::endl;
317  }
318 
319  mpierr = ::MPI_Comm_size( MPI_COMM_WORLD, &nProc_ );
320  if (mpierr != 0) {
321  *out << "Error code=" << mpierr << " detected in MPI_Comm_size()"
322  << std::endl;
323  }
324 
325  haveMPIState_ = true;
326  mpiIsFinalized_ = false;
327 
328 #endif // HAVE_MPI
329 
330 }
331 
332 
334 {
335  if(!haveMPIState_)
336  initialize(&std::cerr);
337 }
338 
339 
340 } // namespace Teuchos
static int getRank()
The rank of the calling process in MPI_COMM_WORLD.
static int getNProc()
The number of processes in MPI_COMM_WORLD.
size_type size() const
The total number of items in the managed array.
static void abort()
abort the program
static bool mpiIsFinalized()
Return whether MPI was already finalized.
GlobalMPISession(int *argc, char ***argv, std::ostream *out=&std::cout)
Calls MPI_Init() if MPI is enabled.
T * getRawPtr() const
Return a raw pointer to beginning of array or NULL if unsized.
static int sum(int localVal)
Sum a set of integers across processes.
A MPI utilities class, providing methods for initializing, finalizing, and querying the global MPI se...
static void barrier()
Call MPI_Barrier() on MPI_COMM_WORLD.
static bool mpiIsInitialized()
Return whether MPI was initialized.
static void allGather(int localVal, const ArrayView< int > &allVals)
Global all-to-all of a set of integers across processes.
#define TEUCHOS_ASSERT_EQUALITY(val1, val2)
This macro is checks that to numbers are equal and if not then throws an exception with a good error ...
static void initialize(std::ostream *out)
~GlobalMPISession()
Call MPI_Finalize() if MPI is enabled.