Teuchos - Trilinos Tools Package  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Teuchos_GlobalMPISession.cpp
1 // @HEADER
2 // *****************************************************************************
3 // Teuchos: Common Tools Package
4 //
5 // Copyright 2004 NTESS and the Teuchos contributors.
6 // SPDX-License-Identifier: BSD-3-Clause
7 // *****************************************************************************
8 // @HEADER
9 
11 #include "Teuchos_Assert.hpp"
12 
13 // The header file does not at all depend on MPI routines or types,
14 // so we can defer inclusion of mpi.h to here. This also fixes Bug
15 // 5631: https://software.sandia.gov/bugzilla/show_bug.cgi?id=5631
16 #ifdef HAVE_MPI
17 # include "mpi.h"
18 #endif
19 
20 #ifdef HAVE_TEUCHOSCORE_KOKKOS
21 # include "Kokkos_Core.hpp"
22 #endif // HAVE_TEUCHOSCORE_KOKKOS
23 
24 
25 
26 namespace Teuchos {
27 
28 
29 bool GlobalMPISession::haveMPIState_ = false;
30 bool GlobalMPISession::mpiIsFinalized_ = false;
31 int GlobalMPISession::rank_ = 0 ;
32 int GlobalMPISession::nProc_ = 1 ;
33 
34 #ifdef HAVE_TEUCHOSCORE_KOKKOS
35 
36 // We have to invoke the std::vector's constructor here,
37 // because it's a class (static) variable.
38 std::vector<std::string> GlobalMPISession::argvCopy_;
39 
40 #endif // HAVE_TEUCHOSCORE_KOKKOS
41 
42 
43 GlobalMPISession::GlobalMPISession( int* argc, char*** argv, std::ostream *out )
44 {
45  std::ostringstream oss;
46 
47  // Above is used to create all output before sending to *out to avoid
48  // jumbled parallel output between processors
49 
50 #ifdef HAVE_MPI
51 
52  int mpierr = 0;
53 
54  // Assert that MPI is not already initialized
55  int mpiHasBeenStarted = 0;
56  MPI_Initialized(&mpiHasBeenStarted);
57  if (mpiHasBeenStarted) {
58  if (out) {
59  *out << "GlobalMPISession(): Error, MPI_Intialized() return true,"
60  << " calling std::terminate()!\n"
61  << std::flush;
62  }
63  std::terminate();
64  }
65 
66  // Initialize MPI
67  mpierr = ::MPI_Init(argc, (char ***) argv);
68  if (mpierr != 0) {
69  if (out) {
70  *out << "GlobalMPISession(): Error, MPI_Init() returned error code="
71  << mpierr << "!=0, calling std::terminate()!\n"
72  << std::flush;
73  }
74  std::terminate();
75  }
76 
77  initialize(out); // Get NProc_ and rank_
78 
79  int nameLen;
80  char procName[MPI_MAX_PROCESSOR_NAME];
81  mpierr = ::MPI_Get_processor_name(procName, &nameLen);
82  if (mpierr != 0) {
83  if (out) {
84  *out << "GlobalMPISession(): Error, MPI_Get_processor_name() error code="
85  << mpierr << "!=0, calling std::terminate()!\n"
86  << std::flush;
87  }
88  std::terminate();
89  }
90 
91  oss << "Teuchos::GlobalMPISession::GlobalMPISession(): started processor with name "
92  << procName << " and rank " << rank_ << "!" << std::endl;
93 
94 #else
95 
96  oss << "Teuchos::GlobalMPISession::GlobalMPISession(): started serial run"
97  << std::endl;
98 
99 #endif
100 
101 #ifndef TEUCHOS_SUPPRESS_PROC_STARTUP_BANNER
102 
103  // See if we should suppress the startup banner
104  bool printStartupBanner = true;
105  const std::string suppress_option("--teuchos-suppress-startup-banner");
106  for ( int opt_i = 0; opt_i < *argc; ++opt_i ) {
107  if ( suppress_option == (*argv)[opt_i] ) {
108  // We are suppressing the output!
109  printStartupBanner = false;
110  // Remove this option!
111  // Note that (*argv)[*argc]==0 but convention so we copy it too!
112  for( int i = opt_i; i < *argc; ++i )
113  (*argv)[i] = (*argv)[i+1];
114  --*argc;
115  }
116  }
117  if (out && printStartupBanner) {
118  *out << oss.str() << std::flush;
119  }
120 
121 #endif
122 
123 #ifdef HAVE_TEUCHOSCORE_KOKKOS
124  // mfh 15/16 Apr 2016: This is the one chance we get to save the
125  // command-line arguments, so that we can (later) initialize Kokkos
126  // with the correct number of threads as specified by (e.g.,) the
127  // --kokkos-threads command-line argument. We won't attempt to
128  // initialize Kokkos now, because not all applications want Kokkos.
129  // Some applications may also prefer to initialize Kokkos with their
130  // own thread count.
131  //
132  // NOTE (mfh 15/16 Apr 2016): While static variables are not thread
133  // safe in general, and this is not thread safe in particular, it
134  // only makes sense to GlobalMPISession's constructor on a single
135  // thread per MPI process anyway, because MPI_Init has the same
136  // requirement.
137 
138  const int numArgs = *argc;
139  argvCopy_.resize (numArgs);
140  for (int c = 0; c < numArgs; ++c) {
141  argvCopy_[c] = std::string ((*argv)[c]); // deep copy
142  }
143 #endif // HAVE_TEUCHOSCORE_KOKKOS
144 }
145 
146 
147 #ifdef HAVE_TEUCHOSCORE_KOKKOS
148 std::vector<std::string> GlobalMPISession::getArgv ()
149 {
150  return argvCopy_;
151 }
152 #endif // HAVE_TEUCHOSCORE_KOKKOS
153 
154 
156 {
157 
158 #ifdef HAVE_TEUCHOSCORE_KOKKOS
159  try {
160  if (Kokkos::is_initialized())
161  Kokkos::finalize();
162  }
163  catch (const std::runtime_error& e) {
164  std::cerr << "Kokkos::finalize failed:\n"
165  << e.what() << "\n";
166  }
167 #endif
168 
169  haveMPIState_ = false;
170 #ifdef HAVE_MPI
171  const int mpierr = ::MPI_Finalize();
172  mpiIsFinalized_ = (mpierr == 0);
173  if (mpierr != 0)
174  std::cerr << "Error code " << mpierr << " returned from MPI_Finalize()\n";
175 #else
176  mpiIsFinalized_ = true;
177 #endif
178 }
179 
181  justInTimeInitialize();
182  #ifdef HAVE_MPI
183  MPI_Abort(MPI_COMM_WORLD, MPI_ERR_UNKNOWN);
184  #else
185  std::abort();
186  #endif
187 }
188 
190  justInTimeInitialize();
191  return haveMPIState_;
192 }
193 
194 
196 {
197  return mpiIsFinalized_;
198 }
199 
200 
202 {
203  justInTimeInitialize();
204  return rank_;
205 }
206 
207 
209  justInTimeInitialize();
210  return nProc_;
211 }
212 
213 
215 {
216  justInTimeInitialize();
217 #ifdef HAVE_MPI
218  MPI_Barrier(MPI_COMM_WORLD);
219 #endif
220 }
221 
222 
223 int GlobalMPISession::sum(int localVal)
224 {
225  justInTimeInitialize();
226 #ifdef HAVE_MPI
227  int globalSum = -1;
228  MPI_Allreduce(&localVal, &globalSum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
229  return globalSum;
230 #else
231  return localVal;
232 #endif
233 }
234 
235 
236 void GlobalMPISession::allGather(int localVal, const ArrayView<int> &allVals)
237 {
238  justInTimeInitialize();
239  TEUCHOS_ASSERT_EQUALITY(allVals.size(), getNProc());
240 #ifdef HAVE_MPI
241  MPI_Allgather( &localVal, 1, MPI_INT, allVals.getRawPtr(), 1, MPI_INT,
242  MPI_COMM_WORLD);
243 #else
244  allVals[0] = localVal;
245 #endif
246 }
247 
248 
249 // private
250 
251 
252 void GlobalMPISession::initialize( std::ostream *out )
253 {
254 #ifdef HAVE_MPI
255 
256  if(mpiIsFinalized_) {
257  // MPI has aleady been finalized so we have a serial machine again!
258  rank_ = 0;
259  nProc_ = 1;
260  return;
261  }
262 
263  if(haveMPIState_) {
264  return; // We already have what we need!
265  }
266 
267  // We don't have the state of MPI so the constructor for this class must not
268  // have been called. However, if MPI has been called in another way we
269  // can still get the state of MPI_COMM_WORLD here.
270 
271  int mpiHasBeenStarted = 0;
272  MPI_Initialized(&mpiHasBeenStarted);
273 
274  if(!mpiHasBeenStarted)
275  return; // We have to give up and just leave NProc_ and rank_ at the default values.
276 
277  // Get the state of MPI
278  // Don't throw exceptions here since this part of the code
279  // is used by TEUCHOS_STANDARD_CATCH_STATEMENTS().
280  // See bug #6192 <https://software.sandia.gov/bugzilla/show_bug.cgi?id=6192>.
281  int mpierr = 0;
282  mpierr = ::MPI_Comm_rank( MPI_COMM_WORLD, &rank_ );
283  if (mpierr != 0) {
284  *out << "Error code=" << mpierr << " detected in MPI_Comm_rank()"
285  << std::endl;
286  }
287 
288  mpierr = ::MPI_Comm_size( MPI_COMM_WORLD, &nProc_ );
289  if (mpierr != 0) {
290  *out << "Error code=" << mpierr << " detected in MPI_Comm_size()"
291  << std::endl;
292  }
293 
294  haveMPIState_ = true;
295  mpiIsFinalized_ = false;
296 
297 #endif // HAVE_MPI
298 
299 }
300 
301 
302 void GlobalMPISession::justInTimeInitialize()
303 {
304  if(!haveMPIState_)
305  initialize(&std::cerr);
306 }
307 
308 
309 } // namespace Teuchos
static int getRank()
The rank of the calling process in MPI_COMM_WORLD.
static int getNProc()
The number of processes in MPI_COMM_WORLD.
size_type size() const
The total number of items in the managed array.
static void abort()
abort the program
static bool mpiIsFinalized()
Return whether MPI was already finalized.
GlobalMPISession(int *argc, char ***argv, std::ostream *out=&std::cout)
Calls MPI_Init() if MPI is enabled.
T * getRawPtr() const
Return a raw pointer to beginning of array or NULL if unsized.
static int sum(int localVal)
Sum a set of integers across processes.
A MPI utilities class, providing methods for initializing, finalizing, and querying the global MPI se...
static void barrier()
Call MPI_Barrier() on MPI_COMM_WORLD.
static bool mpiIsInitialized()
Return whether MPI was initialized.
static void allGather(int localVal, const ArrayView< int > &allVals)
Global all-to-all of a set of integers across processes.
#define TEUCHOS_ASSERT_EQUALITY(val1, val2)
This macro is checks that to numbers are equal and if not then throws an exception with a good error ...
~GlobalMPISession()
Call MPI_Finalize() if MPI is enabled.