Teuchos - Trilinos Tools Package  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Teuchos_MpiReductionOpSetter.cpp
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Teuchos: Common Tools Package
5 // Copyright (2004) Sandia Corporation
6 //
7 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8 // license for use of this work by or on behalf of the U.S. Government.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ***********************************************************************
40 // @HEADER
41 
43 
44 #ifdef HAVE_MPI
45 # ifdef MPIAPI
46 # define CALL_API MPIAPI
47 # else
48 # define CALL_API
49 # endif
50 
51 //
52 // mfh 23 Nov 2014: My commits over the past day or two attempt to
53 // address Bug 6263. In particular, the code as I found it had the
54 // following issues:
55 //
56 // 1. Static RCP instances (that persist past return of main())
57 // 2. Static MPI_Op datum (that persists past MPI_Finalize())
58 // 3. Code won't work with MPI_THREAD_{SERIALIZED,MULTIPLE},
59 // because it assumes that only one MPI_Op for reductions
60 // is needed at any one time
61 //
62 // I'm neglecting Issue #3 for now and focusing on the first two
63 // issues. #1 goes away if one doesn't use RCPs and handles
64 // deallocation manually (we could also use std::shared_ptr, but that
65 // would require C++11). #2 goes away with the standard idiom of an
66 // MPI_Finalize() hook (attach a (key,value) pair to MPI_COMM_SELF).
67 //
68 
69 extern "C" {
70 
71 // The MPI_Op that implements the reduction or scan operation will
72 // call this function. We only need to create the MPI_Op once
73 // (lazily, on demand). This function in turn will invoke
74 // theReductOp_ (see below), which gets set to the current reduction
75 // operation. Thus, we only never need to create one MPI_Op, but we
76 // swap out the function. This is meant to save overhead in creating
77 // and freeing MPI_Op for each reduction or scan.
78 void CALL_API
79 Teuchos_MPI_reduction_op (void* invec, void* inoutvec,
80  int* len, MPI_Datatype* datatype);
81 } // extern "C"
82 
83 namespace { // anonymous
84 
85 //
86 // theMpiOp_: The MPI_Op singleton that implements the Teuchos
87 // reduction or scan operation. We only need to create the MPI_Op
88 // once (lazily, on demand). When we create the MPI_Op, we stash its
89 // "destructor" in MPI_COMM_SELF so that it gets freed at
90 // MPI_Finalize. (This is a standard MPI idiom.)
91 //
92 // This variable is global, persistent (until MPI_Finalize is called),
93 // and initialized lazily.
94 MPI_Op theMpiOp_ = MPI_OP_NULL;
95 
96 // The current reduction or scan "function." (It's actually a class
97 // instance.)
98 //
99 // This static variable is _NOT_ persistent. It does not need
100 // deallocation.
101 const Teuchos::Details::MpiReductionOpBase* theReductOp_ = NULL;
102 
103 // Free the given MPI_Op, and return the error code returned by MPI_Op_free.
104 int
105 freeMpiOp (MPI_Op* op)
106 {
107  // If this function is called as an MPI_Finalize hook, MPI should
108  // still be initialized at this point, and it should be OK to call
109  // MPI functions. Thus, we don't need to check if MPI is
110  // initialized.
111  int err = MPI_SUCCESS;
112  if (op != NULL) {
113  err = MPI_Op_free (op);
114  if (err == MPI_SUCCESS) {
115  // No externally visible side effects unless the above function succeeded.
116  *op = MPI_OP_NULL;
117  }
118  }
119  return err;
120 }
121 
122 // Free the MPI_Op singleton (theMpiOp_), and return the error code
123 // returned by freeMpiOp(). As a side effect, if freeing succeeds,
124 // set theMpiOp_ to MPI_OP_NULL.
125 //
126 // This is the singleton's "destructor" that we attach to
127 // MPI_COMM_SELF as an MPI_Finalize hook.
128 int
129 freeMpiOpCallback (MPI_Comm, int, void*, void*)
130 {
131  // We don't need any of the arguments to this function, since we're
132  // just freeing the singleton.
133  if (theMpiOp_ == MPI_OP_NULL) {
134  return MPI_SUCCESS;
135  } else {
136  return freeMpiOp (&theMpiOp_);
137  }
138 }
139 
140 // Create the MPI_Op singleton that invokes the
141 // Teuchos_MPI_reduction_op callback. Assign the MPI_Op to theMpiOp_,
142 // and set it up with an MPI_Finalize hook so it gets freed
143 // automatically.
144 void createReductOp ()
145 {
146 #if MPI_VERSION >= 2
147 
148  // This function has side effects on the global singleton theMpiOp_.
149  // This check ensures that the function is idempotent. We only need
150  // to create the MPI_Op singleton once.
151  if (theMpiOp_ != MPI_OP_NULL) {
152  return; // We've already called this function; we don't have to again.
153  }
154 
155  MPI_Op mpi_op = MPI_OP_NULL;
156 
157  // FIXME (mfh 23 Nov 2014) I found the following comment here:
158  // "Assume op is commutative". That's what it means to pass 1 as
159  // the second argument. I don't know whether it's a good idea to
160  // keep that assumption.
161  int err = MPI_Op_create (&Teuchos_MPI_reduction_op, 1, &mpi_op);
163  err != MPI_SUCCESS, std::runtime_error, "Teuchos::createReductOp: "
164  "MPI_Op_create (for custom reduction operator) failed!");
165 
166  // Use the standard MPI idiom (attach a (key,value) pair to
167  // MPI_COMM_SELF with a "destructor" function) in order that
168  // theMpiOp_ gets freed at MPI_Finalize, if necessary.
169 
170  // 'key' is an output argument of MPI_Comm_create_keyval.
171  int key = MPI_KEYVAL_INVALID;
172  err = MPI_Comm_create_keyval (MPI_COMM_NULL_COPY_FN, freeMpiOpCallback,
173  &key, NULL);
174  if (err != MPI_SUCCESS) {
175  // Attempt to clean up by freeing the newly created MPI_Op. If
176  // cleaning up fails, just let it slide, since we're already in
177  // trouble if MPI can't create a (key,value) pair.
178  (void) MPI_Op_free (&mpi_op);
180  true, std::runtime_error, "Teuchos::createReductOp: "
181  "MPI_Comm_create_keyval (for custom reduction operator) failed!");
182  }
183  int val = key; // doesn't matter
184 
185  // Attach the attribute to MPI_COMM_SELF.
186  err = MPI_Comm_set_attr (MPI_COMM_SELF, key, &val);
187  if (err != MPI_SUCCESS) {
188  // MPI (versions up to and including 3.0) doesn't promise correct
189  // behavior after any function returns something other than
190  // MPI_SUCCESS. Thus, it's not required to try to free the new
191  // key via MPI_Comm_free_keyval. Furthermore, if something went
192  // wrong with MPI_Comm_set_attr, it's likely that the attribute
193  // mechanism is broken. Thus, it would be unwise to call
194  // MPI_Comm_free_keyval.
195  //
196  // I optimistically assume that the "rest" of MPI is still
197  // working, and attempt to clean up by freeing the newly created
198  // MPI_Op. If cleaning up fails, just let it slide, since we're
199  // already in trouble if MPI can't create a (key,value) pair.
200  (void) MPI_Op_free (&mpi_op);
202  true, std::runtime_error, "Teuchos::createReductOp: "
203  "MPI_Comm_set_attr (for custom reduction operator) failed!");
204  }
205 
206  // It looks weird to "free" the key right away. However, this does
207  // not actually cause the "destructor" to be called. It only gets
208  // called at MPI_FINALIZE. See MPI 3.0 standard, Section 6.7.2,
209  // MPI_COMM_FREE_KEYVAL:
210  //
211  // "Note that it is not erroneous to free an attribute key that is
212  // in use, because the actual free does not transpire until after
213  // all references (in other communicators on the process) to the key
214  // have been freed. These references need to be explicitly freed by
215  // the program, either via calls to MPI_COMM_DELETE_ATTR that free
216  // one attribute instance, or by calls to MPI_COMM_FREE that free
217  // all attribute instances associated with the freed communicator."
218  //
219  // We rely here on the latter mechanism. MPI_FINALIZE calls
220  // MPI_COMM_FREE on MPI_COMM_SELF, so we do not need to call it
221  // explicitly.
222  //
223  // It's not clear what to do if the MPI_* calls above succeeded, but
224  // this call fails (i.e., returns != MPI_SUCCESS). We could throw;
225  // this would make sense to do, because MPI (versions up to and
226  // including 3.0) doesn't promise correct behavior after any MPI
227  // function returns something other than MPI_SUCCESS. We could also
228  // be optimistic and just ignore the return value, hoping that if
229  // the above calls succeeded, then the communicator will get freed
230  // at MPI_FINALIZE, even though the unfreed key may leak memory (see
231  // Bug 6338). I've chosen the latter.
232  (void) MPI_Comm_free_keyval (&key);
233 
234  // The "transaction" succeeded; save the result.
235  theMpiOp_ = mpi_op;
236 
237 #else // MPI_VERSION < 2
238 # error "Sorry, you need an MPI implementation that supports at least MPI 2.0 in order to build this code. MPI 2.0 came out in 1997. I wrote this comment in 2017. If you really _really_ want MPI 1.x support, please file a GitHub issue for this feature request at github.com/trilinos/trilinos/issues with an expression of its priority and we will get to it as soon as we can."
239 #endif // MPI_VERSION >= 2
240 }
241 
242 void
243 setReductOp (const Teuchos::Details::MpiReductionOpBase* reductOp)
244 {
245  if (theMpiOp_ == MPI_OP_NULL) {
246  createReductOp ();
247  }
248  theReductOp_ = reductOp;
249 }
250 
251 } // namespace (anonymous)
252 
253 extern "C" {
254 
255 void CALL_API
256 Teuchos_MPI_reduction_op (void* invec,
257  void* inoutvec,
258  int* len,
259  MPI_Datatype* datatype)
260 {
261  if (theReductOp_ != NULL) {
262  theReductOp_->reduce (invec, inoutvec, len, datatype);
263  }
264 }
265 
266 } // extern "C"
267 
268 namespace Teuchos {
269 namespace Details {
270 
271 MPI_Op setMpiReductionOp (const MpiReductionOpBase& reductOp)
272 {
273  setReductOp (&reductOp);
275  (theMpiOp_ == MPI_OP_NULL, std::logic_error, "Teuchos::Details::"
276  "setMpiReductionOp: Failed to create reduction MPI_Op theMpiOp_. "
277  "This should never happen. "
278  "Please report this bug to the Teuchos developers.");
279  return theMpiOp_;
280 }
281 
282 } // namespace Details
283 } // namespace Teuchos
284 
285 #endif // HAVE_MPI
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
Macro for throwing an exception with breakpointing to ease debugging.
Implementation detail of Teuchos&#39; MPI wrapper.