Kokkos Core Kernels Package
Version of the Day
Main Page
Related Pages
Namespaces
Classes
Files
File List
All
Classes
Namespaces
Files
Functions
Variables
Typedefs
Enumerations
Enumerator
Friends
Groups
Pages
core
src
Kokkos_Atomic.hpp
Go to the documentation of this file.
1
/*
2
//@HEADER
3
// ************************************************************************
4
//
5
// Kokkos v. 3.0
6
// Copyright (2020) National Technology & Engineering
7
// Solutions of Sandia, LLC (NTESS).
8
//
9
// Under the terms of Contract DE-NA0003525 with NTESS,
10
// the U.S. Government retains certain rights in this software.
11
//
12
// Redistribution and use in source and binary forms, with or without
13
// modification, are permitted provided that the following conditions are
14
// met:
15
//
16
// 1. Redistributions of source code must retain the above copyright
17
// notice, this list of conditions and the following disclaimer.
18
//
19
// 2. Redistributions in binary form must reproduce the above copyright
20
// notice, this list of conditions and the following disclaimer in the
21
// documentation and/or other materials provided with the distribution.
22
//
23
// 3. Neither the name of the Corporation nor the names of the
24
// contributors may be used to endorse or promote products derived from
25
// this software without specific prior written permission.
26
//
27
// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
28
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
31
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
32
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
33
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
34
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
35
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
36
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
37
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38
//
39
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
40
//
41
// ************************************************************************
42
//@HEADER
43
*/
44
67
68
#ifndef KOKKOS_ATOMIC_HPP
69
#define KOKKOS_ATOMIC_HPP
70
71
#include <Kokkos_Macros.hpp>
72
#include <Kokkos_HostSpace.hpp>
73
#include <impl/Kokkos_Traits.hpp>
74
75
//----------------------------------------------------------------------------
76
#if defined(_WIN32)
77
#define KOKKOS_ENABLE_WINDOWS_ATOMICS
78
#else
79
#if defined(KOKKOS_ENABLE_CUDA)
80
81
// Compiling NVIDIA device code, must use Cuda atomics:
82
83
#define KOKKOS_ENABLE_CUDA_ATOMICS
84
85
#elif defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_ROCM_GPU)
86
87
#define KOKKOS_ENABLE_ROCM_ATOMICS
88
89
#elif defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HIP_GPU)
90
91
#define KOKKOS_ENABLE_HIP_ATOMICS
92
93
#endif
94
95
#if !defined(KOKKOS_ENABLE_GNU_ATOMICS) && \
96
!defined(KOKKOS_ENABLE_INTEL_ATOMICS) && \
97
!defined(KOKKOS_ENABLE_OPENMP_ATOMICS) && \
98
!defined(KOKKOS_ENABLE_STD_ATOMICS) && \
99
!defined(KOKKOS_ENABLE_SERIAL_ATOMICS)
100
101
// Compiling for non-Cuda atomic implementation has not been pre-selected.
102
// Choose the best implementation for the detected compiler.
103
// Preference: GCC, INTEL, OMP31
104
105
#if defined(KOKKOS_INTERNAL_NOT_PARALLEL)
106
107
#define KOKKOS_ENABLE_SERIAL_ATOMICS
108
109
#elif defined(KOKKOS_COMPILER_GNU) || defined(KOKKOS_COMPILER_CLANG) || \
110
(defined(KOKKOS_COMPILER_NVCC))
111
112
#define KOKKOS_ENABLE_GNU_ATOMICS
113
114
#elif defined(KOKKOS_COMPILER_INTEL) || defined(KOKKOS_COMPILER_CRAYC)
115
116
#define KOKKOS_ENABLE_INTEL_ATOMICS
117
118
#elif defined(_OPENMP) && (201107 <= _OPENMP)
119
120
#define KOKKOS_ENABLE_OPENMP_ATOMICS
121
122
#else
123
124
#error "KOKKOS_ATOMICS_USE : Unsupported compiler"
125
126
#endif
127
128
#endif
/* Not pre-selected atomic implementation */
129
#endif
130
131
#ifdef KOKKOS_ENABLE_CUDA
132
#include <Cuda/Kokkos_Cuda_Locks.hpp>
133
#endif
134
135
namespace
Kokkos {
136
template
<
typename
T>
137
KOKKOS_INLINE_FUNCTION
void
atomic_add(
volatile
T*
const
dest,
const
T src);
138
139
// Atomic increment
140
template
<
typename
T>
141
KOKKOS_INLINE_FUNCTION
void
atomic_increment(
volatile
T* a);
142
143
template
<
typename
T>
144
KOKKOS_INLINE_FUNCTION
void
atomic_decrement(
volatile
T* a);
145
}
// namespace Kokkos
146
147
namespace
Kokkos {
148
149
inline
const
char
* atomic_query_version() {
150
#if defined(KOKKOS_ENABLE_CUDA_ATOMICS)
151
return
"KOKKOS_ENABLE_CUDA_ATOMICS"
;
152
#elif defined(KOKKOS_ENABLE_GNU_ATOMICS)
153
return
"KOKKOS_ENABLE_GNU_ATOMICS"
;
154
#elif defined(KOKKOS_ENABLE_INTEL_ATOMICS)
155
return
"KOKKOS_ENABLE_INTEL_ATOMICS"
;
156
#elif defined(KOKKOS_ENABLE_OPENMP_ATOMICS)
157
return
"KOKKOS_ENABLE_OPENMP_ATOMICS"
;
158
#elif defined(KOKKOS_ENABLE_WINDOWS_ATOMICS)
159
return
"KOKKOS_ENABLE_WINDOWS_ATOMICS"
;
160
#elif defined(KOKKOS_ENABLE_SERIAL_ATOMICS)
161
return
"KOKKOS_ENABLE_SERIAL_ATOMICS"
;
162
#else
163
#error "No valid response for atomic_query_version!"
164
#endif
165
}
166
167
}
// namespace Kokkos
168
169
//----------------------------------------------------------------------------
170
// Atomic Memory Orders
171
//
172
// Implements Strongly-typed analogs of C++ standard memory orders
173
#include "impl/Kokkos_Atomic_Memory_Order.hpp"
174
175
#if defined(KOKKOS_ENABLE_ROCM)
176
namespace
Kokkos {
177
namespace
Impl {
178
extern
KOKKOS_INLINE_FUNCTION
bool
lock_address_rocm_space(
void
* ptr);
179
180
extern
KOKKOS_INLINE_FUNCTION
void
unlock_address_rocm_space(
void
* ptr);
181
}
// namespace Impl
182
}
// namespace Kokkos
183
#include <ROCm/Kokkos_ROCm_Atomic.hpp>
184
#endif
185
#if defined(KOKKOS_ENABLE_HIP)
186
#include <HIP/Kokkos_HIP_Atomic.hpp>
187
#endif
188
189
#ifdef _WIN32
190
#include "impl/Kokkos_Atomic_Windows.hpp"
191
#else
192
//----------------------------------------------------------------------------
193
// Atomic Assembly
194
//
195
// Implements CAS128-bit in assembly
196
197
#include "impl/Kokkos_Atomic_Assembly.hpp"
198
199
//----------------------------------------------------------------------------
200
// Atomic exchange
201
//
202
// template< typename T >
203
// T atomic_exchange( volatile T* const dest , const T val )
204
// { T tmp = *dest ; *dest = val ; return tmp ; }
205
206
#include "impl/Kokkos_Atomic_Exchange.hpp"
207
208
//----------------------------------------------------------------------------
209
// Atomic compare-and-exchange
210
//
211
// template<class T>
212
// bool atomic_compare_exchange_strong(volatile T* const dest, const T compare,
213
// const T val) { bool equal = compare == *dest ; if ( equal ) { *dest = val ; }
214
// return equal ; }
215
216
#include "impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp"
217
218
#endif //_WIN32
219
220
#include "impl/Kokkos_Atomic_Generic.hpp"
221
222
#ifndef _WIN32
223
//----------------------------------------------------------------------------
224
// Atomic fetch and add
225
//
226
// template<class T>
227
// T atomic_fetch_add(volatile T* const dest, const T val)
228
// { T tmp = *dest ; *dest += val ; return tmp ; }
229
230
#include "impl/Kokkos_Atomic_Fetch_Add.hpp"
231
232
//----------------------------------------------------------------------------
233
// Atomic increment
234
//
235
// template<class T>
236
// T atomic_increment(volatile T* const dest)
237
// { dest++; }
238
239
#include "impl/Kokkos_Atomic_Increment.hpp"
240
241
//----------------------------------------------------------------------------
242
// Atomic Decrement
243
//
244
// template<class T>
245
// T atomic_decrement(volatile T* const dest)
246
// { dest--; }
247
248
#include "impl/Kokkos_Atomic_Decrement.hpp"
249
250
//----------------------------------------------------------------------------
251
// Atomic fetch and sub
252
//
253
// template<class T>
254
// T atomic_fetch_sub(volatile T* const dest, const T val)
255
// { T tmp = *dest ; *dest -= val ; return tmp ; }
256
257
#include "impl/Kokkos_Atomic_Fetch_Sub.hpp"
258
259
//----------------------------------------------------------------------------
260
// Atomic fetch and or
261
//
262
// template<class T>
263
// T atomic_fetch_or(volatile T* const dest, const T val)
264
// { T tmp = *dest ; *dest = tmp | val ; return tmp ; }
265
266
#include "impl/Kokkos_Atomic_Fetch_Or.hpp"
267
268
//----------------------------------------------------------------------------
269
// Atomic fetch and and
270
//
271
// template<class T>
272
// T atomic_fetch_and(volatile T* const dest, const T val)
273
// { T tmp = *dest ; *dest = tmp & val ; return tmp ; }
274
275
#include "impl/Kokkos_Atomic_Fetch_And.hpp"
276
277
//----------------------------------------------------------------------------
278
// Atomic MinMax
279
//
280
// template<class T>
281
// T atomic_min(volatile T* const dest, const T val)
282
// { T tmp = *dest ; *dest = min(*dest, val); return tmp ; }
283
// template<class T>
284
// T atomic_max(volatile T* const dest, const T val)
285
// { T tmp = *dest ; *dest = max(*dest, val); return tmp ; }
286
287
#include "impl/Kokkos_Atomic_MinMax.hpp"
288
#endif
/*Not _WIN32*/
289
290
//----------------------------------------------------------------------------
291
// Memory fence
292
//
293
// All loads and stores from this thread will be globally consistent before
294
// continuing
295
//
296
// void memory_fence() {...};
297
#include "impl/Kokkos_Memory_Fence.hpp"
298
299
//----------------------------------------------------------------------------
300
// Provide volatile_load and safe_load
301
//
302
// T volatile_load(T const volatile * const ptr);
303
//
304
// T const& safe_load(T const * const ptr);
305
// XEON PHI
306
// T safe_load(T const * const ptr
307
308
#include "impl/Kokkos_Volatile_Load.hpp"
309
310
//----------------------------------------------------------------------------
311
// Provide atomic loads and stores with memory order semantics
312
313
#include "impl/Kokkos_Atomic_Load.hpp"
314
#include "impl/Kokkos_Atomic_Store.hpp"
315
316
// Generic functions using the above defined functions
317
#include "impl/Kokkos_Atomic_Generic_Secondary.hpp"
318
//----------------------------------------------------------------------------
319
// This atomic-style macro should be an inlined function, not a macro
320
321
#if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \
322
!defined(__CUDA_ARCH__)
323
324
#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0)
325
#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0)
326
327
#else
328
329
#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
330
#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
331
332
#endif
333
334
//----------------------------------------------------------------------------
335
336
#endif
/* KOKKOS_ATOMIC_HPP */
Generated by
1.8.5