Kokkos Core Kernels Package
Version of the Day
Main Page
Related Pages
Namespaces
Classes
Files
File List
All
Classes
Namespaces
Files
Functions
Variables
Typedefs
Enumerations
Enumerator
Friends
Groups
Pages
core
src
Kokkos_Atomic.hpp
Go to the documentation of this file.
1
/*
2
//@HEADER
3
// ************************************************************************
4
//
5
// Kokkos v. 2.0
6
// Copyright (2014) Sandia Corporation
7
//
8
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9
// the U.S. Government retains certain rights in this software.
10
//
11
// Redistribution and use in source and binary forms, with or without
12
// modification, are permitted provided that the following conditions are
13
// met:
14
//
15
// 1. Redistributions of source code must retain the above copyright
16
// notice, this list of conditions and the following disclaimer.
17
//
18
// 2. Redistributions in binary form must reproduce the above copyright
19
// notice, this list of conditions and the following disclaimer in the
20
// documentation and/or other materials provided with the distribution.
21
//
22
// 3. Neither the name of the Corporation nor the names of the
23
// contributors may be used to endorse or promote products derived from
24
// this software without specific prior written permission.
25
//
26
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37
//
38
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
39
//
40
// ************************************************************************
41
//@HEADER
42
*/
43
66
67
#ifndef KOKKOS_ATOMIC_HPP
68
#define KOKKOS_ATOMIC_HPP
69
70
#include <Kokkos_Macros.hpp>
71
#include <Kokkos_HostSpace.hpp>
72
#include <impl/Kokkos_Traits.hpp>
73
74
//----------------------------------------------------------------------------
75
#if defined(_WIN32)
76
#define KOKKOS_ENABLE_WINDOWS_ATOMICS
77
#else
78
#if defined( KOKKOS_ENABLE_CUDA )
79
80
// Compiling NVIDIA device code, must use Cuda atomics:
81
82
#define KOKKOS_ENABLE_CUDA_ATOMICS
83
84
#elif defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_ROCM_GPU)
85
86
#define KOKKOS_ENABLE_ROCM_ATOMICS
87
88
#endif
89
90
#if ! defined( KOKKOS_ENABLE_GNU_ATOMICS ) && \
91
! defined( KOKKOS_ENABLE_INTEL_ATOMICS ) && \
92
! defined( KOKKOS_ENABLE_OPENMP_ATOMICS ) && \
93
! defined( KOKKOS_ENABLE_STD_ATOMICS ) && \
94
! defined( KOKKOS_ENABLE_SERIAL_ATOMICS )
95
96
// Compiling for non-Cuda atomic implementation has not been pre-selected.
97
// Choose the best implementation for the detected compiler.
98
// Preference: GCC, INTEL, OMP31
99
100
#if defined( KOKKOS_INTERNAL_NOT_PARALLEL )
101
102
#define KOKKOS_ENABLE_SERIAL_ATOMICS
103
104
#elif defined( KOKKOS_COMPILER_GNU ) || \
105
defined( KOKKOS_COMPILER_CLANG ) || \
106
( defined ( KOKKOS_COMPILER_NVCC ) )
107
108
#define KOKKOS_ENABLE_GNU_ATOMICS
109
110
#elif defined( KOKKOS_COMPILER_INTEL ) || \
111
defined( KOKKOS_COMPILER_CRAYC )
112
113
#define KOKKOS_ENABLE_INTEL_ATOMICS
114
115
#elif defined( _OPENMP ) && ( 201107 <= _OPENMP )
116
117
#define KOKKOS_ENABLE_OPENMP_ATOMICS
118
119
#else
120
121
#error "KOKKOS_ATOMICS_USE : Unsupported compiler"
122
123
#endif
124
125
#endif
/* Not pre-selected atomic implementation */
126
#endif
127
128
#ifdef KOKKOS_ENABLE_CUDA
129
#include <Cuda/Kokkos_Cuda_Locks.hpp>
130
#endif
131
132
namespace
Kokkos {
133
template
<
typename
T>
134
KOKKOS_INLINE_FUNCTION
135
void
atomic_add(
volatile
T *
const
dest,
const
T src);
136
137
// Atomic increment
138
template
<
typename
T>
139
KOKKOS_INLINE_FUNCTION
140
void
atomic_increment(
volatile
T* a);
141
142
template
<
typename
T>
143
KOKKOS_INLINE_FUNCTION
144
void
atomic_decrement(
volatile
T* a);
145
}
146
147
namespace
Kokkos {
148
149
150
inline
151
const
char
* atomic_query_version()
152
{
153
#if defined( KOKKOS_ENABLE_CUDA_ATOMICS )
154
return
"KOKKOS_ENABLE_CUDA_ATOMICS"
;
155
#elif defined( KOKKOS_ENABLE_GNU_ATOMICS )
156
return
"KOKKOS_ENABLE_GNU_ATOMICS"
;
157
#elif defined( KOKKOS_ENABLE_INTEL_ATOMICS )
158
return
"KOKKOS_ENABLE_INTEL_ATOMICS"
;
159
#elif defined( KOKKOS_ENABLE_OPENMP_ATOMICS )
160
return
"KOKKOS_ENABLE_OPENMP_ATOMICS"
;
161
#elif defined( KOKKOS_ENABLE_WINDOWS_ATOMICS )
162
return
"KOKKOS_ENABLE_WINDOWS_ATOMICS"
;
163
#elif defined( KOKKOS_ENABLE_SERIAL_ATOMICS )
164
return
"KOKKOS_ENABLE_SERIAL_ATOMICS"
;
165
#else
166
#error "No valid response for atomic_query_version!"
167
#endif
168
}
169
170
}
// namespace Kokkos
171
172
//----------------------------------------------------------------------------
173
// Atomic Memory Orders
174
//
175
// Implements Strongly-typed analogs of C++ standard memory orders
176
#include "impl/Kokkos_Atomic_Memory_Order.hpp"
177
178
#if defined( KOKKOS_ENABLE_ROCM )
179
namespace
Kokkos {
180
namespace
Impl {
181
extern
KOKKOS_INLINE_FUNCTION
182
bool
lock_address_rocm_space(
void
* ptr);
183
184
extern
KOKKOS_INLINE_FUNCTION
185
void
unlock_address_rocm_space(
void
* ptr);
186
}
187
}
188
#include <ROCm/Kokkos_ROCm_Atomic.hpp>
189
#endif
190
191
#ifdef _WIN32
192
#include "impl/Kokkos_Atomic_Windows.hpp"
193
#else
194
195
//----------------------------------------------------------------------------
196
// Atomic Assembly
197
//
198
// Implements CAS128-bit in assembly
199
200
#include "impl/Kokkos_Atomic_Assembly.hpp"
201
202
//----------------------------------------------------------------------------
203
// Atomic exchange
204
//
205
// template< typename T >
206
// T atomic_exchange( volatile T* const dest , const T val )
207
// { T tmp = *dest ; *dest = val ; return tmp ; }
208
209
#include "impl/Kokkos_Atomic_Exchange.hpp"
210
211
//----------------------------------------------------------------------------
212
// Atomic compare-and-exchange
213
//
214
// template<class T>
215
// bool atomic_compare_exchange_strong(volatile T* const dest, const T compare, const T val)
216
// { bool equal = compare == *dest ; if ( equal ) { *dest = val ; } return equal ; }
217
218
#include "impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp"
219
220
//----------------------------------------------------------------------------
221
// Atomic fetch and add
222
//
223
// template<class T>
224
// T atomic_fetch_add(volatile T* const dest, const T val)
225
// { T tmp = *dest ; *dest += val ; return tmp ; }
226
227
#include "impl/Kokkos_Atomic_Fetch_Add.hpp"
228
229
//----------------------------------------------------------------------------
230
// Atomic increment
231
//
232
// template<class T>
233
// T atomic_increment(volatile T* const dest)
234
// { dest++; }
235
236
#include "impl/Kokkos_Atomic_Increment.hpp"
237
238
//----------------------------------------------------------------------------
239
// Atomic Decrement
240
//
241
// template<class T>
242
// T atomic_decrement(volatile T* const dest)
243
// { dest--; }
244
245
#include "impl/Kokkos_Atomic_Decrement.hpp"
246
247
//----------------------------------------------------------------------------
248
// Atomic fetch and sub
249
//
250
// template<class T>
251
// T atomic_fetch_sub(volatile T* const dest, const T val)
252
// { T tmp = *dest ; *dest -= val ; return tmp ; }
253
254
#include "impl/Kokkos_Atomic_Fetch_Sub.hpp"
255
256
//----------------------------------------------------------------------------
257
// Atomic fetch and or
258
//
259
// template<class T>
260
// T atomic_fetch_or(volatile T* const dest, const T val)
261
// { T tmp = *dest ; *dest = tmp | val ; return tmp ; }
262
263
#include "impl/Kokkos_Atomic_Fetch_Or.hpp"
264
265
//----------------------------------------------------------------------------
266
// Atomic fetch and and
267
//
268
// template<class T>
269
// T atomic_fetch_and(volatile T* const dest, const T val)
270
// { T tmp = *dest ; *dest = tmp & val ; return tmp ; }
271
272
#include "impl/Kokkos_Atomic_Fetch_And.hpp"
273
#endif
/*Not _WIN32*/
274
275
//----------------------------------------------------------------------------
276
// Memory fence
277
//
278
// All loads and stores from this thread will be globally consistent before continuing
279
//
280
// void memory_fence() {...};
281
#include "impl/Kokkos_Memory_Fence.hpp"
282
283
//----------------------------------------------------------------------------
284
// Provide volatile_load and safe_load
285
//
286
// T volatile_load(T const volatile * const ptr);
287
//
288
// T const& safe_load(T const * const ptr);
289
// XEON PHI
290
// T safe_load(T const * const ptr
291
292
#include "impl/Kokkos_Volatile_Load.hpp"
293
294
#ifndef _WIN32
295
#include "impl/Kokkos_Atomic_Generic.hpp"
296
#endif
297
298
//----------------------------------------------------------------------------
299
// Provide atomic loads and stores with memory order semantics
300
301
#include "impl/Kokkos_Atomic_Load.hpp"
302
#include "impl/Kokkos_Atomic_Store.hpp"
303
304
305
//----------------------------------------------------------------------------
306
// This atomic-style macro should be an inlined function, not a macro
307
308
#if defined( KOKKOS_COMPILER_GNU ) && !defined(__PGIC__) && !defined(__CUDA_ARCH__)
309
310
#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr,0,0)
311
#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr,1,0)
312
313
#else
314
315
#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
316
#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
317
318
#endif
319
320
//----------------------------------------------------------------------------
321
322
#endif
/* KOKKOS_ATOMIC_HPP */
323
Generated by
1.8.5