Kokkos Core Kernels Package
Version of the Day
Main Page
Related Pages
Namespaces
Classes
Files
File List
All
Classes
Namespaces
Files
Functions
Variables
Typedefs
Enumerations
Enumerator
Friends
Groups
Pages
core
src
Kokkos_Atomic.hpp
Go to the documentation of this file.
1
/*
2
//@HEADER
3
// ************************************************************************
4
//
5
// Kokkos v. 2.0
6
// Copyright (2014) Sandia Corporation
7
//
8
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9
// the U.S. Government retains certain rights in this software.
10
//
11
// Redistribution and use in source and binary forms, with or without
12
// modification, are permitted provided that the following conditions are
13
// met:
14
//
15
// 1. Redistributions of source code must retain the above copyright
16
// notice, this list of conditions and the following disclaimer.
17
//
18
// 2. Redistributions in binary form must reproduce the above copyright
19
// notice, this list of conditions and the following disclaimer in the
20
// documentation and/or other materials provided with the distribution.
21
//
22
// 3. Neither the name of the Corporation nor the names of the
23
// contributors may be used to endorse or promote products derived from
24
// this software without specific prior written permission.
25
//
26
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37
//
38
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
39
//
40
// ************************************************************************
41
//@HEADER
42
*/
43
66
67
#ifndef KOKKOS_ATOMIC_HPP
68
#define KOKKOS_ATOMIC_HPP
69
70
#include <Kokkos_Macros.hpp>
71
#include <Kokkos_HostSpace.hpp>
72
#include <impl/Kokkos_Traits.hpp>
73
74
//----------------------------------------------------------------------------
75
#if defined(_WIN32)
76
#define KOKKOS_ENABLE_WINDOWS_ATOMICS
77
#else
78
#if defined( KOKKOS_ENABLE_CUDA )
79
80
// Compiling NVIDIA device code, must use Cuda atomics:
81
82
#define KOKKOS_ENABLE_CUDA_ATOMICS
83
84
#elif defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_ROCM_GPU)
85
86
#define KOKKOS_ENABLE_ROCM_ATOMICS
87
88
#endif
89
90
#if ! defined( KOKKOS_ENABLE_GNU_ATOMICS ) && \
91
! defined( KOKKOS_ENABLE_INTEL_ATOMICS ) && \
92
! defined( KOKKOS_ENABLE_OPENMP_ATOMICS ) && \
93
! defined( KOKKOS_ENABLE_SERIAL_ATOMICS )
94
95
// Compiling for non-Cuda atomic implementation has not been pre-selected.
96
// Choose the best implementation for the detected compiler.
97
// Preference: GCC, INTEL, OMP31
98
99
#if defined( KOKKOS_INTERNAL_NOT_PARALLEL )
100
101
#define KOKKOS_ENABLE_SERIAL_ATOMICS
102
103
#elif defined( KOKKOS_COMPILER_GNU ) || \
104
defined( KOKKOS_COMPILER_CLANG ) || \
105
( defined ( KOKKOS_COMPILER_NVCC ) )
106
107
#define KOKKOS_ENABLE_GNU_ATOMICS
108
109
#elif defined( KOKKOS_COMPILER_INTEL ) || \
110
defined( KOKKOS_COMPILER_CRAYC )
111
112
#define KOKKOS_ENABLE_INTEL_ATOMICS
113
114
#elif defined( _OPENMP ) && ( 201107 <= _OPENMP )
115
116
#define KOKKOS_ENABLE_OPENMP_ATOMICS
117
118
#else
119
120
#error "KOKKOS_ATOMICS_USE : Unsupported compiler"
121
122
#endif
123
124
#endif
/* Not pre-selected atomic implementation */
125
#endif
126
127
#ifdef KOKKOS_ENABLE_CUDA
128
#include <Cuda/Kokkos_Cuda_Locks.hpp>
129
#endif
130
131
namespace
Kokkos {
132
template
<
typename
T>
133
KOKKOS_INLINE_FUNCTION
134
void
atomic_add(
volatile
T *
const
dest,
const
T src);
135
136
// Atomic increment
137
template
<
typename
T>
138
KOKKOS_INLINE_FUNCTION
139
void
atomic_increment(
volatile
T* a);
140
141
template
<
typename
T>
142
KOKKOS_INLINE_FUNCTION
143
void
atomic_decrement(
volatile
T* a);
144
}
145
146
namespace
Kokkos {
147
148
149
inline
150
const
char
* atomic_query_version()
151
{
152
#if defined( KOKKOS_ENABLE_CUDA_ATOMICS )
153
return
"KOKKOS_ENABLE_CUDA_ATOMICS"
;
154
#elif defined( KOKKOS_ENABLE_GNU_ATOMICS )
155
return
"KOKKOS_ENABLE_GNU_ATOMICS"
;
156
#elif defined( KOKKOS_ENABLE_INTEL_ATOMICS )
157
return
"KOKKOS_ENABLE_INTEL_ATOMICS"
;
158
#elif defined( KOKKOS_ENABLE_OPENMP_ATOMICS )
159
return
"KOKKOS_ENABLE_OPENMP_ATOMICS"
;
160
#elif defined( KOKKOS_ENABLE_WINDOWS_ATOMICS )
161
return
"KOKKOS_ENABLE_WINDOWS_ATOMICS"
;
162
#elif defined( KOKKOS_ENABLE_SERIAL_ATOMICS )
163
return
"KOKKOS_ENABLE_SERIAL_ATOMICS"
;
164
#else
165
#error "No valid response for atomic_query_version!"
166
#endif
167
}
168
169
}
// namespace Kokkos
170
171
#if defined( KOKKOS_ENABLE_ROCM )
172
namespace
Kokkos {
173
namespace
Impl {
174
extern
KOKKOS_INLINE_FUNCTION
175
bool
lock_address_rocm_space(
void
* ptr);
176
177
extern
KOKKOS_INLINE_FUNCTION
178
void
unlock_address_rocm_space(
void
* ptr);
179
}
180
}
181
#include <ROCm/Kokkos_ROCm_Atomic.hpp>
182
#endif
183
184
#ifdef _WIN32
185
#include "impl/Kokkos_Atomic_Windows.hpp"
186
#else
187
188
//----------------------------------------------------------------------------
189
// Atomic Assembly
190
//
191
// Implements CAS128-bit in assembly
192
193
#include "impl/Kokkos_Atomic_Assembly.hpp"
194
195
//----------------------------------------------------------------------------
196
// Atomic exchange
197
//
198
// template< typename T >
199
// T atomic_exchange( volatile T* const dest , const T val )
200
// { T tmp = *dest ; *dest = val ; return tmp ; }
201
202
#include "impl/Kokkos_Atomic_Exchange.hpp"
203
204
//----------------------------------------------------------------------------
205
// Atomic compare-and-exchange
206
//
207
// template<class T>
208
// bool atomic_compare_exchange_strong(volatile T* const dest, const T compare, const T val)
209
// { bool equal = compare == *dest ; if ( equal ) { *dest = val ; } return equal ; }
210
211
#include "impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp"
212
213
//----------------------------------------------------------------------------
214
// Atomic fetch and add
215
//
216
// template<class T>
217
// T atomic_fetch_add(volatile T* const dest, const T val)
218
// { T tmp = *dest ; *dest += val ; return tmp ; }
219
220
#include "impl/Kokkos_Atomic_Fetch_Add.hpp"
221
222
//----------------------------------------------------------------------------
223
// Atomic increment
224
//
225
// template<class T>
226
// T atomic_increment(volatile T* const dest)
227
// { dest++; }
228
229
#include "impl/Kokkos_Atomic_Increment.hpp"
230
231
//----------------------------------------------------------------------------
232
// Atomic Decrement
233
//
234
// template<class T>
235
// T atomic_decrement(volatile T* const dest)
236
// { dest--; }
237
238
#include "impl/Kokkos_Atomic_Decrement.hpp"
239
240
//----------------------------------------------------------------------------
241
// Atomic fetch and sub
242
//
243
// template<class T>
244
// T atomic_fetch_sub(volatile T* const dest, const T val)
245
// { T tmp = *dest ; *dest -= val ; return tmp ; }
246
247
#include "impl/Kokkos_Atomic_Fetch_Sub.hpp"
248
249
//----------------------------------------------------------------------------
250
// Atomic fetch and or
251
//
252
// template<class T>
253
// T atomic_fetch_or(volatile T* const dest, const T val)
254
// { T tmp = *dest ; *dest = tmp | val ; return tmp ; }
255
256
#include "impl/Kokkos_Atomic_Fetch_Or.hpp"
257
258
//----------------------------------------------------------------------------
259
// Atomic fetch and and
260
//
261
// template<class T>
262
// T atomic_fetch_and(volatile T* const dest, const T val)
263
// { T tmp = *dest ; *dest = tmp & val ; return tmp ; }
264
265
#include "impl/Kokkos_Atomic_Fetch_And.hpp"
266
#endif
/*Not _WIN32*/
267
268
//----------------------------------------------------------------------------
269
// Memory fence
270
//
271
// All loads and stores from this thread will be globally consistent before continuing
272
//
273
// void memory_fence() {...};
274
#include "impl/Kokkos_Memory_Fence.hpp"
275
276
//----------------------------------------------------------------------------
277
// Provide volatile_load and safe_load
278
//
279
// T volatile_load(T const volatile * const ptr);
280
//
281
// T const& safe_load(T const * const ptr);
282
// XEON PHI
283
// T safe_load(T const * const ptr
284
285
#include "impl/Kokkos_Volatile_Load.hpp"
286
287
#ifndef _WIN32
288
#include "impl/Kokkos_Atomic_Generic.hpp"
289
#endif
290
//----------------------------------------------------------------------------
291
// This atomic-style macro should be an inlined function, not a macro
292
293
#if defined( KOKKOS_COMPILER_GNU ) && !defined(__PGIC__) && !defined(__CUDA_ARCH__)
294
295
#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr,0,0)
296
#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr,1,0)
297
298
#else
299
300
#define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
301
#define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
302
303
#endif
304
305
//----------------------------------------------------------------------------
306
307
#endif
/* KOKKOS_ATOMIC_HPP */
308
Generated by
1.8.5