17 #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18 #include <Kokkos_Macros.hpp>
20 "Including non-public Kokkos header files is not allowed.");
22 #ifndef KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
23 #define KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
24 #include <Kokkos_Macros.hpp>
25 #include <desul/atomics.hpp>
27 #include <impl/Kokkos_Utilities.hpp>
28 #include <impl/Kokkos_Volatile_Load.hpp>
32 #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4
33 KOKKOS_DEPRECATED
inline const char* atomic_query_version() {
34 return "KOKKOS_DESUL_ATOMICS";
38 #if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \
39 !defined(__CUDA_ARCH__)
41 #define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0)
42 #define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0)
46 #define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
47 #define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
52 #ifdef KOKKOS_ENABLE_ATOMICS_BYPASS
53 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeCaller()
55 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice()
60 using not_deduced_atomic_t =
61 std::add_const_t<std::remove_volatile_t<type_identity_t<T>>>;
63 template <
class T,
class R>
64 using enable_if_atomic_t =
65 std::enable_if_t<!std::is_reference_v<T> && !std::is_const_v<T>,
66 std::remove_volatile_t<R>>;
72 KOKKOS_INLINE_FUNCTION
void memory_fence() { desul::atomic_thread_fence(desul::MemoryOrderSeqCst(), KOKKOS_DESUL_MEM_SCOPE); }
73 KOKKOS_INLINE_FUNCTION
void load_fence() { desul::atomic_thread_fence(desul::MemoryOrderAcquire(), KOKKOS_DESUL_MEM_SCOPE); }
74 KOKKOS_INLINE_FUNCTION
void store_fence() { desul::atomic_thread_fence(desul::MemoryOrderRelease(), KOKKOS_DESUL_MEM_SCOPE); }
77 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_load (T
const* ptr) {
return desul::atomic_load (
const_cast<std::remove_volatile_t<T>*
>(ptr), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
78 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_store(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_store(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
79 #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4
80 template<
class T> KOKKOS_DEPRECATED_WITH_COMMENT(
"Use atomic_store() instead!") KOKKOS_FUNCTION Impl::enable_if_atomic_t<T,
void> atomic_assign(T* ptr, Impl::not_deduced_atomic_t<T> val) { atomic_store(ptr, val); }
84 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_add(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_fetch_add(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
85 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_sub(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_fetch_sub(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
86 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_max(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_fetch_max(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
87 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_min(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_fetch_min(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
88 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_mul(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_fetch_mul(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
89 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_div(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_fetch_div(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
90 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_mod(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_fetch_mod(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
91 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_and(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_fetch_and(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
92 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_or (T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_fetch_or (
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
93 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_xor(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_fetch_xor(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
94 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_nand(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_fetch_nand(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
95 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_lshift(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_fetch_lshift(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
96 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_rshift(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_fetch_rshift(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
97 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_inc(T* ptr) {
return desul::atomic_fetch_inc(
const_cast<std::remove_volatile_t<T>*
>(ptr), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
98 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_fetch_dec(T* ptr) {
return desul::atomic_fetch_dec(
const_cast<std::remove_volatile_t<T>*
>(ptr), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
101 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_add_fetch(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_add_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
102 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_sub_fetch(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_sub_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
103 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_max_fetch(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_max_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
104 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_min_fetch(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_min_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
105 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_mul_fetch(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_mul_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
106 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_div_fetch(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_div_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
107 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_mod_fetch(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_mod_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
108 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_and_fetch(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_and_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
109 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_or_fetch (T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_or_fetch (
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
110 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_xor_fetch(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_xor_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
111 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_nand_fetch(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_nand_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
112 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_lshift_fetch(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_lshift_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
113 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_rshift_fetch(T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_rshift_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
114 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_inc_fetch(T* ptr) {
return desul::atomic_inc_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
115 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_dec_fetch(T* ptr) {
return desul::atomic_dec_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
118 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_add(T* ptr, Impl::not_deduced_atomic_t<T> val) { desul::atomic_add(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
119 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_sub(T* ptr, Impl::not_deduced_atomic_t<T> val) { desul::atomic_sub(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
120 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_max(T* ptr, Impl::not_deduced_atomic_t<T> val) { desul::atomic_max(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
121 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_min(T* ptr, Impl::not_deduced_atomic_t<T> val) { desul::atomic_min(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
122 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_mul(T* ptr, Impl::not_deduced_atomic_t<T> val) { desul::atomic_mul(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
123 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_div(T* ptr, Impl::not_deduced_atomic_t<T> val) { desul::atomic_div(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
124 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_mod(T* ptr, Impl::not_deduced_atomic_t<T> val) { (void)desul::atomic_fetch_mod(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
125 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_and(T* ptr, Impl::not_deduced_atomic_t<T> val) { (void)desul::atomic_fetch_and(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
126 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_or (T* ptr, Impl::not_deduced_atomic_t<T> val) { (void)desul::atomic_fetch_or (
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
127 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_xor(T* ptr, Impl::not_deduced_atomic_t<T> val) { (void)desul::atomic_fetch_xor(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
128 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_nand(T* ptr, Impl::not_deduced_atomic_t<T> val) { (void)desul::atomic_nand_fetch(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
129 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_lshift(T* ptr, Impl::not_deduced_atomic_t<T> val) { (void)desul::atomic_fetch_lshift(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
130 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_rshift(T* ptr, Impl::not_deduced_atomic_t<T> val) { (void)desul::atomic_fetch_rshift(
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
131 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_inc(T* ptr) { desul::atomic_inc(
const_cast<std::remove_volatile_t<T>*
>(ptr), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
132 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, void> atomic_dec(T* ptr) { desul::atomic_dec(
const_cast<std::remove_volatile_t<T>*
>(ptr), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
133 #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4
134 template<
class T> KOKKOS_DEPRECATED_WITH_COMMENT(
"Use atomic_inc() instead!") KOKKOS_FUNCTION Impl::enable_if_atomic_t<T,
void> atomic_increment(T* ptr) { atomic_inc(ptr); }
135 template<
class T> KOKKOS_DEPRECATED_WITH_COMMENT(
"Use atomic_dec() instead!") KOKKOS_FUNCTION Impl::enable_if_atomic_t<T,
void> atomic_decrement(T* ptr) { atomic_dec(ptr); }
139 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_exchange (T* ptr, Impl::not_deduced_atomic_t<T> val) {
return desul::atomic_exchange (
const_cast<std::remove_volatile_t<T>*
>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
140 template<
class T> KOKKOS_FUNCTION Impl::enable_if_atomic_t<T, T> atomic_compare_exchange(T* ptr, Impl::not_deduced_atomic_t<T> expected, Impl::not_deduced_atomic_t<T> desired) {
return desul::atomic_compare_exchange(
const_cast<std::remove_volatile_t<T>*
>(ptr), expected, desired, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
141 #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4
142 template<
class T> KOKKOS_DEPRECATED_WITH_COMMENT(
"Use atomic_compare_exchange() instead!") KOKKOS_FUNCTION Impl::enable_if_atomic_t<T,
bool> atomic_compare_exchange_strong(T* ptr, Impl::not_deduced_atomic_t<T> expected, Impl::not_deduced_atomic_t<T> desired) {
return expected == atomic_compare_exchange(ptr, expected, desired); }
148 namespace Kokkos::Impl {
150 template <
class T,
class MemOrderSuccess,
class MemOrderFailure>
151 KOKKOS_FUNCTION
bool atomic_compare_exchange_strong(T*
const dest, T& expected,
153 MemOrderSuccess succ,
154 MemOrderFailure fail) {
155 return desul::atomic_compare_exchange_strong(dest, expected, desired, succ,
156 fail, KOKKOS_DESUL_MEM_SCOPE);
159 template <
class T,
class MemoryOrder>
160 KOKKOS_FUNCTION T atomic_load(
const T*
const src, MemoryOrder order) {
161 return desul::atomic_load(src, order, KOKKOS_DESUL_MEM_SCOPE);
164 template <
class T,
class MemoryOrder>
165 KOKKOS_FUNCTION
void atomic_store(T*
const src,
const T val,
167 return desul::atomic_store(src, val, order, KOKKOS_DESUL_MEM_SCOPE);
172 #undef KOKKOS_DESUL_MEM_SCOPE