Kokkos Core Kernels Package  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Groups Pages
Kokkos_Atomics_Desul_Wrapper.hpp
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 
17 #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18 #include <Kokkos_Macros.hpp>
19 static_assert(false,
20  "Including non-public Kokkos header files is not allowed.");
21 #endif
22 #ifndef KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
23 #define KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
24 #include <Kokkos_Macros.hpp>
25 #include <desul/atomics.hpp>
26 
27 #include <impl/Kokkos_Volatile_Load.hpp>
28 
29 // clang-format off
30 namespace Kokkos {
31 
32 // FIXME: These functions don't have any use/test in unit tests ...
33 // ==========================================================
34 inline const char* atomic_query_version() { return "KOKKOS_DESUL_ATOMICS"; }
35 
36 #if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \
37  !defined(__CUDA_ARCH__)
38 
39 #define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0)
40 #define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0)
41 
42 #else
43 
44 #define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
45 #define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
46 
47 #endif
48 // ============================================================
49 
50 #ifdef KOKKOS_ENABLE_ATOMICS_BYPASS
51 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeCaller()
52 #else
53 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice()
54 #endif
55 
56 template<class T> KOKKOS_INLINE_FUNCTION
57 T atomic_load(T* const dest) { return desul::atomic_load(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
58 
59 template<class T> KOKKOS_INLINE_FUNCTION
60 void atomic_store(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_store(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
61 
62 template<class T> KOKKOS_INLINE_FUNCTION
63 void atomic_assign(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { atomic_store(dest,val); }
64 
65 KOKKOS_INLINE_FUNCTION
66 void memory_fence() {
67  desul::atomic_thread_fence(desul::MemoryOrderSeqCst(), KOKKOS_DESUL_MEM_SCOPE);
68 }
69 
70 KOKKOS_INLINE_FUNCTION
71 void load_fence() { return desul::atomic_thread_fence(desul::MemoryOrderAcquire(), KOKKOS_DESUL_MEM_SCOPE); }
72 
73 KOKKOS_INLINE_FUNCTION
74 void store_fence() { return desul::atomic_thread_fence(desul::MemoryOrderRelease(), KOKKOS_DESUL_MEM_SCOPE); }
75 
76 // atomic_fetch_op
77 template<class T> KOKKOS_INLINE_FUNCTION
78 T atomic_fetch_add (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
79 
80 template<class T> KOKKOS_INLINE_FUNCTION
81 T atomic_fetch_sub (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
82 
83 template<class T> KOKKOS_INLINE_FUNCTION
84 T atomic_fetch_max (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
85 
86 template<class T> KOKKOS_INLINE_FUNCTION
87 T atomic_fetch_min (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
88 
89 template<class T> KOKKOS_INLINE_FUNCTION
90 T atomic_fetch_mul (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
91 
92 template<class T> KOKKOS_INLINE_FUNCTION
93 T atomic_fetch_div (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
94 
95 template<class T> KOKKOS_INLINE_FUNCTION
96 T atomic_fetch_mod (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_mod (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
97 
98 template<class T> KOKKOS_INLINE_FUNCTION
99 T atomic_fetch_and (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
100 
101 template<class T> KOKKOS_INLINE_FUNCTION
102 T atomic_fetch_or (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
103 
104 template<class T> KOKKOS_INLINE_FUNCTION
105 T atomic_fetch_xor (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_xor (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
106 
107 template<class T> KOKKOS_INLINE_FUNCTION
108 T atomic_fetch_nand(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_nand(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
109 
110 template<class T> KOKKOS_INLINE_FUNCTION
111 T atomic_fetch_lshift(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_lshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
112 
113 template<class T> KOKKOS_INLINE_FUNCTION
114 T atomic_fetch_rshift(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_rshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
115 
116 template<class T> KOKKOS_INLINE_FUNCTION
117 T atomic_fetch_inc(T* const dest) { return desul::atomic_fetch_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
118 
119 template<class T> KOKKOS_INLINE_FUNCTION
120 T atomic_fetch_dec(T* const dest) { return desul::atomic_fetch_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
121 
122 
123 // atomic_op_fetch
124 template<class T> KOKKOS_INLINE_FUNCTION
125 T atomic_add_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_add_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
126 
127 template<class T> KOKKOS_INLINE_FUNCTION
128 T atomic_sub_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_sub_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
129 
130 template<class T> KOKKOS_INLINE_FUNCTION
131 T atomic_max_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_max_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
132 
133 template<class T> KOKKOS_INLINE_FUNCTION
134 T atomic_min_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_min_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
135 
136 template<class T> KOKKOS_INLINE_FUNCTION
137 T atomic_mul_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mul_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
138 
139 template<class T> KOKKOS_INLINE_FUNCTION
140 T atomic_div_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_div_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
141 
142 template<class T> KOKKOS_INLINE_FUNCTION
143 T atomic_mod_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mod_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
144 
145 template<class T> KOKKOS_INLINE_FUNCTION
146 T atomic_and_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_and_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
147 
148 template<class T> KOKKOS_INLINE_FUNCTION
149 T atomic_or_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_or_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
150 
151 template<class T> KOKKOS_INLINE_FUNCTION
152 T atomic_xor_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_xor_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
153 
154 template<class T> KOKKOS_INLINE_FUNCTION
155 T atomic_nand_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_nand_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
156 
157 template<class T> KOKKOS_INLINE_FUNCTION
158 T atomic_lshift_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_lshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
159 
160 template<class T> KOKKOS_INLINE_FUNCTION
161 T atomic_rshift_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_rshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
162 
163 template<class T> KOKKOS_INLINE_FUNCTION
164 T atomic_inc_fetch(T* const dest) { return desul::atomic_inc_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
165 
166 template<class T> KOKKOS_INLINE_FUNCTION
167 T atomic_dec_fetch(T* const dest) { return desul::atomic_dec_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
168 
169 
170 // atomic_op
171 template<class T> KOKKOS_INLINE_FUNCTION
172 void atomic_add(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
173 
174 template<class T> KOKKOS_INLINE_FUNCTION
175 void atomic_sub(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
176 
177 template<class T> KOKKOS_INLINE_FUNCTION
178 void atomic_mul(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
179 
180 template<class T> KOKKOS_INLINE_FUNCTION
181 void atomic_div(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
182 
183 template<class T> KOKKOS_INLINE_FUNCTION
184 void atomic_min(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
185 
186 template<class T> KOKKOS_INLINE_FUNCTION
187 void atomic_max(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
188 
189 // FIXME: Desul doesn't have atomic_and yet so call fetch_and
190 template<class T> KOKKOS_INLINE_FUNCTION
191 void atomic_and(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
192 
193 // FIXME: Desul doesn't have atomic_or yet so call fetch_or
194 template<class T> KOKKOS_INLINE_FUNCTION
195 void atomic_or(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
196 
197 template<class T> KOKKOS_INLINE_FUNCTION
198 void atomic_inc(T* const dest) { return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
199 
200 template<class T> KOKKOS_INLINE_FUNCTION
201 void atomic_dec(T* const dest) { return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
202 
203 template<class T> KOKKOS_INLINE_FUNCTION
204 void atomic_increment(T* const dest) { return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
205 
206 template<class T> KOKKOS_INLINE_FUNCTION
207 void atomic_decrement(T* const dest) { return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
208 
209 // Exchange
210 
211 template<class T> KOKKOS_INLINE_FUNCTION
212 T atomic_exchange(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_exchange(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
213 
214 template<class T> KOKKOS_INLINE_FUNCTION
215 bool atomic_compare_exchange_strong(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> expected, desul::Impl::dont_deduce_this_parameter_t<const T> desired) {
216  T expected_ref = expected;
217  return desul::atomic_compare_exchange_strong(dest, expected_ref, desired,
218  desul::MemoryOrderRelaxed(), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
219 }
220 
221 template<class T> KOKKOS_INLINE_FUNCTION
222 T atomic_compare_exchange(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> compare, desul::Impl::dont_deduce_this_parameter_t<const T> desired) {
223  return desul::atomic_compare_exchange(dest, compare, desired,
224  desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
225 }
226 
227 namespace Impl {
228  template<class T, class MemOrderSuccess, class MemOrderFailure> KOKKOS_INLINE_FUNCTION
229  bool atomic_compare_exchange_strong(T* const dest, T& expected, const T desired, MemOrderSuccess succ, MemOrderFailure fail) {
230  return desul::atomic_compare_exchange_strong(dest, expected, desired, succ, fail, KOKKOS_DESUL_MEM_SCOPE);
231  }
232  template<class T, class MemoryOrder>
233  KOKKOS_INLINE_FUNCTION
234  T atomic_load(const T* const src, MemoryOrder order) {
235  return desul::atomic_load(src, order, KOKKOS_DESUL_MEM_SCOPE);
236  }
237  template<class T, class MemoryOrder>
238  KOKKOS_INLINE_FUNCTION
239  void atomic_store(T* const src, const T val, MemoryOrder order) {
240  return desul::atomic_store(src, val, order, KOKKOS_DESUL_MEM_SCOPE);
241  }
242 } // namespace Impl
243 
244 } // namespace Kokkos
245 
246 #undef KOKKOS_DESUL_MEM_SCOPE
247 
248 // clang-format on
249 #endif
Generated on Tue Nov 12 2024 09:17:34 for Kokkos Core Kernels Package by   doxygen 1.8.5