Kokkos Core Kernels Package  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Groups Pages
Kokkos_Atomics_Desul_Wrapper.hpp
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 
17 #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18 #include <Kokkos_Macros.hpp>
19 static_assert(false,
20  "Including non-public Kokkos header files is not allowed.");
21 #endif
22 #ifndef KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
23 #define KOKKOS_DESUL_ATOMICS_WRAPPER_HPP_
24 #include <Kokkos_Macros.hpp>
25 
26 #include <Kokkos_Atomics_Desul_Config.hpp>
27 #include <desul/atomics.hpp>
28 
29 #include <impl/Kokkos_Volatile_Load.hpp>
30 
31 // clang-format off
32 namespace Kokkos {
33 
34 // FIXME: These functions don't have any use/test in unit tests ...
35 // ==========================================================
36 inline const char* atomic_query_version() { return "KOKKOS_DESUL_ATOMICS"; }
37 
38 #if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \
39  !defined(__CUDA_ARCH__)
40 
41 #define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) __builtin_prefetch(addr, 0, 0)
42 #define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) __builtin_prefetch(addr, 1, 0)
43 
44 #else
45 
46 #define KOKKOS_NONTEMPORAL_PREFETCH_LOAD(addr) ((void)0)
47 #define KOKKOS_NONTEMPORAL_PREFETCH_STORE(addr) ((void)0)
48 
49 #endif
50 // ============================================================
51 
52 #ifdef KOKKOS_ENABLE_ATOMICS_BYPASS
53 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeCaller()
54 #else
55 #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice()
56 #endif
57 
58 template<class T> KOKKOS_INLINE_FUNCTION
59 T atomic_load(T* const dest) { return desul::atomic_load(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
60 
61 template<class T> KOKKOS_INLINE_FUNCTION
62 void atomic_store(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_store(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
63 
64 template<class T> KOKKOS_INLINE_FUNCTION
65 void atomic_assign(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { atomic_store(dest,val); }
66 
67 KOKKOS_INLINE_FUNCTION
68 void memory_fence() {
69  desul::atomic_thread_fence(desul::MemoryOrderSeqCst(), KOKKOS_DESUL_MEM_SCOPE);
70 }
71 
72 KOKKOS_INLINE_FUNCTION
73 void load_fence() { return desul::atomic_thread_fence(desul::MemoryOrderAcquire(), KOKKOS_DESUL_MEM_SCOPE); }
74 
75 KOKKOS_INLINE_FUNCTION
76 void store_fence() { return desul::atomic_thread_fence(desul::MemoryOrderRelease(), KOKKOS_DESUL_MEM_SCOPE); }
77 
78 // atomic_fetch_op
79 template<class T> KOKKOS_INLINE_FUNCTION
80 T atomic_fetch_add (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
81 
82 template<class T> KOKKOS_INLINE_FUNCTION
83 T atomic_fetch_sub (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
84 
85 template<class T> KOKKOS_INLINE_FUNCTION
86 T atomic_fetch_max (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
87 
88 template<class T> KOKKOS_INLINE_FUNCTION
89 T atomic_fetch_min (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
90 
91 template<class T> KOKKOS_INLINE_FUNCTION
92 T atomic_fetch_mul (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
93 
94 template<class T> KOKKOS_INLINE_FUNCTION
95 T atomic_fetch_div (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
96 
97 template<class T> KOKKOS_INLINE_FUNCTION
98 T atomic_fetch_mod (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_mod (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
99 
100 template<class T> KOKKOS_INLINE_FUNCTION
101 T atomic_fetch_and (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
102 
103 template<class T> KOKKOS_INLINE_FUNCTION
104 T atomic_fetch_or (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
105 
106 template<class T> KOKKOS_INLINE_FUNCTION
107 T atomic_fetch_xor (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_xor (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
108 
109 template<class T> KOKKOS_INLINE_FUNCTION
110 T atomic_fetch_nand(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_nand(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
111 
112 template<class T> KOKKOS_INLINE_FUNCTION
113 T atomic_fetch_lshift(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_lshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
114 
115 template<class T> KOKKOS_INLINE_FUNCTION
116 T atomic_fetch_rshift(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_fetch_rshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
117 
118 template<class T> KOKKOS_INLINE_FUNCTION
119 T atomic_fetch_inc(T* const dest) { return desul::atomic_fetch_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
120 
121 template<class T> KOKKOS_INLINE_FUNCTION
122 T atomic_fetch_dec(T* const dest) { return desul::atomic_fetch_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
123 
124 
125 // atomic_op_fetch
126 template<class T> KOKKOS_INLINE_FUNCTION
127 T atomic_add_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_add_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
128 
129 template<class T> KOKKOS_INLINE_FUNCTION
130 T atomic_sub_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_sub_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
131 
132 template<class T> KOKKOS_INLINE_FUNCTION
133 T atomic_max_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_max_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
134 
135 template<class T> KOKKOS_INLINE_FUNCTION
136 T atomic_min_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_min_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
137 
138 template<class T> KOKKOS_INLINE_FUNCTION
139 T atomic_mul_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mul_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
140 
141 template<class T> KOKKOS_INLINE_FUNCTION
142 T atomic_div_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_div_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
143 
144 template<class T> KOKKOS_INLINE_FUNCTION
145 T atomic_mod_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mod_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
146 
147 template<class T> KOKKOS_INLINE_FUNCTION
148 T atomic_and_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_and_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
149 
150 template<class T> KOKKOS_INLINE_FUNCTION
151 T atomic_or_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_or_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
152 
153 template<class T> KOKKOS_INLINE_FUNCTION
154 T atomic_xor_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_xor_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
155 
156 template<class T> KOKKOS_INLINE_FUNCTION
157 T atomic_nand_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_nand_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
158 
159 template<class T> KOKKOS_INLINE_FUNCTION
160 T atomic_lshift_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_lshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
161 
162 template<class T> KOKKOS_INLINE_FUNCTION
163 T atomic_rshift_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_rshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
164 
165 template<class T> KOKKOS_INLINE_FUNCTION
166 T atomic_inc_fetch(T* const dest) { return desul::atomic_inc_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
167 
168 template<class T> KOKKOS_INLINE_FUNCTION
169 T atomic_dec_fetch(T* const dest) { return desul::atomic_dec_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
170 
171 
172 // atomic_op
173 template<class T> KOKKOS_INLINE_FUNCTION
174 void atomic_add(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
175 
176 template<class T> KOKKOS_INLINE_FUNCTION
177 void atomic_sub(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
178 
179 template<class T> KOKKOS_INLINE_FUNCTION
180 void atomic_mul(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
181 
182 template<class T> KOKKOS_INLINE_FUNCTION
183 void atomic_div(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
184 
185 template<class T> KOKKOS_INLINE_FUNCTION
186 void atomic_min(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
187 
188 template<class T> KOKKOS_INLINE_FUNCTION
189 void atomic_max(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
190 
191 // FIXME: Desul doesn't have atomic_and yet so call fetch_and
192 template<class T> KOKKOS_INLINE_FUNCTION
193 void atomic_and(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
194 
195 // FIXME: Desul doesn't have atomic_or yet so call fetch_or
196 template<class T> KOKKOS_INLINE_FUNCTION
197 void atomic_or(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { (void) desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
198 
199 template<class T> KOKKOS_INLINE_FUNCTION
200 void atomic_inc(T* const dest) { return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
201 
202 template<class T> KOKKOS_INLINE_FUNCTION
203 void atomic_dec(T* const dest) { return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
204 
205 template<class T> KOKKOS_INLINE_FUNCTION
206 void atomic_increment(T* const dest) { return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
207 
208 template<class T> KOKKOS_INLINE_FUNCTION
209 void atomic_decrement(T* const dest) { return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
210 
211 // Exchange
212 
213 template<class T> KOKKOS_INLINE_FUNCTION
214 T atomic_exchange(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> val) { return desul::atomic_exchange(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); }
215 
216 template<class T> KOKKOS_INLINE_FUNCTION
217 bool atomic_compare_exchange_strong(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> expected, desul::Impl::dont_deduce_this_parameter_t<const T> desired) {
218  T expected_ref = expected;
219  return desul::atomic_compare_exchange_strong(dest, expected_ref, desired,
220  desul::MemoryOrderRelaxed(), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
221 }
222 
223 template<class T> KOKKOS_INLINE_FUNCTION
224 T atomic_compare_exchange(T* const dest, desul::Impl::dont_deduce_this_parameter_t<const T> compare, desul::Impl::dont_deduce_this_parameter_t<const T> desired) {
225  return desul::atomic_compare_exchange(dest, compare, desired,
226  desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE);
227 }
228 
229 namespace Impl {
230  template<class T, class MemOrderSuccess, class MemOrderFailure> KOKKOS_INLINE_FUNCTION
231  bool atomic_compare_exchange_strong(T* const dest, T& expected, const T desired, MemOrderSuccess succ, MemOrderFailure fail) {
232  return desul::atomic_compare_exchange_strong(dest, expected, desired, succ, fail, KOKKOS_DESUL_MEM_SCOPE);
233  }
234  template<class T, class MemoryOrder>
235  KOKKOS_INLINE_FUNCTION
236  T atomic_load(const T* const src, MemoryOrder order) {
237  return desul::atomic_load(src, order, KOKKOS_DESUL_MEM_SCOPE);
238  }
239  template<class T, class MemoryOrder>
240  KOKKOS_INLINE_FUNCTION
241  void atomic_store(T* const src, const T val, MemoryOrder order) {
242  return desul::atomic_store(src, val, order, KOKKOS_DESUL_MEM_SCOPE);
243  }
244 } // namespace Impl
245 
246 } // namespace Kokkos
247 
248 #undef KOKKOS_DESUL_MEM_SCOPE
249 
250 // clang-format on
251 #endif
Generated on Thu Apr 18 2024 09:19:21 for Kokkos Core Kernels Package by   doxygen 1.8.5