Kokkos Core Kernels Package  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Groups Pages
Kokkos_Core.hpp
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 
17 #ifndef KOKKOS_CORE_HPP
18 #define KOKKOS_CORE_HPP
19 #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
20 #define KOKKOS_IMPL_PUBLIC_INCLUDE
21 #define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_CORE
22 #endif
23 
24 //----------------------------------------------------------------------------
25 // In the case windows.h is included before Kokkos_Core.hpp there might be
26 // errors due to the potentially defined macros with name "min" and "max" in
27 // windows.h. These collide with the use of "min" and "max" in names inside
28 // Kokkos. The macros will be redefined at the end of Kokkos_Core.hpp
29 #if defined(min)
30 #pragma push_macro("min")
31 #undef min
32 #define KOKKOS_IMPL_PUSH_MACRO_MIN
33 #endif
34 #if defined(max)
35 #pragma push_macro("max")
36 #undef max
37 #define KOKKOS_IMPL_PUSH_MACRO_MAX
38 #endif
39 
40 //----------------------------------------------------------------------------
41 // Include the execution space header files for the enabled execution spaces.
42 
43 #include <Kokkos_Core_fwd.hpp>
44 
45 #include <KokkosCore_Config_DeclareBackend.hpp>
46 
47 #include <Kokkos_Half.hpp>
48 #include <Kokkos_AnonymousSpace.hpp>
49 #include <Kokkos_Pair.hpp>
50 #include <Kokkos_Clamp.hpp>
51 #include <Kokkos_MinMax.hpp>
52 #include <Kokkos_MathematicalConstants.hpp>
53 #include <Kokkos_MathematicalFunctions.hpp>
54 #include <Kokkos_MathematicalSpecialFunctions.hpp>
55 #include <Kokkos_NumericTraits.hpp>
56 #include <Kokkos_BitManipulation.hpp>
57 #include <Kokkos_Swap.hpp>
58 #include <Kokkos_MemoryPool.hpp>
59 #include <Kokkos_Array.hpp>
60 #include <Kokkos_View.hpp>
61 #include <Kokkos_Vectorization.hpp>
62 #include <Kokkos_Atomic.hpp>
63 #include <Kokkos_hwloc.hpp>
64 #include <Kokkos_Timer.hpp>
65 #include <Kokkos_Tuners.hpp>
66 #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4
67 #include <Kokkos_TaskScheduler.hpp>
68 #endif
69 #include <Kokkos_Complex.hpp>
70 #include <Kokkos_CopyViews.hpp>
71 #include <impl/Kokkos_TeamMDPolicy.hpp>
72 #include <impl/Kokkos_InitializationSettings.hpp>
73 #include <functional>
74 #include <iosfwd>
75 #include <memory>
76 #include <vector>
77 
78 //----------------------------------------------------------------------------
79 
80 namespace Kokkos {
81 
82 void initialize(int& argc, char* argv[]);
83 
84 void initialize(
85  InitializationSettings const& settings = InitializationSettings());
86 
87 namespace Impl {
88 
89 void pre_initialize(const InitializationSettings& settings);
90 
91 void post_initialize(const InitializationSettings& settings);
92 
93 void pre_finalize();
94 
95 void post_finalize();
96 
97 void declare_configuration_metadata(const std::string& category,
98  const std::string& key,
99  const std::string& value);
100 
101 } // namespace Impl
102 
103 [[nodiscard]] bool is_initialized() noexcept;
104 [[nodiscard]] bool is_finalized() noexcept;
105 
106 [[nodiscard]] int device_id() noexcept;
107 [[nodiscard]] int num_devices() noexcept;
108 [[nodiscard]] int num_threads() noexcept;
109 
110 bool show_warnings() noexcept;
111 bool tune_internals() noexcept;
112 
114 void finalize();
115 
136 void push_finalize_hook(std::function<void()> f);
137 
138 void fence(const std::string& name /*= "Kokkos::fence: Unnamed Global Fence"*/);
139 
141 void print_configuration(std::ostream& os, bool verbose = false);
142 
143 } // namespace Kokkos
144 
145 //----------------------------------------------------------------------------
146 //----------------------------------------------------------------------------
147 
148 namespace Kokkos {
149 
150 namespace Impl {
151 
152 static inline void check_init_final([[maybe_unused]] char const* func_name) {
153 // FIXME_THREADS: Checking for calls to kokkos_malloc, kokkos_realloc,
154 // kokkos_free before initialize or after finalize is currently disabled
155 // for the Threads backend. Refer issue #7944.
156 #if !defined(KOKKOS_ENABLE_THREADS)
157  if (is_finalized()) {
158  std::stringstream ss;
159  ss << "Kokkos ERROR: attempting to perform C-style memory management "
160  "via ";
161  ss << func_name << "() **after** Kokkos::finalize() was called\n";
162  Kokkos::abort(ss.str().c_str());
163  } else if (!is_initialized()) {
164  std::stringstream ss;
165  ss << "Kokkos ERROR: attempting to perform C-style memory management "
166  "via ";
167  ss << func_name << "() **before** Kokkos::initialize() was called\n";
168  Kokkos::abort(ss.str().c_str());
169  }
170 #endif
171 }
172 
173 } // namespace Impl
174 
175 /* Allocate memory from a memory space.
176  * The allocation is tracked in Kokkos memory tracking system, so
177  * leaked memory can be identified.
178  */
179 template <class Space = Kokkos::DefaultExecutionSpace::memory_space>
180 inline void* kokkos_malloc(const std::string& arg_alloc_label,
181  const size_t arg_alloc_size) {
182  Impl::check_init_final("kokkos_malloc");
183  using MemorySpace = typename Space::memory_space;
184  return Impl::SharedAllocationRecord<MemorySpace>::allocate_tracked(
185  MemorySpace(), arg_alloc_label, arg_alloc_size);
186 }
187 
188 template <class Space = Kokkos::DefaultExecutionSpace::memory_space>
189 inline void* kokkos_malloc(const size_t arg_alloc_size) {
190  Impl::check_init_final("kokkos_malloc");
191  using MemorySpace = typename Space::memory_space;
192  return Impl::SharedAllocationRecord<MemorySpace>::allocate_tracked(
193  MemorySpace(), "no-label", arg_alloc_size);
194 }
195 
196 template <class Space = Kokkos::DefaultExecutionSpace::memory_space>
197 inline void kokkos_free(void* arg_alloc) {
198  Impl::check_init_final("kokkos_free");
199  using MemorySpace = typename Space::memory_space;
200  return Impl::SharedAllocationRecord<MemorySpace>::deallocate_tracked(
201  arg_alloc);
202 }
203 
204 template <class Space = Kokkos::DefaultExecutionSpace::memory_space>
205 inline void* kokkos_realloc(void* arg_alloc, const size_t arg_alloc_size) {
206  Impl::check_init_final("kokkos_realloc");
207  using MemorySpace = typename Space::memory_space;
208  return Impl::SharedAllocationRecord<MemorySpace>::reallocate_tracked(
209  arg_alloc, arg_alloc_size);
210 }
211 
212 } // namespace Kokkos
213 
214 namespace Kokkos {
215 
224 namespace Impl {
225 
226 inline std::string scopeguard_correct_usage() {
227  return std::string(
228  "Do instead:\n"
229  " std::unique_ptr<Kokkos::ScopeGuard> guard =\n"
230  " !Kokkos::is_initialized() && !Kokkos::is_finalized()?\n"
231  " new ScopeGuard(argc,argv) : nullptr;\n");
232 }
233 
234 inline std::string scopeguard_create_while_initialized_warning() {
235  return std::string(
236  "Kokkos Error: Creating a ScopeGuard while Kokkos is initialized "
237  "is illegal.\n")
238  .append(scopeguard_correct_usage());
239 }
240 
241 inline std::string scopeguard_create_after_finalize_warning() {
242  return std::string(
243  "Kokkos Error: Creating a ScopeGuard after Kokkos was finalized "
244  "is illegal.\n")
245  .append(scopeguard_correct_usage());
246 }
247 
248 inline std::string scopeguard_destruct_after_finalize_warning() {
249  return std::string(
250  "Kokkos Error: Destroying a ScopeGuard after Kokkos was finalized "
251  "is illegal.\n")
252  .append(scopeguard_correct_usage());
253 }
254 
255 } // namespace Impl
256 
257 class KOKKOS_ATTRIBUTE_NODISCARD ScopeGuard {
258  public:
259  template <class... Args>
260 #if defined(__has_cpp_attribute) && __has_cpp_attribute(nodiscard) >= 201907
261  [[nodiscard]]
262 #endif
263  ScopeGuard(Args&&... args) {
264  if (is_initialized()) {
265  Kokkos::abort(
266  Impl::scopeguard_create_while_initialized_warning().c_str());
267  }
268  if (is_finalized()) {
269  Kokkos::abort(Impl::scopeguard_create_after_finalize_warning().c_str());
270  }
271  initialize(static_cast<Args&&>(args)...);
272  }
273 
274  ~ScopeGuard() {
275  if (is_finalized()) {
276  Kokkos::abort(Impl::scopeguard_destruct_after_finalize_warning().c_str());
277  }
278  finalize();
279  }
280 
281  ScopeGuard& operator=(const ScopeGuard&) = delete;
282  ScopeGuard& operator=(ScopeGuard&&) = delete;
283  ScopeGuard(const ScopeGuard&) = delete;
284  ScopeGuard(ScopeGuard&&) = delete;
285 };
286 
287 } // namespace Kokkos
288 
289 namespace Kokkos {
290 namespace Experimental {
291 namespace Impl {
292 // Customization point for backends. Default behavior is to return the passed in
293 // instance, ignoring weights
294 template <class ExecSpace, class T>
295 std::vector<ExecSpace> impl_partition_space(const ExecSpace& base_instance,
296  const std::vector<T>& weights) {
297  std::vector<ExecSpace> instances;
298  instances.reserve(weights.size());
299  std::generate_n(std::back_inserter(instances), weights.size(),
300  [&base_instance]() { return base_instance; });
301 
302  return instances;
303 }
304 } // namespace Impl
305 
306 // Partitioning an Execution Space
307 // Input:
308 // - Base execution space
309 // - integer arguments for relative weight, either input per weight or vector
310 // of weights
311 // Ouput:
312 // - Array (or vector) of execution spaces partitioned based on weights
313 template <class ExecSpace, class... Args>
314 std::array<ExecSpace, sizeof...(Args)> partition_space(
315  ExecSpace const& base_instance, Args... args) {
316  static_assert(is_execution_space<ExecSpace>::value,
317  "Kokkos Error: partition_space expects an Execution Space as "
318  "first argument");
319  static_assert(
320  (... && std::is_arithmetic_v<Args>),
321  "Kokkos Error: partitioning arguments must be integers or floats");
322 
323  // Get vector of instances from backend specific impl
324  std::vector<std::common_type_t<Args...>> weights = {args...};
325  auto instances_vec = Impl::impl_partition_space(base_instance, weights);
326 
327  // Convert to std::array and return
328  std::array<ExecSpace, sizeof...(Args)> instances;
329  std::copy(instances_vec.begin(), instances_vec.end(), instances.begin());
330  return instances;
331 }
332 
333 template <class ExecSpace, class T>
334 std::vector<ExecSpace> partition_space(ExecSpace const& base_instance,
335  std::vector<T> const& weights) {
336  static_assert(is_execution_space<ExecSpace>::value,
337  "Kokkos Error: partition_space expects an Execution Space as "
338  "first argument");
339  static_assert(
340  std::is_arithmetic_v<T>,
341  "Kokkos Error: partitioning arguments must be integers or floats");
342 
343  // Return vector of instances from backend specific impl
344  return Impl::impl_partition_space(base_instance, weights);
345 }
346 } // namespace Experimental
347 } // namespace Kokkos
348 
349 #include <Kokkos_Crs.hpp>
350 #include <Kokkos_WorkGraphPolicy.hpp>
351 // Including this in Kokkos_Parallel_Reduce.hpp led to a circular dependency
352 // because Kokkos::Sum is used in Kokkos_Combined_Reducer.hpp and the default.
353 // The real answer is to finally break up Kokkos_Parallel_Reduce.hpp into
354 // smaller parts...
355 #include <impl/Kokkos_Combined_Reducer.hpp>
356 // Yet another workaround to deal with circular dependency issues because the
357 // implementation of the RAII wrapper is using Kokkos::single.
358 #include <Kokkos_AcquireUniqueTokenImpl.hpp>
359 
360 //----------------------------------------------------------------------------
361 // Redefinition of the macros min and max if we pushed them at entry of
362 // Kokkos_Core.hpp
363 #if defined(KOKKOS_IMPL_PUSH_MACRO_MIN)
364 #pragma pop_macro("min")
365 #undef KOKKOS_IMPL_PUSH_MACRO_MIN
366 #endif
367 #if defined(KOKKOS_IMPL_PUSH_MACRO_MAX)
368 #pragma pop_macro("max")
369 #undef KOKKOS_IMPL_PUSH_MACRO_MAX
370 #endif
371 
372 //----------------------------------------------------------------------------
373 //----------------------------------------------------------------------------
374 
375 #ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_CORE
376 #undef KOKKOS_IMPL_PUBLIC_INCLUDE
377 #undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_CORE
378 #endif
379 #endif
Declaration and definition of Kokkos::Vectorization interface.
Declaration and definition of Kokkos::pair.
Atomic functions.