Kokkos Core Kernels Package  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Groups Pages
Kokkos_HIP_Space.hpp
1 /*
2 //@HEADER
3 // ************************************************************************
4 //
5 // Kokkos v. 3.0
6 // Copyright (2020) National Technology & Engineering
7 // Solutions of Sandia, LLC (NTESS).
8 //
9 // Under the terms of Contract DE-NA0003525 with NTESS,
10 // the U.S. Government retains certain rights in this software.
11 //
12 // Redistribution and use in source and binary forms, with or without
13 // modification, are permitted provided that the following conditions are
14 // met:
15 //
16 // 1. Redistributions of source code must retain the above copyright
17 // notice, this list of conditions and the following disclaimer.
18 //
19 // 2. Redistributions in binary form must reproduce the above copyright
20 // notice, this list of conditions and the following disclaimer in the
21 // documentation and/or other materials provided with the distribution.
22 //
23 // 3. Neither the name of the Corporation nor the names of the
24 // contributors may be used to endorse or promote products derived from
25 // this software without specific prior written permission.
26 //
27 // THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
28 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
31 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
32 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
33 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
34 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
35 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
36 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
37 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 //
39 // Questions? Contact Christian R. Trott (crtrott@sandia.gov)
40 //
41 // ************************************************************************
42 //@HEADER
43 */
44 
45 #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
46 #include <Kokkos_Macros.hpp>
47 #ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3
48 static_assert(false,
49  "Including non-public Kokkos header files is not allowed.");
50 #else
51 KOKKOS_IMPL_WARNING("Including non-public Kokkos header files is not allowed.")
52 #endif
53 #endif
54 #ifndef KOKKOS_HIPSPACE_HPP
55 #define KOKKOS_HIPSPACE_HPP
56 
57 #include <Kokkos_Core_fwd.hpp>
58 
59 #if defined(KOKKOS_ENABLE_HIP)
60 
61 #include <iosfwd>
62 #include <typeinfo>
63 #include <string>
64 #include <cstddef>
65 #include <iosfwd>
66 
67 #include <Kokkos_HostSpace.hpp>
68 #include <Kokkos_Layout.hpp>
69 #include <Kokkos_ScratchSpace.hpp>
70 #include <HIP/Kokkos_HIP_Error.hpp> // HIP_SAFE_CALL
71 
72 #include <impl/Kokkos_Profiling_Interface.hpp>
73 #include <impl/Kokkos_HostSharedPtr.hpp>
74 #include <impl/Kokkos_InitializationSettings.hpp>
75 
76 #include <hip/hip_runtime_api.h>
77 /*--------------------------------------------------------------------------*/
78 
79 namespace Kokkos {
80 namespace Impl {
81 
82 template <typename T>
83 struct is_hip_type_space : public std::false_type {};
84 
85 } // namespace Impl
86 
87 namespace Experimental {
90 class HIPSpace {
91  public:
93  using memory_space = HIPSpace;
94  using execution_space = Kokkos::Experimental::HIP;
95  using device_type = Kokkos::Device<execution_space, memory_space>;
96 
97  using size_type = unsigned int;
98 
99  /*--------------------------------*/
100 
101  HIPSpace();
102  HIPSpace(HIPSpace&& rhs) = default;
103  HIPSpace(const HIPSpace& rhs) = default;
104  HIPSpace& operator=(HIPSpace&& rhs) = default;
105  HIPSpace& operator=(const HIPSpace& rhs) = default;
106  ~HIPSpace() = default;
107 
109  void* allocate(const size_t arg_alloc_size) const;
110  void* allocate(const char* arg_label, const size_t arg_alloc_size,
111  const size_t arg_logical_size = 0) const;
112 
114  void deallocate(void* const arg_alloc_ptr, const size_t arg_alloc_size) const;
115  void deallocate(const char* arg_label, void* const arg_alloc_ptr,
116  const size_t arg_alloc_size,
117  const size_t arg_logical_size = 0) const;
118 
119  private:
120  template <class, class, class, class>
121  friend class LogicalMemorySpace;
122  void* impl_allocate(const char* arg_label, const size_t arg_alloc_size,
123  const size_t arg_logical_size = 0,
124  const Kokkos::Tools::SpaceHandle =
125  Kokkos::Tools::make_space_handle(name())) const;
126  void impl_deallocate(const char* arg_label, void* const arg_alloc_ptr,
127  const size_t arg_alloc_size,
128  const size_t arg_logical_size = 0,
129  const Kokkos::Tools::SpaceHandle =
130  Kokkos::Tools::make_space_handle(name())) const;
131 
132  public:
134  static constexpr const char* name() { return "HIP"; }
135 
136 #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3
137  /*--------------------------------*/
139  KOKKOS_DEPRECATED static void access_error();
140  KOKKOS_DEPRECATED static void access_error(const void* const);
141 #endif
142 
143  private:
144  int m_device;
145 
146  friend class Kokkos::Impl::SharedAllocationRecord<
147  Kokkos::Experimental::HIPSpace, void>;
148 };
149 
150 } // namespace Experimental
151 
152 template <>
153 struct Impl::is_hip_type_space<Experimental::HIPSpace> : public std::true_type {
154 };
155 
156 } // namespace Kokkos
157 
158 /*--------------------------------------------------------------------------*/
159 /*--------------------------------------------------------------------------*/
160 
161 namespace Kokkos {
162 namespace Experimental {
166 class HIPHostPinnedSpace {
167  public:
169 
170  using execution_space = HostSpace::execution_space;
171  using memory_space = HIPHostPinnedSpace;
172  using device_type = Kokkos::Device<execution_space, memory_space>;
173  using size_type = unsigned int;
174 
175  /*--------------------------------*/
176 
177  HIPHostPinnedSpace();
178  HIPHostPinnedSpace(HIPHostPinnedSpace&& rhs) = default;
179  HIPHostPinnedSpace(const HIPHostPinnedSpace& rhs) = default;
180  HIPHostPinnedSpace& operator=(HIPHostPinnedSpace&& rhs) = default;
181  HIPHostPinnedSpace& operator=(const HIPHostPinnedSpace& rhs) = default;
182  ~HIPHostPinnedSpace() = default;
183 
185  void* allocate(const size_t arg_alloc_size) const;
186  void* allocate(const char* arg_label, const size_t arg_alloc_size,
187  const size_t arg_logical_size = 0) const;
188 
190  void deallocate(void* const arg_alloc_ptr, const size_t arg_alloc_size) const;
191  void deallocate(const char* arg_label, void* const arg_alloc_ptr,
192  const size_t arg_alloc_size,
193  const size_t arg_logical_size = 0) const;
194 
195  private:
196  template <class, class, class, class>
197  friend class LogicalMemorySpace;
198  void* impl_allocate(const char* arg_label, const size_t arg_alloc_size,
199  const size_t arg_logical_size = 0,
200  const Kokkos::Tools::SpaceHandle =
201  Kokkos::Tools::make_space_handle(name())) const;
202  void impl_deallocate(const char* arg_label, void* const arg_alloc_ptr,
203  const size_t arg_alloc_size,
204  const size_t arg_logical_size = 0,
205  const Kokkos::Tools::SpaceHandle =
206  Kokkos::Tools::make_space_handle(name())) const;
207 
208  public:
210  static constexpr const char* name() { return "HIPHostPinned"; }
211 
212  /*--------------------------------*/
213 };
214 } // namespace Experimental
215 
216 template <>
217 struct Impl::is_hip_type_space<Experimental::HIPHostPinnedSpace>
218  : public std::true_type {};
219 
220 } // namespace Kokkos
221 
222 /*--------------------------------------------------------------------------*/
223 /*--------------------------------------------------------------------------*/
224 
225 namespace Kokkos {
226 namespace Experimental {
230 class HIPManagedSpace {
231  public:
233 
237  using memory_space = HIPManagedSpace;
239  using execution_space = Kokkos::Experimental::HIP;
240  using device_type = Kokkos::Device<execution_space, memory_space>;
241  using size_type = unsigned int;
242 
243  /*--------------------------------*/
244 
245  HIPManagedSpace();
246  HIPManagedSpace(HIPManagedSpace&& rhs) = default;
247  HIPManagedSpace(const HIPManagedSpace& rhs) = default;
248  HIPManagedSpace& operator=(HIPManagedSpace&& rhs) = default;
249  HIPManagedSpace& operator=(const HIPManagedSpace& rhs) = default;
250  ~HIPManagedSpace() = default;
251 
253  void* allocate(const size_t arg_alloc_size) const;
254  void* allocate(const char* arg_label, const size_t arg_alloc_size,
255  const size_t arg_logical_size = 0) const;
256 
258  void deallocate(void* const arg_alloc_ptr, const size_t arg_alloc_size) const;
259  void deallocate(const char* arg_label, void* const arg_alloc_ptr,
260  const size_t arg_alloc_size,
261  const size_t arg_logical_size = 0) const;
262 
263  private:
264  int m_device;
265  template <class, class, class, class>
266  friend class LogicalMemorySpace;
267  void* impl_allocate(const char* arg_label, const size_t arg_alloc_size,
268  const size_t arg_logical_size = 0,
269  const Kokkos::Tools::SpaceHandle =
270  Kokkos::Tools::make_space_handle(name())) const;
271  void impl_deallocate(const char* arg_label, void* const arg_alloc_ptr,
272  const size_t arg_alloc_size,
273  const size_t arg_logical_size = 0,
274  const Kokkos::Tools::SpaceHandle =
275  Kokkos::Tools::make_space_handle(name())) const;
276 
277  public:
279  static constexpr const char* name() { return "HIPManaged"; }
280 
281  /*--------------------------------*/
282 };
283 } // namespace Experimental
284 
285 template <>
286 struct Impl::is_hip_type_space<Experimental::HIPManagedSpace>
287  : public std::true_type {};
288 
289 } // namespace Kokkos
290 
291 /*--------------------------------------------------------------------------*/
292 /*--------------------------------------------------------------------------*/
293 
294 namespace Kokkos {
295 namespace Impl {
296 
297 static_assert(
298  Kokkos::Impl::MemorySpaceAccess<Kokkos::Experimental::HIPSpace,
299  Kokkos::Experimental::HIPSpace>::assignable,
300  "");
301 
302 //----------------------------------------
303 
304 template <>
305 struct MemorySpaceAccess<Kokkos::HostSpace, Kokkos::Experimental::HIPSpace> {
306  enum : bool { assignable = false };
307  enum : bool { accessible = false };
308  enum : bool { deepcopy = true };
309 };
310 
311 template <>
312 struct MemorySpaceAccess<Kokkos::HostSpace,
313  Kokkos::Experimental::HIPHostPinnedSpace> {
314  // HostSpace::execution_space == HIPHostPinnedSpace::execution_space
315  enum : bool { assignable = true };
316  enum : bool { accessible = true };
317  enum : bool { deepcopy = true };
318 };
319 
320 template <>
321 struct MemorySpaceAccess<Kokkos::HostSpace,
322  Kokkos::Experimental::HIPManagedSpace> {
323  // HostSpace::execution_space != HIPManagedSpace::execution_space
324  enum : bool { assignable = false };
325  enum : bool { accessible = true };
326  enum : bool { deepcopy = true };
327 };
328 
329 //----------------------------------------
330 
331 template <>
332 struct MemorySpaceAccess<Kokkos::Experimental::HIPSpace, Kokkos::HostSpace> {
333  enum : bool { assignable = false };
334  enum : bool { accessible = false };
335  enum : bool { deepcopy = true };
336 };
337 
338 template <>
339 struct MemorySpaceAccess<Kokkos::Experimental::HIPSpace,
340  Kokkos::Experimental::HIPHostPinnedSpace> {
341  // HIPSpace::execution_space != HIPHostPinnedSpace::execution_space
342  enum : bool { assignable = false };
343  enum : bool { accessible = true }; // HIPSpace::execution_space
344  enum : bool { deepcopy = true };
345 };
346 
347 template <>
348 struct MemorySpaceAccess<Kokkos::Experimental::HIPSpace,
349  Kokkos::Experimental::HIPManagedSpace> {
350  // HIPSpace::execution_space == HIPManagedSpace::execution_space
351  enum : bool { assignable = true };
352  enum : bool { accessible = true };
353  enum : bool { deepcopy = true };
354 };
355 
356 //----------------------------------------
357 // HIPHostPinnedSpace::execution_space == HostSpace::execution_space
358 // HIPHostPinnedSpace accessible to both HIP and Host
359 
360 template <>
361 struct MemorySpaceAccess<Kokkos::Experimental::HIPHostPinnedSpace,
363  enum : bool { assignable = false }; // Cannot access from HIP
364  enum : bool { accessible = true }; // HIPHostPinnedSpace::execution_space
365  enum : bool { deepcopy = true };
366 };
367 
368 template <>
369 struct MemorySpaceAccess<Kokkos::Experimental::HIPHostPinnedSpace,
370  Kokkos::Experimental::HIPSpace> {
371  enum : bool { assignable = false }; // Cannot access from Host
372  enum : bool { accessible = false };
373  enum : bool { deepcopy = true };
374 };
375 
376 template <>
377 struct MemorySpaceAccess<Kokkos::Experimental::HIPHostPinnedSpace,
378  Kokkos::Experimental::HIPManagedSpace> {
379  enum : bool { assignable = false }; // different exec_space
380  enum : bool { accessible = true };
381  enum : bool { deepcopy = true };
382 };
383 
384 //----------------------------------------
385 // HIPManagedSpace::execution_space != HostSpace::execution_space
386 // HIPManagedSpace accessible to both HIP and Host
387 
388 template <>
389 struct MemorySpaceAccess<Kokkos::Experimental::HIPManagedSpace,
391  enum : bool { assignable = false };
392  enum : bool { accessible = false }; // HIPHostPinnedSpace::execution_space
393  enum : bool { deepcopy = true };
394 };
395 
396 template <>
397 struct MemorySpaceAccess<Kokkos::Experimental::HIPManagedSpace,
398  Kokkos::Experimental::HIPSpace> {
399  enum : bool { assignable = false };
400  enum : bool { accessible = true };
401  enum : bool { deepcopy = true };
402 };
403 
404 template <>
405 struct MemorySpaceAccess<Kokkos::Experimental::HIPManagedSpace,
406  Kokkos::Experimental::HIPHostPinnedSpace> {
407  enum : bool { assignable = false }; // different exec_space
408  enum : bool { accessible = true };
409  enum : bool { deepcopy = true };
410 };
411 
412 }; // namespace Impl
413 //----------------------------------------
414 
415 } // namespace Kokkos
416 
417 /*--------------------------------------------------------------------------*/
418 /*--------------------------------------------------------------------------*/
419 
420 namespace Kokkos {
421 namespace Impl {
422 
423 void DeepCopyHIP(void* dst, const void* src, size_t n);
424 void DeepCopyAsyncHIP(const Kokkos::Experimental::HIP& instance, void* dst,
425  const void* src, size_t n);
426 void DeepCopyAsyncHIP(void* dst, const void* src, size_t n);
427 
428 template <class MemSpace>
429 struct DeepCopy<MemSpace, HostSpace, Kokkos::Experimental::HIP,
430  std::enable_if_t<is_hip_type_space<MemSpace>::value>> {
431  DeepCopy(void* dst, const void* src, size_t n) { DeepCopyHIP(dst, src, n); }
432  DeepCopy(const Kokkos::Experimental::HIP& instance, void* dst,
433  const void* src, size_t n) {
434  DeepCopyAsyncHIP(instance, dst, src, n);
435  }
436 };
437 
438 template <class MemSpace>
439 struct DeepCopy<HostSpace, MemSpace, Kokkos::Experimental::HIP,
440  std::enable_if_t<is_hip_type_space<MemSpace>::value>> {
441  DeepCopy(void* dst, const void* src, size_t n) { DeepCopyHIP(dst, src, n); }
442  DeepCopy(const Kokkos::Experimental::HIP& instance, void* dst,
443  const void* src, size_t n) {
444  DeepCopyAsyncHIP(instance, dst, src, n);
445  }
446 };
447 
448 template <class MemSpace1, class MemSpace2>
449 struct DeepCopy<MemSpace1, MemSpace2, Kokkos::Experimental::HIP,
450  std::enable_if_t<is_hip_type_space<MemSpace1>::value &&
451  is_hip_type_space<MemSpace2>::value>> {
452  DeepCopy(void* dst, const void* src, size_t n) { DeepCopyHIP(dst, src, n); }
453  DeepCopy(const Kokkos::Experimental::HIP& instance, void* dst,
454  const void* src, size_t n) {
455  DeepCopyAsyncHIP(instance, dst, src, n);
456  }
457 };
458 
459 template <class MemSpace1, class MemSpace2, class ExecutionSpace>
460 struct DeepCopy<
461  MemSpace1, MemSpace2, ExecutionSpace,
462  std::enable_if_t<
463  is_hip_type_space<MemSpace1>::value &&
464  is_hip_type_space<MemSpace2>::value &&
465  !std::is_same<ExecutionSpace, Kokkos::Experimental::HIP>::value>> {
466  inline DeepCopy(void* dst, const void* src, size_t n) {
467  DeepCopyHIP(dst, src, n);
468  }
469 
470  inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src,
471  size_t n) {
472  exec.fence(fence_string());
473  DeepCopyAsyncHIP(dst, src, n);
474  }
475 
476  private:
477  static const std::string& fence_string() {
478  static const std::string string =
479  std::string("Kokkos::Impl::DeepCopy<") + MemSpace1::name() + "Space, " +
480  MemSpace2::name() +
481  "Space, ExecutionSpace>::DeepCopy: fence before copy";
482  return string;
483  }
484 };
485 
486 template <class MemSpace, class ExecutionSpace>
487 struct DeepCopy<
488  MemSpace, HostSpace, ExecutionSpace,
489  std::enable_if_t<
490  is_hip_type_space<MemSpace>::value &&
491  !std::is_same<ExecutionSpace, Kokkos::Experimental::HIP>::value>> {
492  inline DeepCopy(void* dst, const void* src, size_t n) {
493  DeepCopyHIP(dst, src, n);
494  }
495 
496  inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src,
497  size_t n) {
498  exec.fence(fence_string());
499  DeepCopyAsyncHIP(dst, src, n);
500  }
501 
502  private:
503  static const std::string& fence_string() {
504  static const std::string string =
505  std::string("Kokkos::Impl::DeepCopy<") + MemSpace::name() +
506  "Space, HostSpace, ExecutionSpace>::DeepCopy: fence before copy";
507  return string;
508  }
509 };
510 
511 template <class MemSpace, class ExecutionSpace>
512 struct DeepCopy<
513  HostSpace, MemSpace, ExecutionSpace,
514  std::enable_if_t<
515  is_hip_type_space<MemSpace>::value &&
516  !std::is_same<ExecutionSpace, Kokkos::Experimental::HIP>::value>> {
517  inline DeepCopy(void* dst, const void* src, size_t n) {
518  DeepCopyHIP(dst, src, n);
519  }
520 
521  inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src,
522  size_t n) {
523  exec.fence(fence_string());
524  DeepCopyAsyncHIP(dst, src, n);
525  }
526 
527  private:
528  static const std::string& fence_string() {
529  static const std::string string =
530  std::string("Kokkos::Impl::DeepCopy<HostSpace, ") + MemSpace::name() +
531  "Space, ExecutionSpace>::DeepCopy: fence before copy";
532  return string;
533  }
534 };
535 } // namespace Impl
536 } // namespace Kokkos
537 
538 //----------------------------------------------------------------------------
539 //----------------------------------------------------------------------------
540 
541 namespace Kokkos {
542 namespace Impl {
543 
544 template <>
545 class SharedAllocationRecord<Kokkos::Experimental::HIPSpace, void>
546  : public HostInaccessibleSharedAllocationRecordCommon<
547  Kokkos::Experimental::HIPSpace> {
548  private:
549  friend class SharedAllocationRecordCommon<Kokkos::Experimental::HIPSpace>;
550  friend class HostInaccessibleSharedAllocationRecordCommon<
551  Kokkos::Experimental::HIPSpace>;
552  using base_t = HostInaccessibleSharedAllocationRecordCommon<
553  Kokkos::Experimental::HIPSpace>;
554  using RecordBase = SharedAllocationRecord<void, void>;
555 
556  SharedAllocationRecord(const SharedAllocationRecord&) = delete;
557  SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete;
558 
559 #ifdef KOKKOS_ENABLE_DEBUG
560  static RecordBase s_root_record;
561 #endif
562 
563  const Kokkos::Experimental::HIPSpace m_space;
564 
565  protected:
566  ~SharedAllocationRecord();
567 
568  template <typename ExecutionSpace>
569  SharedAllocationRecord(
570  const ExecutionSpace& /*exec*/,
571  const Kokkos::Experimental::HIPSpace& arg_space,
572  const std::string& arg_label, const size_t arg_alloc_size,
573  const RecordBase::function_type arg_dealloc = &base_t::deallocate)
574  : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size,
575  arg_dealloc) {}
576 
577  SharedAllocationRecord(
578  const Kokkos::Experimental::HIP& exec_space,
579  const Kokkos::Experimental::HIPSpace& arg_space,
580  const std::string& arg_label, const size_t arg_alloc_size,
581  const RecordBase::function_type arg_dealloc = &base_t::deallocate);
582 
583  SharedAllocationRecord(
584  const Kokkos::Experimental::HIPSpace& arg_space,
585  const std::string& arg_label, const size_t arg_alloc_size,
586  const RecordBase::function_type arg_dealloc = &base_t::deallocate);
587 };
588 
589 template <>
590 class SharedAllocationRecord<Kokkos::Experimental::HIPHostPinnedSpace, void>
591  : public SharedAllocationRecordCommon<
592  Kokkos::Experimental::HIPHostPinnedSpace> {
593  private:
594  friend class SharedAllocationRecordCommon<
595  Kokkos::Experimental::HIPHostPinnedSpace>;
596  using base_t =
597  SharedAllocationRecordCommon<Kokkos::Experimental::HIPHostPinnedSpace>;
598  using RecordBase = SharedAllocationRecord<void, void>;
599 
600  SharedAllocationRecord(const SharedAllocationRecord&) = delete;
601  SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete;
602 
603 #ifdef KOKKOS_ENABLE_DEBUG
604  static RecordBase s_root_record;
605 #endif
606 
607  const Kokkos::Experimental::HIPHostPinnedSpace m_space;
608 
609  protected:
610  ~SharedAllocationRecord();
611  SharedAllocationRecord() = default;
612 
613  template <typename ExecutionSpace>
614  SharedAllocationRecord(
615  const ExecutionSpace& /*exec_space*/,
616  const Kokkos::Experimental::HIPHostPinnedSpace& arg_space,
617  const std::string& arg_label, const size_t arg_alloc_size,
618  const RecordBase::function_type arg_dealloc = &base_t::deallocate)
619  : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size,
620  arg_dealloc) {}
621 
622  SharedAllocationRecord(
623  const Kokkos::Experimental::HIPHostPinnedSpace& arg_space,
624  const std::string& arg_label, const size_t arg_alloc_size,
625  const RecordBase::function_type arg_dealloc = &base_t::deallocate);
626 };
627 
628 template <>
629 class SharedAllocationRecord<Kokkos::Experimental::HIPManagedSpace, void>
630  : public SharedAllocationRecordCommon<
631  Kokkos::Experimental::HIPManagedSpace> {
632  private:
633  friend class SharedAllocationRecordCommon<
634  Kokkos::Experimental::HIPManagedSpace>;
635  using base_t =
636  SharedAllocationRecordCommon<Kokkos::Experimental::HIPManagedSpace>;
637  using RecordBase = SharedAllocationRecord<void, void>;
638 
639  SharedAllocationRecord(const SharedAllocationRecord&) = delete;
640  SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete;
641 
642 #ifdef KOKKOS_ENABLE_DEBUG
643  static RecordBase s_root_record;
644 #endif
645 
646  const Kokkos::Experimental::HIPManagedSpace m_space;
647 
648  protected:
649  ~SharedAllocationRecord();
650  SharedAllocationRecord() = default;
651 
652  template <typename ExecutionSpace>
653  SharedAllocationRecord(
654  const ExecutionSpace& /*exec_space*/,
655  const Kokkos::Experimental::HIPManagedSpace& arg_space,
656  const std::string& arg_label, const size_t arg_alloc_size,
657  const RecordBase::function_type arg_dealloc = &base_t::deallocate)
658  : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size,
659  arg_dealloc) {}
660 
661  SharedAllocationRecord(
662  const Kokkos::Experimental::HIPManagedSpace& arg_space,
663  const std::string& arg_label, const size_t arg_alloc_size,
664  const RecordBase::function_type arg_dealloc = &base_t::deallocate);
665 };
666 } // namespace Impl
667 } // namespace Kokkos
668 
669 //----------------------------------------------------------------------------
670 //----------------------------------------------------------------------------
671 
672 namespace Kokkos {
673 namespace Experimental {
674 namespace Impl {
675 class HIPInternal;
676 }
679 class HIP {
680  public:
681  //------------------------------------
683 
684 
686  using execution_space = HIP;
687  using memory_space = HIPSpace;
688  using device_type = Kokkos::Device<execution_space, memory_space>;
689 
690  using array_layout = LayoutLeft;
691  using size_type = HIPSpace::size_type;
692 
693  using scratch_memory_space = ScratchMemorySpace<HIP>;
694 
695  HIP();
696  HIP(hipStream_t stream, bool manage_stream = false);
697 
699  //------------------------------------
701 
702 
703  KOKKOS_INLINE_FUNCTION static int in_parallel() {
704 #if defined(__HIP_DEVICE_COMPILE__)
705  return true;
706 #else
707  return false;
708 #endif
709  }
710 
717  static void impl_static_fence(const std::string& name);
718 
719  void fence(const std::string& name =
720  "Kokkos::HIP::fence(): Unnamed Instance Fence") const;
721 
722  hipStream_t hip_stream() const;
723 
725  void print_configuration(std::ostream& os, bool verbose = false) const;
726 
728  static void impl_finalize();
729 
733  int hip_device() const;
734  static hipDeviceProp_t const& hip_device_prop();
735 
736  static void impl_initialize(InitializationSettings const&);
737 
738  static int impl_is_initialized();
739 
740  // static size_type device_arch();
741 
742  static size_type detect_device_count();
743 
744  static int concurrency();
745  static const char* name();
746 
747  inline Impl::HIPInternal* impl_internal_space_instance() const {
748  return m_space_instance.get();
749  }
750 
751  uint32_t impl_instance_id() const noexcept;
752 
753  private:
754  Kokkos::Impl::HostSharedPtr<Impl::HIPInternal> m_space_instance;
755 };
756 } // namespace Experimental
757 namespace Tools {
758 namespace Experimental {
759 template <>
760 struct DeviceTypeTraits<Kokkos::Experimental::HIP> {
761  static constexpr DeviceType id = DeviceType::HIP;
762  static int device_id(const Kokkos::Experimental::HIP& exec) {
763  return exec.hip_device();
764  }
765 };
766 } // namespace Experimental
767 } // namespace Tools
768 
769 namespace Impl {
770 template <class DT, class... DP>
771 struct ZeroMemset<Kokkos::Experimental::HIP, DT, DP...> {
772  ZeroMemset(const Kokkos::Experimental::HIP& exec_space,
773  const View<DT, DP...>& dst,
774  typename View<DT, DP...>::const_value_type&) {
775  KOKKOS_IMPL_HIP_SAFE_CALL(hipMemsetAsync(
776  dst.data(), 0,
777  dst.size() * sizeof(typename View<DT, DP...>::value_type),
778  exec_space.hip_stream()));
779  }
780 
781  ZeroMemset(const View<DT, DP...>& dst,
782  typename View<DT, DP...>::const_value_type&) {
783  KOKKOS_IMPL_HIP_SAFE_CALL(
784  hipMemset(dst.data(), 0,
785  dst.size() * sizeof(typename View<DT, DP...>::value_type)));
786  }
787 };
788 } // namespace Impl
789 } // namespace Kokkos
790 
791 namespace Kokkos {
792 namespace Impl {
793 
794 template <>
795 struct MemorySpaceAccess<Kokkos::Experimental::HIPSpace,
796  Kokkos::Experimental::HIP::scratch_memory_space> {
797  enum : bool { assignable = false };
798  enum : bool { accessible = true };
799  enum : bool { deepcopy = false };
800 };
801 
802 } // namespace Impl
803 } // namespace Kokkos
804 
805 #endif /* #if defined( KOKKOS_ENABLE_HIP ) */
806 #endif /* #define KOKKOS_HIPSPACE_HPP */
Memory management for host memory.
Declaration of various MemoryLayout options.
DefaultHostExecutionSpace execution_space
Default execution space for this memory space.
Access relationship between DstMemorySpace and SrcMemorySpace.