45 #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
46 #include <Kokkos_Macros.hpp>
47 #ifndef KOKKOS_ENABLE_DEPRECATED_CODE_3
49 "Including non-public Kokkos header files is not allowed.");
51 KOKKOS_IMPL_WARNING(
"Including non-public Kokkos header files is not allowed.")
54 #ifndef KOKKOS_HIPSPACE_HPP
55 #define KOKKOS_HIPSPACE_HPP
57 #include <Kokkos_Core_fwd.hpp>
59 #if defined(KOKKOS_ENABLE_HIP)
67 #include <Kokkos_HostSpace.hpp>
69 #include <Kokkos_ScratchSpace.hpp>
70 #include <HIP/Kokkos_HIP_Error.hpp>
72 #include <impl/Kokkos_Profiling_Interface.hpp>
73 #include <impl/Kokkos_HostSharedPtr.hpp>
74 #include <impl/Kokkos_InitializationSettings.hpp>
76 #include <hip/hip_runtime_api.h>
83 struct is_hip_type_space :
public std::false_type {};
87 namespace Experimental {
93 using memory_space = HIPSpace;
94 using execution_space = Kokkos::Experimental::HIP;
95 using device_type = Kokkos::Device<execution_space, memory_space>;
97 using size_type =
unsigned int;
102 HIPSpace(HIPSpace&& rhs) =
default;
103 HIPSpace(
const HIPSpace& rhs) =
default;
104 HIPSpace& operator=(HIPSpace&& rhs) =
default;
105 HIPSpace& operator=(
const HIPSpace& rhs) =
default;
106 ~HIPSpace() =
default;
109 void* allocate(
const size_t arg_alloc_size)
const;
110 void* allocate(
const char* arg_label,
const size_t arg_alloc_size,
111 const size_t arg_logical_size = 0)
const;
114 void deallocate(
void*
const arg_alloc_ptr,
const size_t arg_alloc_size)
const;
115 void deallocate(
const char* arg_label,
void*
const arg_alloc_ptr,
116 const size_t arg_alloc_size,
117 const size_t arg_logical_size = 0)
const;
120 template <
class,
class,
class,
class>
121 friend class LogicalMemorySpace;
122 void* impl_allocate(
const char* arg_label,
const size_t arg_alloc_size,
123 const size_t arg_logical_size = 0,
124 const Kokkos::Tools::SpaceHandle =
125 Kokkos::Tools::make_space_handle(name()))
const;
126 void impl_deallocate(
const char* arg_label,
void*
const arg_alloc_ptr,
127 const size_t arg_alloc_size,
128 const size_t arg_logical_size = 0,
129 const Kokkos::Tools::SpaceHandle =
130 Kokkos::Tools::make_space_handle(name()))
const;
134 static constexpr
const char* name() {
return "HIP"; }
136 #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_3
139 KOKKOS_DEPRECATED
static void access_error();
140 KOKKOS_DEPRECATED
static void access_error(
const void*
const);
146 friend class Kokkos::Impl::SharedAllocationRecord<
147 Kokkos::Experimental::HIPSpace, void>;
153 struct Impl::is_hip_type_space<Experimental::HIPSpace> :
public std::true_type {
162 namespace Experimental {
166 class HIPHostPinnedSpace {
171 using memory_space = HIPHostPinnedSpace;
172 using device_type = Kokkos::Device<execution_space, memory_space>;
173 using size_type =
unsigned int;
177 HIPHostPinnedSpace();
178 HIPHostPinnedSpace(HIPHostPinnedSpace&& rhs) =
default;
179 HIPHostPinnedSpace(
const HIPHostPinnedSpace& rhs) =
default;
180 HIPHostPinnedSpace& operator=(HIPHostPinnedSpace&& rhs) =
default;
181 HIPHostPinnedSpace& operator=(
const HIPHostPinnedSpace& rhs) =
default;
182 ~HIPHostPinnedSpace() =
default;
185 void* allocate(
const size_t arg_alloc_size)
const;
186 void* allocate(
const char* arg_label,
const size_t arg_alloc_size,
187 const size_t arg_logical_size = 0)
const;
190 void deallocate(
void*
const arg_alloc_ptr,
const size_t arg_alloc_size)
const;
191 void deallocate(
const char* arg_label,
void*
const arg_alloc_ptr,
192 const size_t arg_alloc_size,
193 const size_t arg_logical_size = 0)
const;
196 template <
class,
class,
class,
class>
197 friend class LogicalMemorySpace;
198 void* impl_allocate(
const char* arg_label,
const size_t arg_alloc_size,
199 const size_t arg_logical_size = 0,
200 const Kokkos::Tools::SpaceHandle =
201 Kokkos::Tools::make_space_handle(name()))
const;
202 void impl_deallocate(
const char* arg_label,
void*
const arg_alloc_ptr,
203 const size_t arg_alloc_size,
204 const size_t arg_logical_size = 0,
205 const Kokkos::Tools::SpaceHandle =
206 Kokkos::Tools::make_space_handle(name()))
const;
210 static constexpr
const char* name() {
return "HIPHostPinned"; }
217 struct Impl::is_hip_type_space<Experimental::HIPHostPinnedSpace>
218 :
public std::true_type {};
226 namespace Experimental {
230 class HIPManagedSpace {
237 using memory_space = HIPManagedSpace;
239 using execution_space = Kokkos::Experimental::HIP;
240 using device_type = Kokkos::Device<execution_space, memory_space>;
241 using size_type =
unsigned int;
246 HIPManagedSpace(HIPManagedSpace&& rhs) =
default;
247 HIPManagedSpace(
const HIPManagedSpace& rhs) =
default;
248 HIPManagedSpace& operator=(HIPManagedSpace&& rhs) =
default;
249 HIPManagedSpace& operator=(
const HIPManagedSpace& rhs) =
default;
250 ~HIPManagedSpace() =
default;
253 void* allocate(
const size_t arg_alloc_size)
const;
254 void* allocate(
const char* arg_label,
const size_t arg_alloc_size,
255 const size_t arg_logical_size = 0)
const;
258 void deallocate(
void*
const arg_alloc_ptr,
const size_t arg_alloc_size)
const;
259 void deallocate(
const char* arg_label,
void*
const arg_alloc_ptr,
260 const size_t arg_alloc_size,
261 const size_t arg_logical_size = 0)
const;
265 template <
class,
class,
class,
class>
266 friend class LogicalMemorySpace;
267 void* impl_allocate(
const char* arg_label,
const size_t arg_alloc_size,
268 const size_t arg_logical_size = 0,
269 const Kokkos::Tools::SpaceHandle =
270 Kokkos::Tools::make_space_handle(name()))
const;
271 void impl_deallocate(
const char* arg_label,
void*
const arg_alloc_ptr,
272 const size_t arg_alloc_size,
273 const size_t arg_logical_size = 0,
274 const Kokkos::Tools::SpaceHandle =
275 Kokkos::Tools::make_space_handle(name()))
const;
279 static constexpr
const char* name() {
return "HIPManaged"; }
286 struct Impl::is_hip_type_space<Experimental::HIPManagedSpace>
287 :
public std::true_type {};
299 Kokkos::Experimental::HIPSpace>::assignable,
305 struct MemorySpaceAccess<Kokkos::HostSpace, Kokkos::Experimental::HIPSpace> {
306 enum :
bool { assignable =
false };
307 enum :
bool { accessible =
false };
308 enum :
bool { deepcopy =
true };
312 struct MemorySpaceAccess<Kokkos::HostSpace,
313 Kokkos::Experimental::HIPHostPinnedSpace> {
315 enum :
bool { assignable =
true };
316 enum :
bool { accessible =
true };
317 enum :
bool { deepcopy =
true };
321 struct MemorySpaceAccess<Kokkos::HostSpace,
322 Kokkos::Experimental::HIPManagedSpace> {
324 enum :
bool { assignable =
false };
325 enum :
bool { accessible =
true };
326 enum :
bool { deepcopy =
true };
333 enum :
bool { assignable =
false };
334 enum :
bool { accessible =
false };
335 enum :
bool { deepcopy =
true };
339 struct MemorySpaceAccess<Kokkos::Experimental::HIPSpace,
340 Kokkos::Experimental::HIPHostPinnedSpace> {
342 enum :
bool { assignable =
false };
343 enum :
bool { accessible =
true };
344 enum :
bool { deepcopy =
true };
348 struct MemorySpaceAccess<Kokkos::Experimental::HIPSpace,
349 Kokkos::Experimental::HIPManagedSpace> {
351 enum :
bool { assignable =
true };
352 enum :
bool { accessible =
true };
353 enum :
bool { deepcopy =
true };
361 struct MemorySpaceAccess<Kokkos::Experimental::HIPHostPinnedSpace,
363 enum :
bool { assignable =
false };
364 enum :
bool { accessible =
true };
365 enum :
bool { deepcopy =
true };
369 struct MemorySpaceAccess<Kokkos::Experimental::HIPHostPinnedSpace,
370 Kokkos::Experimental::HIPSpace> {
371 enum :
bool { assignable =
false };
372 enum :
bool { accessible =
false };
373 enum :
bool { deepcopy =
true };
377 struct MemorySpaceAccess<Kokkos::Experimental::HIPHostPinnedSpace,
378 Kokkos::Experimental::HIPManagedSpace> {
379 enum :
bool { assignable =
false };
380 enum :
bool { accessible =
true };
381 enum :
bool { deepcopy =
true };
389 struct MemorySpaceAccess<Kokkos::Experimental::HIPManagedSpace,
391 enum :
bool { assignable =
false };
392 enum :
bool { accessible =
false };
393 enum :
bool { deepcopy =
true };
397 struct MemorySpaceAccess<Kokkos::Experimental::HIPManagedSpace,
398 Kokkos::Experimental::HIPSpace> {
399 enum :
bool { assignable =
false };
400 enum :
bool { accessible =
true };
401 enum :
bool { deepcopy =
true };
405 struct MemorySpaceAccess<Kokkos::Experimental::HIPManagedSpace,
406 Kokkos::Experimental::HIPHostPinnedSpace> {
407 enum :
bool { assignable =
false };
408 enum :
bool { accessible =
true };
409 enum :
bool { deepcopy =
true };
423 void DeepCopyHIP(
void* dst,
const void* src,
size_t n);
424 void DeepCopyAsyncHIP(
const Kokkos::Experimental::HIP& instance,
void* dst,
425 const void* src,
size_t n);
426 void DeepCopyAsyncHIP(
void* dst,
const void* src,
size_t n);
428 template <
class MemSpace>
429 struct DeepCopy<MemSpace, HostSpace, Kokkos::Experimental::HIP,
430 std::enable_if_t<is_hip_type_space<MemSpace>::value>> {
431 DeepCopy(
void* dst,
const void* src,
size_t n) { DeepCopyHIP(dst, src, n); }
432 DeepCopy(
const Kokkos::Experimental::HIP& instance,
void* dst,
433 const void* src,
size_t n) {
434 DeepCopyAsyncHIP(instance, dst, src, n);
438 template <
class MemSpace>
439 struct DeepCopy<HostSpace, MemSpace, Kokkos::Experimental::HIP,
440 std::enable_if_t<is_hip_type_space<MemSpace>::value>> {
441 DeepCopy(
void* dst,
const void* src,
size_t n) { DeepCopyHIP(dst, src, n); }
442 DeepCopy(
const Kokkos::Experimental::HIP& instance,
void* dst,
443 const void* src,
size_t n) {
444 DeepCopyAsyncHIP(instance, dst, src, n);
448 template <
class MemSpace1,
class MemSpace2>
449 struct DeepCopy<MemSpace1, MemSpace2, Kokkos::Experimental::HIP,
450 std::enable_if_t<is_hip_type_space<MemSpace1>::value &&
451 is_hip_type_space<MemSpace2>::value>> {
452 DeepCopy(
void* dst,
const void* src,
size_t n) { DeepCopyHIP(dst, src, n); }
453 DeepCopy(
const Kokkos::Experimental::HIP& instance,
void* dst,
454 const void* src,
size_t n) {
455 DeepCopyAsyncHIP(instance, dst, src, n);
459 template <
class MemSpace1,
class MemSpace2,
class ExecutionSpace>
461 MemSpace1, MemSpace2, ExecutionSpace,
463 is_hip_type_space<MemSpace1>::value &&
464 is_hip_type_space<MemSpace2>::value &&
465 !std::is_same<ExecutionSpace, Kokkos::Experimental::HIP>::value>> {
466 inline DeepCopy(
void* dst,
const void* src,
size_t n) {
467 DeepCopyHIP(dst, src, n);
470 inline DeepCopy(
const ExecutionSpace& exec,
void* dst,
const void* src,
472 exec.fence(fence_string());
473 DeepCopyAsyncHIP(dst, src, n);
477 static const std::string& fence_string() {
478 static const std::string
string =
479 std::string(
"Kokkos::Impl::DeepCopy<") + MemSpace1::name() +
"Space, " +
481 "Space, ExecutionSpace>::DeepCopy: fence before copy";
486 template <
class MemSpace,
class ExecutionSpace>
488 MemSpace, HostSpace, ExecutionSpace,
490 is_hip_type_space<MemSpace>::value &&
491 !std::is_same<ExecutionSpace, Kokkos::Experimental::HIP>::value>> {
492 inline DeepCopy(
void* dst,
const void* src,
size_t n) {
493 DeepCopyHIP(dst, src, n);
496 inline DeepCopy(
const ExecutionSpace& exec,
void* dst,
const void* src,
498 exec.fence(fence_string());
499 DeepCopyAsyncHIP(dst, src, n);
503 static const std::string& fence_string() {
504 static const std::string
string =
505 std::string(
"Kokkos::Impl::DeepCopy<") + MemSpace::name() +
506 "Space, HostSpace, ExecutionSpace>::DeepCopy: fence before copy";
511 template <
class MemSpace,
class ExecutionSpace>
513 HostSpace, MemSpace, ExecutionSpace,
515 is_hip_type_space<MemSpace>::value &&
516 !std::is_same<ExecutionSpace, Kokkos::Experimental::HIP>::value>> {
517 inline DeepCopy(
void* dst,
const void* src,
size_t n) {
518 DeepCopyHIP(dst, src, n);
521 inline DeepCopy(
const ExecutionSpace& exec,
void* dst,
const void* src,
523 exec.fence(fence_string());
524 DeepCopyAsyncHIP(dst, src, n);
528 static const std::string& fence_string() {
529 static const std::string
string =
530 std::string(
"Kokkos::Impl::DeepCopy<HostSpace, ") + MemSpace::name() +
531 "Space, ExecutionSpace>::DeepCopy: fence before copy";
545 class SharedAllocationRecord<Kokkos::Experimental::HIPSpace, void>
546 :
public HostInaccessibleSharedAllocationRecordCommon<
547 Kokkos::Experimental::HIPSpace> {
549 friend class SharedAllocationRecordCommon<Kokkos::Experimental::HIPSpace>;
550 friend class HostInaccessibleSharedAllocationRecordCommon<
551 Kokkos::Experimental::HIPSpace>;
552 using base_t = HostInaccessibleSharedAllocationRecordCommon<
553 Kokkos::Experimental::HIPSpace>;
554 using RecordBase = SharedAllocationRecord<void, void>;
556 SharedAllocationRecord(const SharedAllocationRecord&) = delete;
557 SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete;
559 #ifdef KOKKOS_ENABLE_DEBUG
560 static RecordBase s_root_record;
563 const Kokkos::Experimental::HIPSpace m_space;
566 ~SharedAllocationRecord();
568 template <typename ExecutionSpace>
569 SharedAllocationRecord(
570 const ExecutionSpace& ,
571 const Kokkos::Experimental::HIPSpace& arg_space,
572 const std::string& arg_label, const size_t arg_alloc_size,
573 const RecordBase::function_type arg_dealloc = &base_t::deallocate)
574 : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size,
577 SharedAllocationRecord(
578 const Kokkos::Experimental::HIP& exec_space,
579 const Kokkos::Experimental::HIPSpace& arg_space,
580 const std::string& arg_label,
const size_t arg_alloc_size,
581 const RecordBase::function_type arg_dealloc = &base_t::deallocate);
583 SharedAllocationRecord(
584 const Kokkos::Experimental::HIPSpace& arg_space,
585 const std::string& arg_label,
const size_t arg_alloc_size,
586 const RecordBase::function_type arg_dealloc = &base_t::deallocate);
590 class SharedAllocationRecord<Kokkos::Experimental::HIPHostPinnedSpace, void>
591 :
public SharedAllocationRecordCommon<
592 Kokkos::Experimental::HIPHostPinnedSpace> {
594 friend class SharedAllocationRecordCommon<
595 Kokkos::Experimental::HIPHostPinnedSpace>;
597 SharedAllocationRecordCommon<Kokkos::Experimental::HIPHostPinnedSpace>;
598 using RecordBase = SharedAllocationRecord<void, void>;
600 SharedAllocationRecord(const SharedAllocationRecord&) = delete;
601 SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete;
603 #ifdef KOKKOS_ENABLE_DEBUG
604 static RecordBase s_root_record;
607 const Kokkos::Experimental::HIPHostPinnedSpace m_space;
610 ~SharedAllocationRecord();
611 SharedAllocationRecord() = default;
613 template <typename ExecutionSpace>
614 SharedAllocationRecord(
615 const ExecutionSpace& ,
616 const Kokkos::Experimental::HIPHostPinnedSpace& arg_space,
617 const std::string& arg_label, const size_t arg_alloc_size,
618 const RecordBase::function_type arg_dealloc = &base_t::deallocate)
619 : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size,
622 SharedAllocationRecord(
623 const Kokkos::Experimental::HIPHostPinnedSpace& arg_space,
624 const std::string& arg_label,
const size_t arg_alloc_size,
625 const RecordBase::function_type arg_dealloc = &base_t::deallocate);
629 class SharedAllocationRecord<Kokkos::Experimental::HIPManagedSpace, void>
630 :
public SharedAllocationRecordCommon<
631 Kokkos::Experimental::HIPManagedSpace> {
633 friend class SharedAllocationRecordCommon<
634 Kokkos::Experimental::HIPManagedSpace>;
636 SharedAllocationRecordCommon<Kokkos::Experimental::HIPManagedSpace>;
637 using RecordBase = SharedAllocationRecord<void, void>;
639 SharedAllocationRecord(const SharedAllocationRecord&) = delete;
640 SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete;
642 #ifdef KOKKOS_ENABLE_DEBUG
643 static RecordBase s_root_record;
646 const Kokkos::Experimental::HIPManagedSpace m_space;
649 ~SharedAllocationRecord();
650 SharedAllocationRecord() = default;
652 template <typename ExecutionSpace>
653 SharedAllocationRecord(
654 const ExecutionSpace& ,
655 const Kokkos::Experimental::HIPManagedSpace& arg_space,
656 const std::string& arg_label, const size_t arg_alloc_size,
657 const RecordBase::function_type arg_dealloc = &base_t::deallocate)
658 : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size,
661 SharedAllocationRecord(
662 const Kokkos::Experimental::HIPManagedSpace& arg_space,
663 const std::string& arg_label,
const size_t arg_alloc_size,
664 const RecordBase::function_type arg_dealloc = &base_t::deallocate);
673 namespace Experimental {
686 using execution_space = HIP;
687 using memory_space = HIPSpace;
688 using device_type = Kokkos::Device<execution_space, memory_space>;
690 using array_layout = LayoutLeft;
691 using size_type = HIPSpace::size_type;
693 using scratch_memory_space = ScratchMemorySpace<HIP>;
696 HIP(hipStream_t stream,
bool manage_stream =
false);
703 KOKKOS_INLINE_FUNCTION
static int in_parallel() {
704 #if defined(__HIP_DEVICE_COMPILE__)
717 static void impl_static_fence(
const std::string& name);
719 void fence(
const std::string& name =
720 "Kokkos::HIP::fence(): Unnamed Instance Fence")
const;
722 hipStream_t hip_stream()
const;
725 void print_configuration(std::ostream& os,
bool verbose =
false)
const;
728 static void impl_finalize();
733 int hip_device()
const;
734 static hipDeviceProp_t
const& hip_device_prop();
736 static void impl_initialize(InitializationSettings
const&);
738 static int impl_is_initialized();
742 static size_type detect_device_count();
744 static int concurrency();
745 static const char* name();
747 inline Impl::HIPInternal* impl_internal_space_instance()
const {
748 return m_space_instance.get();
751 uint32_t impl_instance_id() const noexcept;
754 Kokkos::Impl::HostSharedPtr<Impl::HIPInternal> m_space_instance;
758 namespace Experimental {
760 struct DeviceTypeTraits<Kokkos::Experimental::HIP> {
761 static constexpr DeviceType
id = DeviceType::HIP;
762 static int device_id(
const Kokkos::Experimental::HIP& exec) {
763 return exec.hip_device();
770 template <
class DT,
class... DP>
771 struct ZeroMemset<Kokkos::Experimental::HIP, DT, DP...> {
772 ZeroMemset(
const Kokkos::Experimental::HIP& exec_space,
773 const View<DT, DP...>& dst,
774 typename View<DT, DP...>::const_value_type&) {
775 KOKKOS_IMPL_HIP_SAFE_CALL(hipMemsetAsync(
777 dst.size() *
sizeof(
typename View<DT, DP...>::value_type),
778 exec_space.hip_stream()));
781 ZeroMemset(
const View<DT, DP...>& dst,
782 typename View<DT, DP...>::const_value_type&) {
783 KOKKOS_IMPL_HIP_SAFE_CALL(
784 hipMemset(dst.data(), 0,
785 dst.size() *
sizeof(
typename View<DT, DP...>::value_type)));
795 struct MemorySpaceAccess<Kokkos::Experimental::HIPSpace,
796 Kokkos::Experimental::HIP::scratch_memory_space> {
797 enum :
bool { assignable =
false };
798 enum :
bool { accessible =
true };
799 enum :
bool { deepcopy =
false };
Memory management for host memory.
Declaration of various MemoryLayout options.
DefaultHostExecutionSpace execution_space
Default execution space for this memory space.
Access relationship between DstMemorySpace and SrcMemorySpace.