44 #ifndef KOKKOS_ROCMSPACE_HPP
45 #define KOKKOS_ROCMSPACE_HPP
47 #include <Kokkos_Core_fwd.hpp>
49 #if defined( KOKKOS_ENABLE_ROCM )
55 #include <Kokkos_HostSpace.hpp>
61 namespace Experimental {
68 typedef ROCmSpace memory_space ;
69 typedef Kokkos::Experimental::ROCm execution_space ;
70 typedef Kokkos::Device<execution_space,memory_space> device_type;
72 typedef unsigned int size_type ;
77 ROCmSpace( ROCmSpace && rhs ) = default ;
78 ROCmSpace(
const ROCmSpace & rhs ) = default ;
79 ROCmSpace & operator = ( ROCmSpace && rhs ) = default ;
80 ROCmSpace & operator = (
const ROCmSpace & rhs ) = default ;
81 ~ROCmSpace() = default ;
84 void * allocate(
const size_t arg_alloc_size )
const ;
87 void deallocate(
void *
const arg_alloc_ptr
88 ,
const size_t arg_alloc_size )
const ;
91 static constexpr
const char* name() {
return m_name; };
95 static void access_error();
96 static void access_error(
const void *
const );
102 static constexpr
const char* m_name =
"ROCm";
103 friend class Kokkos::Impl::SharedAllocationRecord< Kokkos::Experimental::ROCmSpace , void > ;
110 void * rocm_device_allocate(
int);
111 void * rocm_hostpinned_allocate(
int);
112 void rocm_device_free(
void * );
120 void init_lock_arrays_rocm_space();
129 int* atomic_lock_array_rocm_space_ptr(
bool deallocate =
false);
137 int* scratch_lock_array_rocm_space_ptr(
bool deallocate =
false);
145 int* threadid_lock_array_rocm_space_ptr(
bool deallocate =
false);
154 namespace Experimental {
158 class ROCmHostPinnedSpace {
163 typedef HostSpace::execution_space execution_space ;
164 typedef ROCmHostPinnedSpace memory_space ;
165 typedef Kokkos::Device<execution_space,memory_space> device_type;
166 typedef unsigned int size_type ;
170 ROCmHostPinnedSpace();
171 ROCmHostPinnedSpace( ROCmHostPinnedSpace && rhs ) = default ;
172 ROCmHostPinnedSpace(
const ROCmHostPinnedSpace & rhs ) = default ;
173 ROCmHostPinnedSpace & operator = ( ROCmHostPinnedSpace && rhs ) = default ;
174 ROCmHostPinnedSpace & operator = (
const ROCmHostPinnedSpace & rhs ) = default ;
175 ~ROCmHostPinnedSpace() = default ;
178 void * allocate(
const size_t arg_alloc_size )
const ;
181 void deallocate(
void *
const arg_alloc_ptr
182 ,
const size_t arg_alloc_size )
const ;
185 static constexpr
const char* name() {
return m_name; };
189 static constexpr
const char* m_name =
"ROCmHostPinned";
207 struct MemorySpaceAccess< Kokkos::HostSpace , Kokkos::Experimental::ROCmSpace > {
208 enum { assignable =
false };
209 enum { accessible =
false };
210 enum { deepcopy =
true };
214 struct MemorySpaceAccess< Kokkos::HostSpace , Kokkos::Experimental::ROCmHostPinnedSpace > {
216 enum { assignable =
true };
217 enum { accessible =
true };
218 enum { deepcopy =
true };
224 struct MemorySpaceAccess< Kokkos::Experimental::ROCmSpace ,
Kokkos::HostSpace > {
225 enum { assignable =
false };
226 enum { accessible =
false };
227 enum { deepcopy =
true };
231 struct MemorySpaceAccess< Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCmHostPinnedSpace > {
233 enum { assignable =
false };
234 enum { accessible =
true };
235 enum { deepcopy =
true };
244 struct MemorySpaceAccess< Kokkos::Experimental::ROCmHostPinnedSpace ,
Kokkos::HostSpace > {
245 enum { assignable =
false };
246 enum { accessible =
true };
247 enum { deepcopy =
true };
251 struct MemorySpaceAccess< Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCmSpace > {
252 enum { assignable =
false };
253 enum { accessible =
false };
254 enum { deepcopy =
true };
268 hc::completion_future DeepCopyAsyncROCm(
void * dst ,
const void * src ,
size_t n);
270 template<>
struct DeepCopy< Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCm>
272 DeepCopy(
void * dst ,
const void * src ,
size_t );
273 DeepCopy(
const Kokkos::Experimental::ROCm & ,
void * dst ,
const void * src ,
size_t );
276 template<>
struct DeepCopy< Kokkos::Experimental::ROCmSpace , HostSpace , Kokkos::Experimental::ROCm >
278 DeepCopy(
void * dst ,
const void * src ,
size_t );
279 DeepCopy(
const Kokkos::Experimental::ROCm & ,
void * dst ,
const void * src ,
size_t );
282 template<>
struct DeepCopy< HostSpace , Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCm >
284 DeepCopy(
void * dst ,
const void * src ,
size_t );
285 DeepCopy(
const Kokkos::Experimental::ROCm & ,
void * dst ,
const void * src ,
size_t );
288 template<
class ExecutionSpace>
struct DeepCopy< Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCmSpace , ExecutionSpace >
291 DeepCopy(
void * dst ,
const void * src ,
size_t n )
292 { (void) DeepCopy< Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCm >( dst , src , n ); }
295 DeepCopy(
const ExecutionSpace& exec,
void * dst ,
const void * src ,
size_t n )
298 hc::completion_future fut = DeepCopyAsyncROCm (dst,src,n);
304 template<
class ExecutionSpace>
struct DeepCopy< Kokkos::Experimental::ROCmSpace , HostSpace , ExecutionSpace >
307 DeepCopy(
void * dst ,
const void * src ,
size_t n )
308 { (void) DeepCopy< Kokkos::Experimental::ROCmSpace , HostSpace , Kokkos::Experimental::ROCm>( dst , src , n ); }
311 DeepCopy(
const ExecutionSpace& exec,
void * dst ,
const void * src ,
size_t n )
314 DeepCopy (dst,src,n);
318 template<
class ExecutionSpace>
319 struct DeepCopy< HostSpace , Kokkos::Experimental::ROCmSpace , ExecutionSpace >
322 DeepCopy(
void * dst ,
const void * src ,
size_t n )
323 { (void) DeepCopy< HostSpace , Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCm >( dst , src , n ); }
326 DeepCopy(
const ExecutionSpace& exec,
void * dst ,
const void * src ,
size_t n )
329 DeepCopy (dst,src,n);
333 template<>
struct DeepCopy< Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCm>
335 DeepCopy(
void * dst ,
const void * src ,
size_t );
336 DeepCopy(
const Kokkos::Experimental::ROCm & ,
void * dst ,
const void * src ,
size_t );
339 template<>
struct DeepCopy< Kokkos::Experimental::ROCmHostPinnedSpace , HostSpace , Kokkos::Experimental::ROCm >
341 DeepCopy(
void * dst ,
const void * src ,
size_t );
342 DeepCopy(
const Kokkos::Experimental::ROCm & ,
void * dst ,
const void * src ,
size_t );
345 template<>
struct DeepCopy< HostSpace , Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCm >
347 DeepCopy(
void * dst ,
const void * src ,
size_t );
348 DeepCopy(
const Kokkos::Experimental::ROCm & ,
void * dst ,
const void * src ,
size_t );
351 template<
class ExecutionSpace>
352 struct DeepCopy< Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCmHostPinnedSpace , ExecutionSpace>
355 DeepCopy(
void * dst ,
const void * src ,
size_t n )
356 { (void) DeepCopy< Kokkos::Experimental::ROCmSpace , HostSpace , Kokkos::Experimental::ROCm >( dst , src , n ); }
359 DeepCopy(
const ExecutionSpace& exec,
void * dst ,
const void * src ,
size_t n )
362 hc::completion_future fut = DeepCopyAsyncROCm (dst,src,n);
368 template<
class ExecutionSpace>
struct DeepCopy< Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCmSpace , ExecutionSpace >
371 DeepCopy(
void * dst ,
const void * src ,
size_t n )
372 { (void) DeepCopy< HostSpace , Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCm >( dst , src , n ); }
375 DeepCopy(
const ExecutionSpace& exec,
void * dst ,
const void * src ,
size_t n )
378 hc::completion_future fut = DeepCopyAsyncROCm (dst,src,n);
386 template<
class ExecutionSpace>
struct DeepCopy< Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCmHostPinnedSpace , ExecutionSpace >
389 DeepCopy(
void * dst ,
const void * src ,
size_t n )
390 { (void) DeepCopy< Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCm >( dst , src , n ); }
393 DeepCopy(
const ExecutionSpace& exec,
void * dst ,
const void * src ,
size_t n )
399 DeepCopy (dst,src,n);
403 template<
class ExecutionSpace>
struct DeepCopy< Kokkos::Experimental::ROCmHostPinnedSpace , HostSpace , ExecutionSpace >
406 DeepCopy(
void * dst ,
const void * src ,
size_t n )
407 { (void) DeepCopy< Kokkos::Experimental::ROCmHostPinnedSpace , HostSpace , Kokkos::Experimental::ROCm>( dst , src , n ); }
410 DeepCopy(
const ExecutionSpace& exec,
void * dst ,
const void * src ,
size_t n )
413 DeepCopy (dst,src,n);
417 template<
class ExecutionSpace>
418 struct DeepCopy< HostSpace , Kokkos::Experimental::ROCmHostPinnedSpace , ExecutionSpace >
421 DeepCopy(
void * dst ,
const void * src ,
size_t n )
422 { (void) DeepCopy< HostSpace , Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCm >( dst , src , n ); }
425 DeepCopy(
const ExecutionSpace& exec,
void * dst ,
const void * src ,
size_t n )
428 DeepCopy (dst,src,n);
442 struct VerifyExecutionCanAccessMemorySpace< Kokkos::Experimental::ROCmSpace ,
Kokkos::HostSpace >
444 enum { value =
false };
445 KOKKOS_INLINE_FUNCTION
static void verify(
void )
446 { Kokkos::abort(
"ROCm code attempted to access HostSpace memory"); }
448 KOKKOS_INLINE_FUNCTION
static void verify(
const void * )
449 { Kokkos::abort(
"ROCm code attempted to access HostSpace memory"); }
454 struct VerifyExecutionCanAccessMemorySpace< Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCmHostPinnedSpace >
456 enum { value =
true };
457 KOKKOS_INLINE_FUNCTION
static void verify(
void ) { }
458 KOKKOS_INLINE_FUNCTION
static void verify(
const void * ) { }
462 template<
class OtherSpace >
463 struct VerifyExecutionCanAccessMemorySpace<
464 typename enable_if< ! is_same<Kokkos::Experimental::ROCmSpace,OtherSpace>::value , Kokkos::Experimental::ROCmSpace >::type ,
467 enum { value =
false };
468 KOKKOS_INLINE_FUNCTION
static void verify(
void )
469 { Kokkos::abort(
"ROCm code attempted to access unknown Space memory"); }
471 KOKKOS_INLINE_FUNCTION
static void verify(
const void * )
472 { Kokkos::abort(
"ROCm code attempted to access unknown Space memory"); }
478 struct VerifyExecutionCanAccessMemorySpace< Kokkos::HostSpace , Kokkos::Experimental::ROCmSpace >
480 enum { value =
false };
481 inline static void verify(
void ) { Kokkos::Experimental::ROCmSpace::access_error(); }
482 inline static void verify(
const void * p ) { Kokkos::Experimental::ROCmSpace::access_error(p); }
487 struct VerifyExecutionCanAccessMemorySpace< Kokkos::HostSpace , Kokkos::Experimental::ROCmHostPinnedSpace >
489 enum { value =
true };
490 KOKKOS_INLINE_FUNCTION
static void verify(
void ) {}
491 KOKKOS_INLINE_FUNCTION
static void verify(
const void * ) {}
503 class SharedAllocationRecord< Kokkos::Experimental::ROCmSpace , void >
504 :
public SharedAllocationRecord< void , void >
509 typedef SharedAllocationRecord< void , void > RecordBase ;
511 SharedAllocationRecord(
const SharedAllocationRecord & ) = delete ;
512 SharedAllocationRecord & operator = (
const SharedAllocationRecord & ) = delete ;
514 static void deallocate( RecordBase * );
517 static RecordBase s_root_record ;
520 const Kokkos::Experimental::ROCmSpace m_space ;
524 ~SharedAllocationRecord();
526 SharedAllocationRecord(
const Kokkos::Experimental::ROCmSpace & arg_space
527 ,
const std::string & arg_label
528 ,
const size_t arg_alloc_size
529 ,
const RecordBase::function_type arg_dealloc = & deallocate
534 std::string get_label()
const ;
536 static SharedAllocationRecord * allocate(
const Kokkos::Experimental::ROCmSpace & arg_space
537 ,
const std::string & arg_label
538 ,
const size_t arg_alloc_size );
542 void * allocate_tracked(
const Kokkos::Experimental::ROCmSpace & arg_space
543 ,
const std::string & arg_label
544 ,
const size_t arg_alloc_size );
548 void * reallocate_tracked(
void *
const arg_alloc_ptr
549 ,
const size_t arg_alloc_size );
553 void deallocate_tracked(
void *
const arg_alloc_ptr );
555 static SharedAllocationRecord * get_record(
void * arg_alloc_ptr );
557 static void print_records( std::ostream & ,
const Kokkos::Experimental::ROCmSpace & ,
bool detail =
false );
561 class SharedAllocationRecord< Kokkos::Experimental::ROCmHostPinnedSpace , void >
562 :
public SharedAllocationRecord< void , void >
566 typedef SharedAllocationRecord< void , void > RecordBase ;
568 SharedAllocationRecord(
const SharedAllocationRecord & ) = delete ;
569 SharedAllocationRecord & operator = (
const SharedAllocationRecord & ) = delete ;
571 static void deallocate( RecordBase * );
574 static RecordBase s_root_record ;
577 const Kokkos::Experimental::ROCmHostPinnedSpace m_space ;
581 ~SharedAllocationRecord();
582 SharedAllocationRecord() : RecordBase(), m_space() {}
584 SharedAllocationRecord(
const Kokkos::Experimental::ROCmHostPinnedSpace & arg_space
585 ,
const std::string & arg_label
586 ,
const size_t arg_alloc_size
587 ,
const RecordBase::function_type arg_dealloc = & deallocate
592 std::string get_label()
const ;
594 static SharedAllocationRecord * allocate(
const Kokkos::Experimental::ROCmHostPinnedSpace & arg_space
595 ,
const std::string & arg_label
596 ,
const size_t arg_alloc_size
600 void * allocate_tracked(
const Kokkos::Experimental::ROCmHostPinnedSpace & arg_space
601 ,
const std::string & arg_label
602 ,
const size_t arg_alloc_size );
606 void * reallocate_tracked(
void *
const arg_alloc_ptr
607 ,
const size_t arg_alloc_size );
611 void deallocate_tracked(
void *
const arg_alloc_ptr );
614 static SharedAllocationRecord * get_record(
void * arg_alloc_ptr );
616 static void print_records( std::ostream & ,
const Kokkos::Experimental::ROCmHostPinnedSpace & ,
bool detail =
false );
Memory management for host memory.
Access relationship between DstMemorySpace and SrcMemorySpace.