Kokkos Core Kernels Package  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Groups Pages
Kokkos_ROCmSpace.hpp
1 /*
2 //@HEADER
3 // ************************************************************************
4 //
5 // Kokkos v. 2.0
6 // Copyright (2014) Sandia Corporation
7 //
8 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Redistribution and use in source and binary forms, with or without
12 // modification, are permitted provided that the following conditions are
13 // met:
14 //
15 // 1. Redistributions of source code must retain the above copyright
16 // notice, this list of conditions and the following disclaimer.
17 //
18 // 2. Redistributions in binary form must reproduce the above copyright
19 // notice, this list of conditions and the following disclaimer in the
20 // documentation and/or other materials provided with the distribution.
21 //
22 // 3. Neither the name of the Corporation nor the names of the
23 // contributors may be used to endorse or promote products derived from
24 // this software without specific prior written permission.
25 //
26 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 //
38 // Questions? Contact Christian R. Trott (crtrott@sandia.gov)
39 //
40 // ************************************************************************
41 //@HEADER
42 */
43 
44 #ifndef KOKKOS_ROCMSPACE_HPP
45 #define KOKKOS_ROCMSPACE_HPP
46 
47 #include <Kokkos_Core_fwd.hpp>
48 
49 #if defined( KOKKOS_ENABLE_ROCM )
50 
51 #include <iosfwd>
52 #include <typeinfo>
53 #include <string>
54 
55 #include <Kokkos_HostSpace.hpp>
56 
57 
58 /*--------------------------------------------------------------------------*/
59 
60 namespace Kokkos {
61 namespace Experimental {
64 class ROCmSpace {
65 public:
66 
68  typedef ROCmSpace memory_space ;
69  typedef Kokkos::Experimental::ROCm execution_space ;
70  typedef Kokkos::Device<execution_space,memory_space> device_type;
71 
72  typedef unsigned int size_type ;
73 
74  /*--------------------------------*/
75 
76  ROCmSpace();
77  ROCmSpace( ROCmSpace && rhs ) = default ;
78  ROCmSpace( const ROCmSpace & rhs ) = default ;
79  ROCmSpace & operator = ( ROCmSpace && rhs ) = default ;
80  ROCmSpace & operator = ( const ROCmSpace & rhs ) = default ;
81  ~ROCmSpace() = default ;
82 
84  void * allocate( const size_t arg_alloc_size ) const ;
85 
87  void deallocate( void * const arg_alloc_ptr
88  , const size_t arg_alloc_size ) const ;
89 
91  static constexpr const char* name() { return m_name; };
92 
93  /*--------------------------------*/
95  static void access_error();
96  static void access_error( const void * const );
97 
98 private:
99 
100  int m_device ;
101 
102  static constexpr const char* m_name = "ROCm";
103  friend class Kokkos::Impl::SharedAllocationRecord< Kokkos::Experimental::ROCmSpace , void > ;
104 };
105 
106 } // namespace Experimental
107 
108 namespace Impl {
109 
110 void * rocm_device_allocate(int);
111 void * rocm_hostpinned_allocate(int);
112 void rocm_device_free(void * );
113 
120 void init_lock_arrays_rocm_space();
121 
129 int* atomic_lock_array_rocm_space_ptr(bool deallocate = false);
130 
137 int* scratch_lock_array_rocm_space_ptr(bool deallocate = false);
138 
145 int* threadid_lock_array_rocm_space_ptr(bool deallocate = false);
146 }
147 } // namespace Kokkos
148 
149 /*--------------------------------------------------------------------------*/
150 /*--------------------------------------------------------------------------*/
151 
152 
153 namespace Kokkos {
154 namespace Experimental {
158 class ROCmHostPinnedSpace {
159 public:
160 
162 
163  typedef HostSpace::execution_space execution_space ;
164  typedef ROCmHostPinnedSpace memory_space ;
165  typedef Kokkos::Device<execution_space,memory_space> device_type;
166  typedef unsigned int size_type ;
167 
168  /*--------------------------------*/
169 
170  ROCmHostPinnedSpace();
171  ROCmHostPinnedSpace( ROCmHostPinnedSpace && rhs ) = default ;
172  ROCmHostPinnedSpace( const ROCmHostPinnedSpace & rhs ) = default ;
173  ROCmHostPinnedSpace & operator = ( ROCmHostPinnedSpace && rhs ) = default ;
174  ROCmHostPinnedSpace & operator = ( const ROCmHostPinnedSpace & rhs ) = default ;
175  ~ROCmHostPinnedSpace() = default ;
176 
178  void * allocate( const size_t arg_alloc_size ) const ;
179 
181  void deallocate( void * const arg_alloc_ptr
182  , const size_t arg_alloc_size ) const ;
183 
185  static constexpr const char* name() { return m_name; };
186 
187 private:
188 
189  static constexpr const char* m_name = "ROCmHostPinned";
190 
191  /*--------------------------------*/
192 };
193 } // namespace Experimental
194 } // namespace Kokkos
195 
196 /*--------------------------------------------------------------------------*/
197 /*--------------------------------------------------------------------------*/
198 
199 namespace Kokkos {
200 namespace Impl {
201 
202 static_assert( Kokkos::Impl::MemorySpaceAccess< Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCmSpace >::assignable , "" );
203 
204 //----------------------------------------
205 
206 template<>
207 struct MemorySpaceAccess< Kokkos::HostSpace , Kokkos::Experimental::ROCmSpace > {
208  enum { assignable = false };
209  enum { accessible = false };
210  enum { deepcopy = true };
211 };
212 
213 template<>
214 struct MemorySpaceAccess< Kokkos::HostSpace , Kokkos::Experimental::ROCmHostPinnedSpace > {
215  // HostSpace::execution_space == ROCmHostPinnedSpace::execution_space
216  enum { assignable = true };
217  enum { accessible = true };
218  enum { deepcopy = true };
219 };
220 
221 //----------------------------------------
222 
223 template<>
224 struct MemorySpaceAccess< Kokkos::Experimental::ROCmSpace , Kokkos::HostSpace > {
225  enum { assignable = false };
226  enum { accessible = false };
227  enum { deepcopy = true };
228 };
229 
230 template<>
231 struct MemorySpaceAccess< Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCmHostPinnedSpace > {
232  // ROCmSpace::execution_space != ROCmHostPinnedSpace::execution_space
233  enum { assignable = false };
234  enum { accessible = true }; // ROCmSpace::execution_space
235  enum { deepcopy = true };
236 };
237 
238 
239 //----------------------------------------
240 // ROCmHostPinnedSpace::execution_space == HostSpace::execution_space
241 // ROCmHostPinnedSpace accessible to both ROCm and Host
242 
243 template<>
244 struct MemorySpaceAccess< Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::HostSpace > {
245  enum { assignable = false }; // Cannot access from ROCm
246  enum { accessible = true }; // ROCmHostPinnedSpace::execution_space
247  enum { deepcopy = true };
248 };
249 
250 template<>
251 struct MemorySpaceAccess< Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCmSpace > {
252  enum { assignable = false }; // Cannot access from Host
253  enum { accessible = false };
254  enum { deepcopy = true };
255 };
256 
257 };
258 //----------------------------------------
259 
260 } // namespace Kokkos::Impl
261 
262 /*--------------------------------------------------------------------------*/
263 /*--------------------------------------------------------------------------*/
264 
265 namespace Kokkos {
266 namespace Impl {
267 
268 hc::completion_future DeepCopyAsyncROCm( void * dst , const void * src , size_t n);
269 
270 template<> struct DeepCopy< Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCm>
271 {
272  DeepCopy( void * dst , const void * src , size_t );
273  DeepCopy( const Kokkos::Experimental::ROCm & , void * dst , const void * src , size_t );
274 };
275 
276 template<> struct DeepCopy< Kokkos::Experimental::ROCmSpace , HostSpace , Kokkos::Experimental::ROCm >
277 {
278  DeepCopy( void * dst , const void * src , size_t );
279  DeepCopy( const Kokkos::Experimental::ROCm & , void * dst , const void * src , size_t );
280 };
281 
282 template<> struct DeepCopy< HostSpace , Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCm >
283 {
284  DeepCopy( void * dst , const void * src , size_t );
285  DeepCopy( const Kokkos::Experimental::ROCm & , void * dst , const void * src , size_t );
286 };
287 
288 template<class ExecutionSpace> struct DeepCopy< Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCmSpace , ExecutionSpace >
289 {
290  inline
291  DeepCopy( void * dst , const void * src , size_t n )
292  { (void) DeepCopy< Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCm >( dst , src , n ); }
293 
294  inline
295  DeepCopy( const ExecutionSpace& exec, void * dst , const void * src , size_t n )
296  {
297  exec.fence();
298  hc::completion_future fut = DeepCopyAsyncROCm (dst,src,n);
299  fut.wait();
300 // DeepCopy (dst,src,n);
301  }
302 };
303 
304 template<class ExecutionSpace> struct DeepCopy< Kokkos::Experimental::ROCmSpace , HostSpace , ExecutionSpace >
305 {
306  inline
307  DeepCopy( void * dst , const void * src , size_t n )
308  { (void) DeepCopy< Kokkos::Experimental::ROCmSpace , HostSpace , Kokkos::Experimental::ROCm>( dst , src , n ); }
309 
310  inline
311  DeepCopy( const ExecutionSpace& exec, void * dst , const void * src , size_t n )
312  {
313  exec.fence();
314  DeepCopy (dst,src,n);
315  }
316 };
317 
318 template<class ExecutionSpace>
319 struct DeepCopy< HostSpace , Kokkos::Experimental::ROCmSpace , ExecutionSpace >
320 {
321  inline
322  DeepCopy( void * dst , const void * src , size_t n )
323  { (void) DeepCopy< HostSpace , Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCm >( dst , src , n ); }
324 
325  inline
326  DeepCopy( const ExecutionSpace& exec, void * dst , const void * src , size_t n )
327  {
328  exec.fence();
329  DeepCopy (dst,src,n);
330  }
331 };
332 
333 template<> struct DeepCopy< Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCm>
334 {
335  DeepCopy( void * dst , const void * src , size_t );
336  DeepCopy( const Kokkos::Experimental::ROCm & , void * dst , const void * src , size_t );
337 };
338 
339 template<> struct DeepCopy< Kokkos::Experimental::ROCmHostPinnedSpace , HostSpace , Kokkos::Experimental::ROCm >
340 {
341  DeepCopy( void * dst , const void * src , size_t );
342  DeepCopy( const Kokkos::Experimental::ROCm & , void * dst , const void * src , size_t );
343 };
344 
345 template<> struct DeepCopy< HostSpace , Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCm >
346 {
347  DeepCopy( void * dst , const void * src , size_t );
348  DeepCopy( const Kokkos::Experimental::ROCm & , void * dst , const void * src , size_t );
349 };
350 
351 template<class ExecutionSpace>
352 struct DeepCopy< Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCmHostPinnedSpace , ExecutionSpace>
353 {
354  inline
355  DeepCopy( void * dst , const void * src , size_t n )
356  { (void) DeepCopy< Kokkos::Experimental::ROCmSpace , HostSpace , Kokkos::Experimental::ROCm >( dst , src , n ); }
357 
358  inline
359  DeepCopy( const ExecutionSpace& exec, void * dst , const void * src , size_t n )
360  {
361  exec.fence();
362  hc::completion_future fut = DeepCopyAsyncROCm (dst,src,n);
363  fut.wait();
364 // DeepCopyROCm (dst,src,n);
365  }
366 };
367 
368 template<class ExecutionSpace> struct DeepCopy< Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCmSpace , ExecutionSpace >
369 {
370  inline
371  DeepCopy( void * dst , const void * src , size_t n )
372  { (void) DeepCopy< HostSpace , Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCm >( dst , src , n ); }
373 
374  inline
375  DeepCopy( const ExecutionSpace& exec, void * dst , const void * src , size_t n )
376  {
377  exec.fence();
378  hc::completion_future fut = DeepCopyAsyncROCm (dst,src,n);
379  fut.wait();
380 // DeepCopyROCm (dst,src,n);
381  }
382 };
383 
384 
385 
386 template<class ExecutionSpace> struct DeepCopy< Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCmHostPinnedSpace , ExecutionSpace >
387 {
388  inline
389  DeepCopy( void * dst , const void * src , size_t n )
390  { (void) DeepCopy< Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCm >( dst , src , n ); }
391 
392  inline
393  DeepCopy( const ExecutionSpace& exec, void * dst , const void * src , size_t n )
394  {
395  exec.fence();
396 // hc::completion_future fut = DeepCopyAsyncROCm (dst,src,n);
397 // fut.wait();
398 // DeepCopyAsyncROCm (dst,src,n);
399  DeepCopy (dst,src,n);
400  }
401 };
402 
403 template<class ExecutionSpace> struct DeepCopy< Kokkos::Experimental::ROCmHostPinnedSpace , HostSpace , ExecutionSpace >
404 {
405  inline
406  DeepCopy( void * dst , const void * src , size_t n )
407  { (void) DeepCopy< Kokkos::Experimental::ROCmHostPinnedSpace , HostSpace , Kokkos::Experimental::ROCm>( dst , src , n ); }
408 
409  inline
410  DeepCopy( const ExecutionSpace& exec, void * dst , const void * src , size_t n )
411  {
412  exec.fence();
413  DeepCopy (dst,src,n);
414  }
415 };
416 
417 template<class ExecutionSpace>
418 struct DeepCopy< HostSpace , Kokkos::Experimental::ROCmHostPinnedSpace , ExecutionSpace >
419 {
420  inline
421  DeepCopy( void * dst , const void * src , size_t n )
422  { (void) DeepCopy< HostSpace , Kokkos::Experimental::ROCmHostPinnedSpace , Kokkos::Experimental::ROCm >( dst , src , n ); }
423 
424  inline
425  DeepCopy( const ExecutionSpace& exec, void * dst , const void * src , size_t n )
426  {
427  exec.fence();
428  DeepCopy (dst,src,n);
429  }
430 };
431 } // namespace Impl
432 } // namespace Kokkos
433 
434 //----------------------------------------------------------------------------
435 //----------------------------------------------------------------------------
436 
437 namespace Kokkos {
438 namespace Impl {
439 
441 template<>
442 struct VerifyExecutionCanAccessMemorySpace< Kokkos::Experimental::ROCmSpace , Kokkos::HostSpace >
443 {
444  enum { value = false };
445  KOKKOS_INLINE_FUNCTION static void verify( void )
446  { Kokkos::abort("ROCm code attempted to access HostSpace memory"); }
447 
448  KOKKOS_INLINE_FUNCTION static void verify( const void * )
449  { Kokkos::abort("ROCm code attempted to access HostSpace memory"); }
450 };
451 
453 template<>
454 struct VerifyExecutionCanAccessMemorySpace< Kokkos::Experimental::ROCmSpace , Kokkos::Experimental::ROCmHostPinnedSpace >
455 {
456  enum { value = true };
457  KOKKOS_INLINE_FUNCTION static void verify( void ) { }
458  KOKKOS_INLINE_FUNCTION static void verify( const void * ) { }
459 };
460 
462 template< class OtherSpace >
463 struct VerifyExecutionCanAccessMemorySpace<
464  typename enable_if< ! is_same<Kokkos::Experimental::ROCmSpace,OtherSpace>::value , Kokkos::Experimental::ROCmSpace >::type ,
465  OtherSpace >
466 {
467  enum { value = false };
468  KOKKOS_INLINE_FUNCTION static void verify( void )
469  { Kokkos::abort("ROCm code attempted to access unknown Space memory"); }
470 
471  KOKKOS_INLINE_FUNCTION static void verify( const void * )
472  { Kokkos::abort("ROCm code attempted to access unknown Space memory"); }
473 };
474 
475 //----------------------------------------------------------------------------
477 template<>
478 struct VerifyExecutionCanAccessMemorySpace< Kokkos::HostSpace , Kokkos::Experimental::ROCmSpace >
479 {
480  enum { value = false };
481  inline static void verify( void ) { Kokkos::Experimental::ROCmSpace::access_error(); }
482  inline static void verify( const void * p ) { Kokkos::Experimental::ROCmSpace::access_error(p); }
483 };
484 
486 template<>
487 struct VerifyExecutionCanAccessMemorySpace< Kokkos::HostSpace , Kokkos::Experimental::ROCmHostPinnedSpace >
488 {
489  enum { value = true };
490  KOKKOS_INLINE_FUNCTION static void verify( void ) {}
491  KOKKOS_INLINE_FUNCTION static void verify( const void * ) {}
492 };
493 } // namespace Impl
494 } // namespace Kokkos
495 
496 //----------------------------------------------------------------------------
497 //----------------------------------------------------------------------------
498 
499 namespace Kokkos {
500 namespace Impl {
501 
502 template<>
503 class SharedAllocationRecord< Kokkos::Experimental::ROCmSpace , void >
504  : public SharedAllocationRecord< void , void >
505 {
506 private:
507 
508 
509  typedef SharedAllocationRecord< void , void > RecordBase ;
510 
511  SharedAllocationRecord( const SharedAllocationRecord & ) = delete ;
512  SharedAllocationRecord & operator = ( const SharedAllocationRecord & ) = delete ;
513 
514  static void deallocate( RecordBase * );
515 
516 #ifdef KOKKOS_DEBUG
517  static RecordBase s_root_record ;
518 #endif
519 
520  const Kokkos::Experimental::ROCmSpace m_space ;
521 
522 protected:
523 
524  ~SharedAllocationRecord();
525 
526  SharedAllocationRecord( const Kokkos::Experimental::ROCmSpace & arg_space
527  , const std::string & arg_label
528  , const size_t arg_alloc_size
529  , const RecordBase::function_type arg_dealloc = & deallocate
530  );
531 
532 public:
533 
534  std::string get_label() const ;
535 
536  static SharedAllocationRecord * allocate( const Kokkos::Experimental::ROCmSpace & arg_space
537  , const std::string & arg_label
538  , const size_t arg_alloc_size );
539 
541  static
542  void * allocate_tracked( const Kokkos::Experimental::ROCmSpace & arg_space
543  , const std::string & arg_label
544  , const size_t arg_alloc_size );
545 
547  static
548  void * reallocate_tracked( void * const arg_alloc_ptr
549  , const size_t arg_alloc_size );
550 
552  static
553  void deallocate_tracked( void * const arg_alloc_ptr );
554 
555  static SharedAllocationRecord * get_record( void * arg_alloc_ptr );
556 
557  static void print_records( std::ostream & , const Kokkos::Experimental::ROCmSpace & , bool detail = false );
558 };
559 
560 template<>
561 class SharedAllocationRecord< Kokkos::Experimental::ROCmHostPinnedSpace , void >
562  : public SharedAllocationRecord< void , void >
563 {
564 private:
565 
566  typedef SharedAllocationRecord< void , void > RecordBase ;
567 
568  SharedAllocationRecord( const SharedAllocationRecord & ) = delete ;
569  SharedAllocationRecord & operator = ( const SharedAllocationRecord & ) = delete ;
570 
571  static void deallocate( RecordBase * );
572 
573 #ifdef KOKKOS_DEBUG
574  static RecordBase s_root_record ;
575 #endif
576 
577  const Kokkos::Experimental::ROCmHostPinnedSpace m_space ;
578 
579 protected:
580 
581  ~SharedAllocationRecord();
582  SharedAllocationRecord() : RecordBase(), m_space() {}
583 
584  SharedAllocationRecord( const Kokkos::Experimental::ROCmHostPinnedSpace & arg_space
585  , const std::string & arg_label
586  , const size_t arg_alloc_size
587  , const RecordBase::function_type arg_dealloc = & deallocate
588  );
589 
590 public:
591 
592  std::string get_label() const ;
593 
594  static SharedAllocationRecord * allocate( const Kokkos::Experimental::ROCmHostPinnedSpace & arg_space
595  , const std::string & arg_label
596  , const size_t arg_alloc_size
597  );
599  static
600  void * allocate_tracked( const Kokkos::Experimental::ROCmHostPinnedSpace & arg_space
601  , const std::string & arg_label
602  , const size_t arg_alloc_size );
603 
605  static
606  void * reallocate_tracked( void * const arg_alloc_ptr
607  , const size_t arg_alloc_size );
608 
610  static
611  void deallocate_tracked( void * const arg_alloc_ptr );
612 
613 
614  static SharedAllocationRecord * get_record( void * arg_alloc_ptr );
615 
616  static void print_records( std::ostream & , const Kokkos::Experimental::ROCmHostPinnedSpace & , bool detail = false );
617 };
618 } // namespace Impl
619 } // namespace Kokkos
620 
621 //----------------------------------------------------------------------------
622 //----------------------------------------------------------------------------
623 
624 #endif /* #if defined( KOKKOS_ENABLE_ROCM ) */
625 #endif /* #define KOKKOS_ROCMSPACE_HPP */
626 
Kokkos::HostSpace
Memory management for host memory.
Definition: Kokkos_HostSpace.hpp:102
Kokkos::Impl::MemorySpaceAccess
Access relationship between DstMemorySpace and SrcMemorySpace.
Definition: Kokkos_Concepts.hpp:308
Generated by   doxygen 1.8.5