17 #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18 #include <Kokkos_Macros.hpp>
20 "Including non-public Kokkos header files is not allowed.");
22 #ifndef KOKKOS_LOGICALSPACES_HPP
23 #define KOKKOS_LOGICALSPACES_HPP
25 #include <Kokkos_Macros.hpp>
26 #include <Kokkos_Core_fwd.hpp>
27 #include <Kokkos_ScratchSpace.hpp>
28 #include <impl/Kokkos_MemorySpace.hpp>
29 #include <impl/Kokkos_Error.hpp>
30 #include <impl/Kokkos_SharedAlloc.hpp>
31 #include <impl/Kokkos_Profiling.hpp>
34 namespace Experimental {
35 struct DefaultMemorySpaceNamer {
36 static constexpr
const char* get_name() {
37 return "DefaultLogicalMemorySpaceName";
41 struct LogicalSpaceSharesAccess {
42 struct shared_access {};
43 struct no_shared_access {};
51 template <
class BaseSpace,
class DefaultBaseExecutionSpace = void,
52 class Namer = DefaultMemorySpaceNamer,
53 class SharesAccessWithBase = LogicalSpaceSharesAccess::shared_access>
54 class LogicalMemorySpace {
55 #ifdef KOKKOS_ENABLE_OPENMPTARGET
60 static_assert(!std::is_same<BaseSpace, BaseSpace>::value,
61 "Can't use LogicalMemorySpaces in an OpenMPTarget build, we're "
62 "debugging memory issues");
66 using memory_space = LogicalMemorySpace<BaseSpace, DefaultBaseExecutionSpace,
67 Namer, SharesAccessWithBase>;
68 using size_type =
typename BaseSpace::size_type;
78 std::conditional_t<std::is_void<DefaultBaseExecutionSpace>::value,
79 typename BaseSpace::execution_space,
80 DefaultBaseExecutionSpace>;
82 using device_type = Kokkos::Device<execution_space, memory_space>;
86 template <
typename... Args>
90 void*
allocate(
const size_t arg_alloc_size)
const {
91 return allocate(
"[unlabeled]", arg_alloc_size);
93 void*
allocate(
const char* arg_label,
const size_t arg_alloc_size,
94 const size_t arg_logical_size = 0)
const {
95 return impl_allocate(arg_label, arg_alloc_size, arg_logical_size);
100 const size_t arg_alloc_size)
const {
101 deallocate(
"[unlabeled]", arg_alloc_ptr, arg_alloc_size);
103 void deallocate(
const char* arg_label,
void*
const arg_alloc_ptr,
104 const size_t arg_alloc_size,
105 const size_t arg_logical_size = 0)
const {
106 impl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size, arg_logical_size);
110 constexpr
static const char*
name() {
return Namer::get_name(); }
113 BaseSpace underlying_space;
114 template <
class,
class,
class,
class>
116 friend class Kokkos::Impl::SharedAllocationRecord<
memory_space, void>;
118 void* impl_allocate(
const char* arg_label,
const size_t arg_alloc_size,
119 const size_t arg_logical_size = 0,
120 Kokkos::Tools::SpaceHandle arg_handle =
121 Kokkos::Tools::make_space_handle(
name()))
const {
122 return underlying_space.impl_allocate(arg_label, arg_alloc_size,
123 arg_logical_size, arg_handle);
125 void impl_deallocate(
const char* arg_label,
void*
const arg_alloc_ptr,
126 const size_t arg_alloc_size,
127 const size_t arg_logical_size = 0,
128 const Kokkos::Tools::SpaceHandle arg_handle =
129 Kokkos::Tools::make_space_handle(
name()))
const {
130 underlying_space.impl_deallocate(arg_label, arg_alloc_ptr, arg_alloc_size,
131 arg_logical_size, arg_handle);
143 template <
typename BaseSpace,
typename DefaultBaseExecutionSpace,
class Namer,
145 struct MemorySpaceAccess<
146 Kokkos::Experimental::LogicalMemorySpace<
147 BaseSpace, DefaultBaseExecutionSpace, Namer,
148 Kokkos::Experimental::LogicalSpaceSharesAccess::shared_access>,
150 enum { assignable = MemorySpaceAccess<BaseSpace, OtherSpace>::assignable };
151 enum { accessible = MemorySpaceAccess<BaseSpace, OtherSpace>::accessible };
152 enum { deepcopy = MemorySpaceAccess<BaseSpace, OtherSpace>::deepcopy };
155 template <
typename BaseSpace,
typename DefaultBaseExecutionSpace,
class Namer,
157 struct MemorySpaceAccess<
159 Kokkos::Experimental::LogicalMemorySpace<
160 BaseSpace, DefaultBaseExecutionSpace, Namer,
161 Kokkos::Experimental::LogicalSpaceSharesAccess::shared_access>> {
162 enum { assignable = MemorySpaceAccess<OtherSpace, BaseSpace>::assignable };
163 enum { accessible = MemorySpaceAccess<OtherSpace, BaseSpace>::accessible };
164 enum { deepcopy = MemorySpaceAccess<OtherSpace, BaseSpace>::deepcopy };
167 template <
typename BaseSpace,
typename DefaultBaseExecutionSpace,
class Namer>
168 struct MemorySpaceAccess<
169 Kokkos::Experimental::LogicalMemorySpace<
170 BaseSpace, DefaultBaseExecutionSpace, Namer,
171 Kokkos::Experimental::LogicalSpaceSharesAccess::shared_access>,
173 BaseSpace, DefaultBaseExecutionSpace, Namer,
174 Kokkos::Experimental::LogicalSpaceSharesAccess::shared_access>> {
175 enum { assignable =
true };
176 enum { accessible =
true };
177 enum { deepcopy =
true };
189 template <
class BaseSpace,
class DefaultBaseExecutionSpace,
class Namer,
190 class SharesAccessSemanticsWithBase>
191 class SharedAllocationRecord<Kokkos::Experimental::LogicalMemorySpace<
192 BaseSpace, DefaultBaseExecutionSpace, Namer,
193 SharesAccessSemanticsWithBase>,
194 void> :
public SharedAllocationRecord<void, void> {
198 DefaultBaseExecutionSpace, Namer,
199 SharesAccessSemanticsWithBase>;
200 using RecordBase = SharedAllocationRecord<void, void>;
202 SharedAllocationRecord(
const SharedAllocationRecord&) =
delete;
203 SharedAllocationRecord& operator=(
const SharedAllocationRecord&) =
delete;
205 static void deallocate(RecordBase* arg_rec) {
206 delete static_cast<SharedAllocationRecord*
>(arg_rec);
209 #ifdef KOKKOS_ENABLE_DEBUG
212 static RecordBase s_root_record;
215 const SpaceType m_space;
218 ~SharedAllocationRecord() {
219 m_space.
deallocate(RecordBase::m_alloc_ptr->m_label,
220 SharedAllocationRecord<void, void>::m_alloc_ptr,
221 SharedAllocationRecord<void, void>::m_alloc_size,
222 (SharedAllocationRecord<void, void>::m_alloc_size -
223 sizeof(SharedAllocationHeader)));
225 SharedAllocationRecord() =
default;
227 template <
typename ExecutionSpace>
228 SharedAllocationRecord(
229 const ExecutionSpace& ,
const SpaceType& arg_space,
230 const std::string& arg_label,
const size_t arg_alloc_size,
231 const RecordBase::function_type arg_dealloc = &deallocate)
232 : SharedAllocationRecord(arg_space, arg_label, arg_alloc_size,
235 SharedAllocationRecord(
236 const SpaceType& arg_space,
const std::string& arg_label,
237 const size_t arg_alloc_size,
238 const RecordBase::function_type arg_dealloc = &deallocate)
239 : SharedAllocationRecord<void, void>(
240 #ifdef KOKKOS_ENABLE_DEBUG
241 &SharedAllocationRecord<SpaceType, void>::s_root_record,
243 Impl::checked_allocation_with_header(arg_space, arg_label,
245 sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc,
249 RecordBase::m_alloc_ptr->m_record =
250 static_cast<SharedAllocationRecord<void, void>*
>(
this);
252 strncpy(RecordBase::m_alloc_ptr->m_label, arg_label.c_str(),
253 SharedAllocationHeader::maximum_label_length - 1);
255 RecordBase::m_alloc_ptr
256 ->m_label[SharedAllocationHeader::maximum_label_length - 1] =
'\0';
260 inline std::string get_label()
const {
261 return std::string(RecordBase::head()->m_label);
263 KOKKOS_INLINE_FUNCTION
static SharedAllocationRecord* allocate(
264 const SpaceType& arg_space,
const std::string& arg_label,
265 const size_t arg_alloc_size) {
266 KOKKOS_IF_ON_HOST((
return new SharedAllocationRecord(arg_space, arg_label,
268 KOKKOS_IF_ON_DEVICE(((
void)arg_space; (
void)arg_label; (
void)arg_alloc_size;
273 static
void* allocate_tracked(const SpaceType& arg_space,
274 const std::
string& arg_label,
275 const
size_t arg_alloc_size) {
276 if (!arg_alloc_size)
return (
void*)
nullptr;
278 SharedAllocationRecord*
const r =
279 allocate(arg_space, arg_label, arg_alloc_size);
281 RecordBase::increment(r);
287 static void* reallocate_tracked(
void*
const arg_alloc_ptr,
288 const size_t arg_alloc_size) {
289 SharedAllocationRecord*
const r_old = get_record(arg_alloc_ptr);
290 SharedAllocationRecord*
const r_new =
291 allocate(r_old->m_space, r_old->get_label(), arg_alloc_size);
293 Kokkos::Impl::DeepCopy<SpaceType, SpaceType>(
294 r_new->data(), r_old->data(), std::min(r_old->size(), r_new->size()));
296 "SharedAllocationRecord<Kokkos::Experimental::LogicalMemorySpace, "
297 "void>::reallocate_tracked: fence after copying data");
299 RecordBase::increment(r_new);
300 RecordBase::decrement(r_old);
302 return r_new->data();
305 static void deallocate_tracked(
void*
const arg_alloc_ptr) {
306 if (arg_alloc_ptr !=
nullptr) {
307 SharedAllocationRecord*
const r = get_record(arg_alloc_ptr);
309 RecordBase::decrement(r);
313 static SharedAllocationRecord* get_record(
void* alloc_ptr) {
314 using Header = SharedAllocationHeader;
315 using RecordHost = SharedAllocationRecord<SpaceType, void>;
317 SharedAllocationHeader
const*
const head =
318 alloc_ptr ? Header::get_header(alloc_ptr)
319 : (SharedAllocationHeader*)nullptr;
320 RecordHost*
const record =
321 head ?
static_cast<RecordHost*
>(head->m_record) : (RecordHost*)nullptr;
323 if (!alloc_ptr || record->m_alloc_ptr != head) {
324 Kokkos::Impl::throw_runtime_exception(std::string(
325 "Kokkos::Impl::SharedAllocationRecord< LogicalMemorySpace<> , "
326 "void >::get_record ERROR"));
331 #ifdef KOKKOS_ENABLE_DEBUG
332 static void print_records(std::ostream& s,
const SpaceType&,
333 bool detail =
false) {
334 SharedAllocationRecord<void, void>::print_host_accessible_records(
335 s,
"HostSpace", &s_root_record, detail);
338 static void print_records(std::ostream&,
const SpaceType&,
339 bool detail =
false) {
341 throw_runtime_exception(
342 "SharedAllocationRecord<HostSpace>::print_records only works "
343 "with KOKKOS_ENABLE_DEBUG enabled");
347 #ifdef KOKKOS_ENABLE_DEBUG
350 template <
class BaseSpace,
class DefaultBaseExecutionSpace,
class Namer,
351 class SharesAccessSemanticsWithBase>
352 SharedAllocationRecord<void, void>
354 BaseSpace, DefaultBaseExecutionSpace, Namer,
355 SharesAccessSemanticsWithBase>,
356 void>::s_root_record;
369 template <
class Namer,
class BaseSpace,
class DefaultBaseExecutionSpace,
370 class SharesAccess,
class ExecutionSpace>
371 struct DeepCopy<Kokkos::Experimental::LogicalMemorySpace<
372 BaseSpace, DefaultBaseExecutionSpace, Namer, SharesAccess>,
374 BaseSpace, DefaultBaseExecutionSpace, Namer, SharesAccess>,
376 DeepCopy(
void* dst,
void* src,
size_t n) {
377 DeepCopy<BaseSpace, BaseSpace, ExecutionSpace>(dst, src, n);
379 DeepCopy(
const ExecutionSpace& exec,
void* dst,
void* src,
size_t n) {
380 DeepCopy<BaseSpace, BaseSpace, ExecutionSpace>(exec, dst, src, n);
384 template <
class Namer,
class BaseSpace,
class DefaultBaseExecutionSpace,
385 class SharesAccess,
class ExecutionSpace,
class SourceSpace>
386 struct DeepCopy<SourceSpace,
387 Kokkos::Experimental::LogicalMemorySpace<
388 BaseSpace, DefaultBaseExecutionSpace, Namer, SharesAccess>,
390 DeepCopy(
void* dst,
void* src,
size_t n) {
391 DeepCopy<SourceSpace, BaseSpace, ExecutionSpace>(dst, src, n);
393 DeepCopy(
const ExecutionSpace& exec,
void* dst,
void* src,
size_t n) {
394 DeepCopy<SourceSpace, BaseSpace, ExecutionSpace>(exec, dst, src, n);
398 template <
class Namer,
class BaseSpace,
class DefaultBaseExecutionSpace,
399 class SharesAccess,
class ExecutionSpace,
class DestinationSpace>
400 struct DeepCopy<Kokkos::Experimental::LogicalMemorySpace<
401 BaseSpace, DefaultBaseExecutionSpace, Namer, SharesAccess>,
402 DestinationSpace, ExecutionSpace> {
403 DeepCopy(
void* dst,
void* src,
size_t n) {
404 DeepCopy<BaseSpace, DestinationSpace, ExecutionSpace>(dst, src, n);
406 DeepCopy(
const ExecutionSpace& exec,
void* dst,
void* src,
size_t n) {
407 DeepCopy<BaseSpace, DestinationSpace, ExecutionSpace>(exec, dst, src, n);
413 #endif // KOKKOS_LOGICALSPACES_HPP
static constexpr const char * name()
Return Name of the MemorySpace.
void deallocate(void *const arg_alloc_ptr, const size_t arg_alloc_size) const
Deallocate untracked memory in the space.
LogicalMemorySpace< BaseSpace, DefaultBaseExecutionSpace, Namer, SharesAccessWithBase > memory_space
Tag this class as a kokkos memory space.
void * allocate(const size_t arg_alloc_size) const
Allocate untracked memory in the space.
std::conditional_t< std::is_void< DefaultBaseExecutionSpace >::value, typename BaseSpace::execution_space, DefaultBaseExecutionSpace > execution_space
Default execution space for this memory space.
LogicalMemorySpace is a space that is identical to another space, but differentiable by name and temp...