17 #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18 #include <Kokkos_Macros.hpp>
20 "Including non-public Kokkos header files is not allowed.");
22 #ifndef KOKKOS_COPYVIEWS_HPP_
23 #define KOKKOS_COPYVIEWS_HPP_
27 #include <KokkosExp_MDRangePolicy.hpp>
29 #include <impl/Kokkos_HostSpace_ZeroMemset.hpp>
38 template <
class Layout>
39 struct ViewFillLayoutSelector {};
42 struct ViewFillLayoutSelector<Kokkos::LayoutLeft> {
43 static const Kokkos::Iterate iterate = Kokkos::Iterate::Left;
47 struct ViewFillLayoutSelector<Kokkos::LayoutRight> {
48 static const Kokkos::Iterate iterate = Kokkos::Iterate::Right;
57 template <
class ViewType,
class Layout,
class ExecSpace,
typename iType>
58 struct ViewFill<ViewType, Layout, ExecSpace, 0, iType> {
59 using ST =
typename ViewType::non_const_value_type;
60 ViewFill(
const ViewType& a,
const ST& val,
const ExecSpace& space) {
62 ExecSpace>(space, a.data(), &val,
sizeof(ST));
66 template <
class ViewType,
class Layout,
class ExecSpace,
typename iType>
67 struct ViewFill<ViewType, Layout, ExecSpace, 1, iType> {
69 typename ViewType::const_value_type val;
72 ViewFill(
const ViewType& a_,
typename ViewType::const_value_type& val_,
73 const ExecSpace& space)
75 Kokkos::parallel_for(
"Kokkos::ViewFill-1D",
76 policy_type(space, 0, a.extent(0)), *
this);
79 KOKKOS_INLINE_FUNCTION
80 void operator()(
const iType& i)
const { a(i) = val; }
83 template <
class ViewType,
class Layout,
class ExecSpace,
typename iType>
84 struct ViewFill<ViewType, Layout, ExecSpace, 2, iType> {
86 typename ViewType::const_value_type val;
88 using iterate_type = Kokkos::Rank<2, ViewFillLayoutSelector<Layout>::iterate,
89 ViewFillLayoutSelector<Layout>::iterate>;
91 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
93 ViewFill(
const ViewType& a_,
typename ViewType::const_value_type& val_,
94 const ExecSpace& space)
96 Kokkos::parallel_for(
"Kokkos::ViewFill-2D",
97 policy_type(space, {0, 0}, {a.extent(0), a.extent(1)}),
101 KOKKOS_INLINE_FUNCTION
102 void operator()(
const iType& i0,
const iType& i1)
const { a(i0, i1) = val; }
105 template <
class ViewType,
class Layout,
class ExecSpace,
typename iType>
106 struct ViewFill<ViewType, Layout, ExecSpace, 3, iType> {
108 typename ViewType::const_value_type val;
110 using iterate_type = Kokkos::Rank<3, ViewFillLayoutSelector<Layout>::iterate,
111 ViewFillLayoutSelector<Layout>::iterate>;
113 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
115 ViewFill(
const ViewType& a_,
typename ViewType::const_value_type& val_,
116 const ExecSpace& space)
118 Kokkos::parallel_for(
119 "Kokkos::ViewFill-3D",
120 policy_type(space, {0, 0, 0}, {a.extent(0), a.extent(1), a.extent(2)}),
124 KOKKOS_INLINE_FUNCTION
125 void operator()(
const iType& i0,
const iType& i1,
const iType& i2)
const {
130 template <
class ViewType,
class Layout,
class ExecSpace,
typename iType>
131 struct ViewFill<ViewType, Layout, ExecSpace, 4, iType> {
133 typename ViewType::const_value_type val;
135 using iterate_type = Kokkos::Rank<4, ViewFillLayoutSelector<Layout>::iterate,
136 ViewFillLayoutSelector<Layout>::iterate>;
138 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
140 ViewFill(
const ViewType& a_,
typename ViewType::const_value_type& val_,
141 const ExecSpace& space)
143 Kokkos::parallel_for(
144 "Kokkos::ViewFill-4D",
145 policy_type(space, {0, 0, 0, 0},
146 {a.extent(0), a.extent(1), a.extent(2), a.extent(3)}),
150 KOKKOS_INLINE_FUNCTION
151 void operator()(
const iType& i0,
const iType& i1,
const iType& i2,
152 const iType& i3)
const {
153 a(i0, i1, i2, i3) = val;
157 template <
class ViewType,
class Layout,
class ExecSpace,
typename iType>
158 struct ViewFill<ViewType, Layout, ExecSpace, 5, iType> {
160 typename ViewType::const_value_type val;
162 using iterate_type = Kokkos::Rank<5, ViewFillLayoutSelector<Layout>::iterate,
163 ViewFillLayoutSelector<Layout>::iterate>;
165 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
167 ViewFill(
const ViewType& a_,
typename ViewType::const_value_type& val_,
168 const ExecSpace& space)
170 Kokkos::parallel_for(
"Kokkos::ViewFill-5D",
171 policy_type(space, {0, 0, 0, 0, 0},
172 {a.extent(0), a.extent(1), a.extent(2),
173 a.extent(3), a.extent(4)}),
177 KOKKOS_INLINE_FUNCTION
178 void operator()(
const iType& i0,
const iType& i1,
const iType& i2,
179 const iType& i3,
const iType& i4)
const {
180 a(i0, i1, i2, i3, i4) = val;
184 template <
class ViewType,
class Layout,
class ExecSpace,
typename iType>
185 struct ViewFill<ViewType, Layout, ExecSpace, 6, iType> {
187 typename ViewType::const_value_type val;
189 using iterate_type = Kokkos::Rank<6, ViewFillLayoutSelector<Layout>::iterate,
190 ViewFillLayoutSelector<Layout>::iterate>;
192 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
194 ViewFill(
const ViewType& a_,
typename ViewType::const_value_type& val_,
195 const ExecSpace& space)
197 Kokkos::parallel_for(
"Kokkos::ViewFill-6D",
198 policy_type(space, {0, 0, 0, 0, 0, 0},
199 {a.extent(0), a.extent(1), a.extent(2),
200 a.extent(3), a.extent(4), a.extent(5)}),
204 KOKKOS_INLINE_FUNCTION
205 void operator()(
const iType& i0,
const iType& i1,
const iType& i2,
206 const iType& i3,
const iType& i4,
const iType& i5)
const {
207 a(i0, i1, i2, i3, i4, i5) = val;
211 template <
class ViewType,
class Layout,
class ExecSpace,
typename iType>
212 struct ViewFill<ViewType, Layout, ExecSpace, 7, iType> {
214 typename ViewType::const_value_type val;
216 using iterate_type = Kokkos::Rank<6, ViewFillLayoutSelector<Layout>::iterate,
217 ViewFillLayoutSelector<Layout>::iterate>;
219 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
221 ViewFill(
const ViewType& a_,
typename ViewType::const_value_type& val_,
222 const ExecSpace& space)
226 Kokkos::parallel_for(
"Kokkos::ViewFill-7D",
227 policy_type(space, {0, 0, 0, 0, 0, 0},
228 {a.extent(0), a.extent(1), a.extent(3),
229 a.extent(4), a.extent(5), a.extent(6)}),
233 KOKKOS_INLINE_FUNCTION
234 void operator()(
const iType& i0,
const iType& i1,
const iType& i3,
235 const iType& i4,
const iType& i5,
const iType& i6)
const {
236 for (iType i2 = 0; i2 < iType(a.extent(2)); i2++)
237 a(i0, i1, i2, i3, i4, i5, i6) = val;
241 template <
class ViewType,
class Layout,
class ExecSpace,
typename iType>
242 struct ViewFill<ViewType, Layout, ExecSpace, 8, iType> {
244 typename ViewType::const_value_type val;
246 using iterate_type = Kokkos::Rank<6, ViewFillLayoutSelector<Layout>::iterate,
247 ViewFillLayoutSelector<Layout>::iterate>;
249 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
251 ViewFill(
const ViewType& a_,
typename ViewType::const_value_type& val_,
252 const ExecSpace& space)
256 Kokkos::parallel_for(
"Kokkos::ViewFill-8D",
257 policy_type(space, {0, 0, 0, 0, 0, 0},
258 {a.extent(0), a.extent(1), a.extent(3),
259 a.extent(5), a.extent(6), a.extent(7)}),
263 KOKKOS_INLINE_FUNCTION
264 void operator()(
const iType& i0,
const iType& i1,
const iType& i3,
265 const iType& i5,
const iType& i6,
const iType& i7)
const {
266 for (iType i2 = 0; i2 < iType(a.extent(2)); i2++)
267 for (iType i4 = 0; i4 < iType(a.extent(4)); i4++)
268 a(i0, i1, i2, i3, i4, i5, i6, i7) = val;
272 template <
class ViewTypeA,
class ViewTypeB,
class Layout,
class ExecSpace,
274 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 1, iType> {
279 using value_type =
typename ViewTypeA::value_type;
281 ViewCopy(
const ViewTypeA& a_,
const ViewTypeB& b_,
282 const ExecSpace space = ExecSpace())
284 Kokkos::parallel_for(
"Kokkos::ViewCopy-1D",
285 policy_type(space, 0, a.extent(0)), *
this);
288 KOKKOS_INLINE_FUNCTION
289 void operator()(
const iType& i0)
const {
290 a(i0) =
static_cast<value_type
>(b(i0));
294 template <
class ViewTypeA,
class ViewTypeB,
class Layout,
class ExecSpace,
296 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 2, iType> {
299 static const Kokkos::Iterate outer_iteration_pattern =
300 Kokkos::Impl::layout_iterate_type_selector<
301 Layout>::outer_iteration_pattern;
302 static const Kokkos::Iterate inner_iteration_pattern =
303 Kokkos::Impl::layout_iterate_type_selector<
304 Layout>::inner_iteration_pattern;
306 Kokkos::Rank<2, outer_iteration_pattern, inner_iteration_pattern>;
308 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
309 using value_type =
typename ViewTypeA::value_type;
311 ViewCopy(
const ViewTypeA& a_,
const ViewTypeB& b_,
312 const ExecSpace space = ExecSpace())
314 Kokkos::parallel_for(
"Kokkos::ViewCopy-2D",
315 policy_type(space, {0, 0}, {a.extent(0), a.extent(1)}),
319 KOKKOS_INLINE_FUNCTION
320 void operator()(
const iType& i0,
const iType& i1)
const {
321 a(i0, i1) =
static_cast<value_type
>(b(i0, i1));
325 template <
class ViewTypeA,
class ViewTypeB,
class Layout,
class ExecSpace,
327 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 3, iType> {
331 static const Kokkos::Iterate outer_iteration_pattern =
332 Kokkos::Impl::layout_iterate_type_selector<
333 Layout>::outer_iteration_pattern;
334 static const Kokkos::Iterate inner_iteration_pattern =
335 Kokkos::Impl::layout_iterate_type_selector<
336 Layout>::inner_iteration_pattern;
338 Kokkos::Rank<3, outer_iteration_pattern, inner_iteration_pattern>;
340 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
341 using value_type =
typename ViewTypeA::value_type;
343 ViewCopy(
const ViewTypeA& a_,
const ViewTypeB& b_,
344 const ExecSpace space = ExecSpace())
346 Kokkos::parallel_for(
347 "Kokkos::ViewCopy-3D",
348 policy_type(space, {0, 0, 0}, {a.extent(0), a.extent(1), a.extent(2)}),
352 KOKKOS_INLINE_FUNCTION
353 void operator()(
const iType& i0,
const iType& i1,
const iType& i2)
const {
354 a(i0, i1, i2) =
static_cast<value_type
>(b(i0, i1, i2));
358 template <
class ViewTypeA,
class ViewTypeB,
class Layout,
class ExecSpace,
360 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 4, iType> {
364 static const Kokkos::Iterate outer_iteration_pattern =
365 Kokkos::Impl::layout_iterate_type_selector<
366 Layout>::outer_iteration_pattern;
367 static const Kokkos::Iterate inner_iteration_pattern =
368 Kokkos::Impl::layout_iterate_type_selector<
369 Layout>::inner_iteration_pattern;
371 Kokkos::Rank<4, outer_iteration_pattern, inner_iteration_pattern>;
373 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
375 ViewCopy(
const ViewTypeA& a_,
const ViewTypeB& b_,
376 const ExecSpace space = ExecSpace())
378 Kokkos::parallel_for(
379 "Kokkos::ViewCopy-4D",
380 policy_type(space, {0, 0, 0, 0},
381 {a.extent(0), a.extent(1), a.extent(2), a.extent(3)}),
385 KOKKOS_INLINE_FUNCTION
386 void operator()(
const iType& i0,
const iType& i1,
const iType& i2,
387 const iType& i3)
const {
388 a(i0, i1, i2, i3) = b(i0, i1, i2, i3);
392 template <
class ViewTypeA,
class ViewTypeB,
class Layout,
class ExecSpace,
394 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 5, iType> {
398 static const Kokkos::Iterate outer_iteration_pattern =
399 Kokkos::Impl::layout_iterate_type_selector<
400 Layout>::outer_iteration_pattern;
401 static const Kokkos::Iterate inner_iteration_pattern =
402 Kokkos::Impl::layout_iterate_type_selector<
403 Layout>::inner_iteration_pattern;
405 Kokkos::Rank<5, outer_iteration_pattern, inner_iteration_pattern>;
407 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
409 ViewCopy(
const ViewTypeA& a_,
const ViewTypeB& b_,
410 const ExecSpace space = ExecSpace())
412 Kokkos::parallel_for(
"Kokkos::ViewCopy-5D",
413 policy_type(space, {0, 0, 0, 0, 0},
414 {a.extent(0), a.extent(1), a.extent(2),
415 a.extent(3), a.extent(4)}),
419 KOKKOS_INLINE_FUNCTION
420 void operator()(
const iType& i0,
const iType& i1,
const iType& i2,
421 const iType& i3,
const iType& i4)
const {
422 a(i0, i1, i2, i3, i4) = b(i0, i1, i2, i3, i4);
426 template <
class ViewTypeA,
class ViewTypeB,
class Layout,
class ExecSpace,
428 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 6, iType> {
432 static const Kokkos::Iterate outer_iteration_pattern =
433 Kokkos::Impl::layout_iterate_type_selector<
434 Layout>::outer_iteration_pattern;
435 static const Kokkos::Iterate inner_iteration_pattern =
436 Kokkos::Impl::layout_iterate_type_selector<
437 Layout>::inner_iteration_pattern;
439 Kokkos::Rank<6, outer_iteration_pattern, inner_iteration_pattern>;
441 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
443 ViewCopy(
const ViewTypeA& a_,
const ViewTypeB& b_,
444 const ExecSpace space = ExecSpace())
446 Kokkos::parallel_for(
"Kokkos::ViewCopy-6D",
447 policy_type(space, {0, 0, 0, 0, 0, 0},
448 {a.extent(0), a.extent(1), a.extent(2),
449 a.extent(3), a.extent(4), a.extent(5)}),
453 KOKKOS_INLINE_FUNCTION
454 void operator()(
const iType& i0,
const iType& i1,
const iType& i2,
455 const iType& i3,
const iType& i4,
const iType& i5)
const {
456 a(i0, i1, i2, i3, i4, i5) = b(i0, i1, i2, i3, i4, i5);
460 template <
class ViewTypeA,
class ViewTypeB,
class Layout,
class ExecSpace,
462 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 7, iType> {
466 static const Kokkos::Iterate outer_iteration_pattern =
467 Kokkos::Impl::layout_iterate_type_selector<
468 Layout>::outer_iteration_pattern;
469 static const Kokkos::Iterate inner_iteration_pattern =
470 Kokkos::Impl::layout_iterate_type_selector<
471 Layout>::inner_iteration_pattern;
473 Kokkos::Rank<6, outer_iteration_pattern, inner_iteration_pattern>;
475 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
477 ViewCopy(
const ViewTypeA& a_,
const ViewTypeB& b_,
478 const ExecSpace space = ExecSpace())
482 Kokkos::parallel_for(
"Kokkos::ViewCopy-7D",
483 policy_type(space, {0, 0, 0, 0, 0, 0},
484 {a.extent(0), a.extent(1), a.extent(3),
485 a.extent(4), a.extent(5), a.extent(6)}),
489 KOKKOS_INLINE_FUNCTION
490 void operator()(
const iType& i0,
const iType& i1,
const iType& i3,
491 const iType& i4,
const iType& i5,
const iType& i6)
const {
492 for (iType i2 = 0; i2 < iType(a.extent(2)); i2++)
493 a(i0, i1, i2, i3, i4, i5, i6) = b(i0, i1, i2, i3, i4, i5, i6);
497 template <
class ViewTypeA,
class ViewTypeB,
class Layout,
class ExecSpace,
499 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 8, iType> {
503 static const Kokkos::Iterate outer_iteration_pattern =
504 Kokkos::Impl::layout_iterate_type_selector<
505 Layout>::outer_iteration_pattern;
506 static const Kokkos::Iterate inner_iteration_pattern =
507 Kokkos::Impl::layout_iterate_type_selector<
508 Layout>::inner_iteration_pattern;
510 Kokkos::Rank<6, outer_iteration_pattern, inner_iteration_pattern>;
512 Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
514 ViewCopy(
const ViewTypeA& a_,
const ViewTypeB& b_,
515 const ExecSpace space = ExecSpace())
519 Kokkos::parallel_for(
"Kokkos::ViewCopy-8D",
520 policy_type(space, {0, 0, 0, 0, 0, 0},
521 {a.extent(0), a.extent(1), a.extent(3),
522 a.extent(5), a.extent(6), a.extent(7)}),
526 KOKKOS_INLINE_FUNCTION
527 void operator()(
const iType& i0,
const iType& i1,
const iType& i3,
528 const iType& i5,
const iType& i6,
const iType& i7)
const {
529 for (iType i2 = 0; i2 < iType(a.extent(2)); i2++)
530 for (iType i4 = 0; i4 < iType(a.extent(4)); i4++)
531 a(i0, i1, i2, i3, i4, i5, i6, i7) = b(i0, i1, i2, i3, i4, i5, i6, i7);
541 template <
class DstType>
542 Kokkos::Iterate get_iteration_order(
const DstType& dst) {
543 int64_t strides[DstType::rank + 1];
545 Kokkos::Iterate iterate;
546 if (std::is_same_v<typename DstType::array_layout, Kokkos::LayoutRight>) {
547 iterate = Kokkos::Iterate::Right;
548 }
else if (std::is_same_v<
typename DstType::array_layout,
550 iterate = Kokkos::Iterate::Left;
551 }
else if (std::is_same_v<
typename DstType::array_layout,
553 if (strides[0] > strides[DstType::rank - 1])
554 iterate = Kokkos::Iterate::Right;
556 iterate = Kokkos::Iterate::Left;
558 if (std::is_same_v<
typename DstType::execution_space::array_layout,
560 iterate = Kokkos::Iterate::Right;
562 iterate = Kokkos::Iterate::Left;
567 template <
class ExecutionSpace,
class DstType,
class SrcType>
568 void view_copy(
const ExecutionSpace& space,
const DstType& dst,
569 const SrcType& src) {
570 using dst_memory_space =
typename DstType::memory_space;
571 using src_memory_space =
typename SrcType::memory_space;
573 constexpr
bool ExecCanAccessSrc =
575 constexpr
bool ExecCanAccessDst =
578 if (!(ExecCanAccessSrc && ExecCanAccessDst)) {
579 Kokkos::Impl::throw_runtime_exception(
580 "Kokkos::Impl::view_copy called with invalid execution space");
583 Kokkos::Iterate iterate = get_iteration_order(dst);
585 if ((dst.span() >= size_t(std::numeric_limits<int>::max())) ||
586 (src.span() >= size_t(std::numeric_limits<int>::max()))) {
587 if (iterate == Kokkos::Iterate::Right)
588 Kokkos::Impl::ViewCopy<
589 typename DstType::uniform_runtime_nomemspace_type,
590 typename SrcType::uniform_runtime_const_nomemspace_type,
594 Kokkos::Impl::ViewCopy<
595 typename DstType::uniform_runtime_nomemspace_type,
596 typename SrcType::uniform_runtime_const_nomemspace_type,
600 if (iterate == Kokkos::Iterate::Right)
601 Kokkos::Impl::ViewCopy<
602 typename DstType::uniform_runtime_nomemspace_type,
603 typename SrcType::uniform_runtime_const_nomemspace_type,
607 Kokkos::Impl::ViewCopy<
608 typename DstType::uniform_runtime_nomemspace_type,
609 typename SrcType::uniform_runtime_const_nomemspace_type,
616 template <
class DstType,
class SrcType>
617 void view_copy(
const DstType& dst,
const SrcType& src) {
618 using dst_execution_space =
typename DstType::execution_space;
619 using src_execution_space =
typename SrcType::execution_space;
620 using dst_memory_space =
typename DstType::memory_space;
621 using src_memory_space =
typename SrcType::memory_space;
623 constexpr
bool DstExecCanAccessSrc =
625 src_memory_space>::accessible;
627 constexpr
bool SrcExecCanAccessDst =
629 dst_memory_space>::accessible;
631 if (!DstExecCanAccessSrc && !SrcExecCanAccessDst) {
632 std::ostringstream ss;
633 ss <<
"Error: Kokkos::deep_copy with no available copy mechanism: "
634 <<
"from source view (\"" << src.label() <<
"\") to destination view (\""
635 << dst.label() <<
"\").\n"
636 <<
"There is no common execution space that can access both source's "
638 <<
"(" << src_memory_space().name() <<
") and destination's space ("
639 << dst_memory_space().name() <<
"), "
640 <<
"so source and destination\n"
641 <<
"must be contiguous and have the same layout.\n";
642 Kokkos::Impl::throw_runtime_exception(ss.str());
645 using ExecutionSpace =
646 std::conditional_t<DstExecCanAccessSrc, dst_execution_space,
647 src_execution_space>;
650 Kokkos::Iterate iterate = get_iteration_order(dst);
652 if ((dst.span() >= size_t(std::numeric_limits<int>::max())) ||
653 (src.span() >= size_t(std::numeric_limits<int>::max()))) {
654 if (iterate == Kokkos::Iterate::Right)
655 Kokkos::Impl::ViewCopy<
656 typename DstType::uniform_runtime_nomemspace_type,
657 typename SrcType::uniform_runtime_const_nomemspace_type,
661 Kokkos::Impl::ViewCopy<
662 typename DstType::uniform_runtime_nomemspace_type,
663 typename SrcType::uniform_runtime_const_nomemspace_type,
666 if (iterate == Kokkos::Iterate::Right)
667 Kokkos::Impl::ViewCopy<
668 typename DstType::uniform_runtime_nomemspace_type,
669 typename SrcType::uniform_runtime_const_nomemspace_type,
672 Kokkos::Impl::ViewCopy<
673 typename DstType::uniform_runtime_nomemspace_type,
674 typename SrcType::uniform_runtime_const_nomemspace_type,
679 template <
class DstType,
class SrcType,
class... Args>
680 struct CommonSubview {
681 using dst_subview_type =
typename Kokkos::Subview<DstType, Args...>;
682 using src_subview_type =
typename Kokkos::Subview<SrcType, Args...>;
683 dst_subview_type dst_sub;
684 src_subview_type src_sub;
685 CommonSubview(
const DstType& dst,
const SrcType& src,
const Args&... args)
686 : dst_sub(dst, args...), src_sub(src, args...) {}
689 template <
class DstType,
class SrcType,
int Rank = DstType::rank>
692 template <
class DstType,
class SrcType>
693 struct ViewRemap<DstType, SrcType, 1> {
696 template <
typename... OptExecSpace>
697 ViewRemap(
const DstType& dst,
const SrcType& src,
698 const OptExecSpace&... exec_space) {
700 sizeof...(OptExecSpace) <= 1,
701 "OptExecSpace must be either empty or be an execution space!");
703 if (dst.extent(0) == src.extent(0)) {
704 view_copy(exec_space..., dst, src);
706 p_type ext0(0, std::min(dst.extent(0), src.extent(0)));
707 CommonSubview common_subview(dst, src, ext0);
708 view_copy(exec_space..., common_subview.dst_sub, common_subview.src_sub);
713 template <
class DstType,
class SrcType, std::size_t... I>
714 auto create_common_subview_first_and_last_match(
const DstType& dst,
716 std::index_sequence<I...>) {
718 CommonSubview common_subview(
719 dst, src, Kokkos::ALL,
720 (p_type(0, std::min(dst.extent(I + 1), src.extent(I + 1))))...,
722 return common_subview;
725 template <
class DstType,
class SrcType, std::size_t... I>
726 auto create_common_subview_first_match(
const DstType& dst,
const SrcType& src,
727 std::index_sequence<I...>) {
729 CommonSubview common_subview(
730 dst, src, Kokkos::ALL,
731 (p_type(0, std::min(dst.extent(I + 1), src.extent(I + 1))))...);
732 return common_subview;
735 template <
class DstType,
class SrcType, std::size_t... I>
736 auto create_common_subview_last_match(
const DstType& dst,
const SrcType& src,
737 std::index_sequence<I...>) {
739 CommonSubview common_subview(
740 dst, src, (p_type(0, std::min(dst.extent(I), src.extent(I))))...,
742 return common_subview;
745 template <
class DstType,
class SrcType, std::size_t... I>
746 auto create_common_subview_no_match(
const DstType& dst,
const SrcType& src,
747 std::index_sequence<I...>) {
749 CommonSubview common_subview(
750 dst, src, (p_type(0, std::min(dst.extent(I), src.extent(I))))...);
751 return common_subview;
754 template <
class DstType,
class SrcType,
int Rank>
758 template <
typename... OptExecSpace>
759 ViewRemap(
const DstType& dst,
const SrcType& src,
760 const OptExecSpace&... exec_space) {
762 sizeof...(OptExecSpace) <= 1,
763 "OptExecSpace must be either empty or be an execution space!");
765 if (dst.extent(0) == src.extent(0)) {
766 if (dst.extent(Rank - 1) == src.extent(Rank - 1)) {
767 if constexpr (Rank < 3)
768 view_copy(exec_space..., dst, src);
770 auto common_subview = create_common_subview_first_and_last_match(
771 dst, src, std::make_index_sequence<Rank - 2>{});
772 view_copy(exec_space..., common_subview.dst_sub,
773 common_subview.src_sub);
776 auto common_subview = create_common_subview_first_match(
777 dst, src, std::make_index_sequence<Rank - 1>{});
778 view_copy(exec_space..., common_subview.dst_sub,
779 common_subview.src_sub);
782 if (dst.extent(Rank - 1) == src.extent(Rank - 1)) {
783 auto common_subview = create_common_subview_last_match(
784 dst, src, std::make_index_sequence<Rank - 1>{});
785 view_copy(exec_space..., common_subview.dst_sub,
786 common_subview.src_sub);
788 auto common_subview = create_common_subview_no_match(
789 dst, src, std::make_index_sequence<Rank>{});
790 view_copy(exec_space..., common_subview.dst_sub,
791 common_subview.src_sub);
797 template <
typename ExecutionSpace,
class DT,
class... DP>
798 inline void contiguous_fill(
799 const ExecutionSpace& exec_space,
const View<DT, DP...>& dst,
800 typename ViewTraits<DT, DP...>::const_value_type& value) {
801 using ViewType =
View<DT, DP...>;
802 using ViewTypeFlat = Kokkos::View<
804 Kokkos::Device<
typename ViewType::execution_space,
805 std::conditional_t<ViewType::rank == 0,
806 typename ViewType::memory_space,
807 Kokkos::AnonymousSpace>>,
808 Kokkos::MemoryTraits<>>;
810 ViewTypeFlat dst_flat(dst.data(), dst.size());
811 if (dst.span() <
static_cast<size_t>(std::numeric_limits<int>::max())) {
812 Kokkos::Impl::ViewFill<ViewTypeFlat, Kokkos::LayoutRight, ExecutionSpace,
813 ViewTypeFlat::rank,
int>(dst_flat, value,
816 Kokkos::Impl::ViewFill<ViewTypeFlat, Kokkos::LayoutRight, ExecutionSpace,
817 ViewTypeFlat::rank, int64_t>(dst_flat, value,
822 template <
typename ExecutionSpace>
824 ZeroMemset(
const ExecutionSpace& exec_space,
void* dst,
size_t cnt) {
827 Kokkos::View<std::byte*, ExecutionSpace, Kokkos::MemoryUnmanaged>(
828 static_cast<std::byte*>(dst), cnt),
836 template <
typename T>
837 bool has_all_zero_bits(
const T& value) {
838 static_assert(std::is_trivially_copyable_v<T>);
840 if constexpr (std::is_scalar_v<T>) {
844 KOKKOS_IMPL_DISABLE_UNREACHABLE_WARNINGS_PUSH()
845 if constexpr (std::is_standard_layout_v<T> &&
846 std::has_unique_object_representations_v<T>) {
847 constexpr std::byte all_zeroes[
sizeof(T)] = {};
848 return std::memcmp(&value, all_zeroes,
sizeof(T)) == 0;
852 KOKKOS_IMPL_DISABLE_UNREACHABLE_WARNINGS_POP()
855 template <
typename ExecutionSpace,
class DT,
class... DP>
856 inline std::enable_if_t<
857 std::is_trivially_copyable_v<
typename ViewTraits<DT, DP...>::value_type> &&
858 !ViewTraits<DT, DP...>::impl_is_customized>
859 contiguous_fill_or_memset(
860 const ExecutionSpace& exec_space,
const View<DT, DP...>& dst,
861 typename ViewTraits<DT, DP...>::const_value_type& value) {
863 if (has_all_zero_bits(value)
864 #ifdef KOKKOS_ENABLE_OPENMP
865 && !std::is_same_v<ExecutionSpace, Kokkos::OpenMP>
868 ZeroMemset(exec_space, dst.data(),
869 dst.size() *
sizeof(
typename ViewTraits<DT, DP...>::value_type));
871 contiguous_fill(exec_space, dst, value);
874 template <
typename ExecutionSpace,
class DT,
class... DP>
875 inline std::enable_if_t<
876 !std::is_trivially_copyable_v<
typename ViewTraits<DT, DP...>::value_type> ||
877 ViewTraits<DT, DP...>::impl_is_customized>
878 contiguous_fill_or_memset(
879 const ExecutionSpace& exec_space,
const View<DT, DP...>& dst,
880 typename ViewTraits<DT, DP...>::const_value_type& value) {
881 contiguous_fill(exec_space, dst, value);
884 template <
class DT,
class... DP>
885 void contiguous_fill_or_memset(
886 const View<DT, DP...>& dst,
887 typename ViewTraits<DT, DP...>::const_value_type& value) {
888 using ViewType =
View<DT, DP...>;
889 using exec_space_type =
typename ViewType::execution_space;
891 contiguous_fill_or_memset(exec_space_type(), dst, value);
896 template <
class DT,
class... DP>
897 inline void deep_copy(
898 const View<DT, DP...>& dst,
899 typename ViewTraits<DT, DP...>::const_value_type& value,
900 std::enable_if_t<std::is_same_v<
typename ViewTraits<DT, DP...>::specialize,
902 using ViewType =
View<DT, DP...>;
903 using exec_space_type =
typename ViewType::execution_space;
905 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy !=
nullptr) {
906 Kokkos::Profiling::beginDeepCopy(
907 Kokkos::Profiling::make_space_handle(ViewType::memory_space::name()),
908 dst.label(), dst.data(),
910 "Scalar", &value, dst.span() *
sizeof(
typename ViewType::value_type));
913 if (dst.data() ==
nullptr) {
915 "Kokkos::deep_copy: scalar copy, fence because destination is null");
916 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
917 Kokkos::Profiling::endDeepCopy();
922 Kokkos::fence(
"Kokkos::deep_copy: scalar copy, pre copy fence");
923 static_assert(std::is_same_v<
typename ViewType::non_const_value_type,
924 typename ViewType::value_type>,
925 "deep_copy requires non-const type");
929 if (dst.span_is_contiguous() && !ViewType::traits::impl_is_customized) {
930 Impl::contiguous_fill_or_memset(dst, value);
931 Kokkos::fence(
"Kokkos::deep_copy: scalar copy, post copy fence");
932 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
933 Kokkos::Profiling::endDeepCopy();
939 int64_t strides[ViewType::rank + 1];
941 Kokkos::Iterate iterate;
942 if (std::is_same_v<typename ViewType::array_layout, Kokkos::LayoutRight>) {
943 iterate = Kokkos::Iterate::Right;
944 }
else if (std::is_same_v<
typename ViewType::array_layout,
946 iterate = Kokkos::Iterate::Left;
947 }
else if (std::is_same_v<
typename ViewType::array_layout,
949 if (strides[0] > strides[ViewType::rank > 0 ? ViewType::rank - 1 : 0])
950 iterate = Kokkos::Iterate::Right;
952 iterate = Kokkos::Iterate::Left;
954 if (std::is_same_v<
typename ViewType::execution_space::array_layout,
955 Kokkos::LayoutRight>)
956 iterate = Kokkos::Iterate::Right;
958 iterate = Kokkos::Iterate::Left;
963 using ViewTypeUniform =
964 std::conditional_t<ViewType::rank == 0,
965 typename ViewType::uniform_runtime_type,
966 typename ViewType::uniform_runtime_nomemspace_type>;
967 if (dst.span() >
static_cast<size_t>(std::numeric_limits<int>::max())) {
968 if (iterate == Kokkos::Iterate::Right)
969 Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutRight,
970 exec_space_type, ViewType::rank, int64_t>(
971 dst, value, exec_space_type());
974 exec_space_type, ViewType::rank, int64_t>(
975 dst, value, exec_space_type());
977 if (iterate == Kokkos::Iterate::Right)
978 Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutRight,
979 exec_space_type, ViewType::rank,
int>(
980 dst, value, exec_space_type());
983 exec_space_type, ViewType::rank,
int>(
984 dst, value, exec_space_type());
986 Kokkos::fence(
"Kokkos::deep_copy: scalar copy, post copy fence");
988 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
989 Kokkos::Profiling::endDeepCopy();
994 template <
class ST,
class... SP>
995 inline void deep_copy(
996 typename ViewTraits<ST, SP...>::non_const_value_type& dst,
997 const View<ST, SP...>& src,
998 std::enable_if_t<std::is_same_v<
typename ViewTraits<ST, SP...>::specialize,
1000 using src_traits = ViewTraits<ST, SP...>;
1001 using src_memory_space =
typename src_traits::memory_space;
1003 static_assert(src_traits::rank == 0,
1004 "ERROR: Non-rank-zero view in deep_copy( value , View )");
1006 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy !=
nullptr) {
1007 Kokkos::Profiling::beginDeepCopy(
1010 Kokkos::Profiling::make_space_handle(src_memory_space::name()),
1011 src.label(), src.data(),
1012 src.span() *
sizeof(
typename src_traits::value_type));
1015 if (src.data() ==
nullptr) {
1016 Kokkos::fence(
"Kokkos::deep_copy: copy into scalar, src is null");
1018 Kokkos::fence(
"Kokkos::deep_copy: copy into scalar, pre copy fence");
1019 Kokkos::Impl::DeepCopy<HostSpace, src_memory_space>(&dst, src.data(),
1021 Kokkos::fence(
"Kokkos::deep_copy: copy into scalar, post copy fence");
1024 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
1025 Kokkos::Profiling::endDeepCopy();
1031 template <
class DT,
class... DP,
class ST,
class... SP>
1032 inline void deep_copy(
1033 const View<DT, DP...>& dst,
const View<ST, SP...>& src,
1035 (std::is_void_v<
typename ViewTraits<DT, DP...>::specialize> &&
1036 std::is_void_v<
typename ViewTraits<ST, SP...>::specialize> &&
1037 (
unsigned(ViewTraits<DT, DP...>::rank) ==
unsigned(0) &&
1038 unsigned(ViewTraits<ST, SP...>::rank) ==
unsigned(0)))>* =
nullptr) {
1039 using dst_type =
View<DT, DP...>;
1040 using src_type =
View<ST, SP...>;
1042 using value_type =
typename dst_type::value_type;
1043 using dst_memory_space =
typename dst_type::memory_space;
1044 using src_memory_space =
typename src_type::memory_space;
1046 static_assert(std::is_same_v<
typename dst_type::value_type,
1047 typename src_type::non_const_value_type>,
1048 "deep_copy requires matching non-const destination type");
1050 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy !=
nullptr) {
1051 Kokkos::Profiling::beginDeepCopy(
1052 Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
1053 dst.label(), dst.data(),
1054 Kokkos::Profiling::make_space_handle(src_memory_space::name()),
1055 src.label(), src.data(),
1056 src.span() *
sizeof(
typename dst_type::value_type));
1059 if (dst.data() ==
nullptr && src.data() ==
nullptr) {
1061 "Kokkos::deep_copy: scalar to scalar copy, both pointers null");
1062 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
1063 Kokkos::Profiling::endDeepCopy();
1068 Kokkos::fence(
"Kokkos::deep_copy: scalar to scalar copy, pre copy fence");
1069 if (dst.data() != src.data()) {
1070 Kokkos::Impl::DeepCopy<dst_memory_space, src_memory_space>(
1071 dst.data(), src.data(),
sizeof(value_type));
1072 Kokkos::fence(
"Kokkos::deep_copy: scalar to scalar copy, post copy fence");
1074 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
1075 Kokkos::Profiling::endDeepCopy();
1083 template <
class DT,
class... DP,
class ST,
class... SP>
1084 inline void deep_copy(
1085 const View<DT, DP...>& dst,
const View<ST, SP...>& src,
1087 (std::is_void_v<
typename ViewTraits<DT, DP...>::specialize> &&
1088 std::is_void_v<
typename ViewTraits<ST, SP...>::specialize> &&
1089 (
unsigned(ViewTraits<DT, DP...>::rank) != 0 ||
1090 unsigned(ViewTraits<ST, SP...>::rank) != 0))>* =
nullptr) {
1091 using dst_type =
View<DT, DP...>;
1092 using src_type =
View<ST, SP...>;
1093 using dst_memory_space =
typename dst_type::memory_space;
1094 using src_memory_space =
typename src_type::memory_space;
1095 using dst_ptr_type = decltype(dst.data());
1096 using src_ptr_type = decltype(src.data());
1098 static_assert(std::is_same_v<
typename dst_type::value_type,
1099 typename dst_type::non_const_value_type>,
1100 "deep_copy requires non-const destination type");
1102 static_assert((
unsigned(dst_type::rank) ==
unsigned(src_type::rank)),
1103 "deep_copy requires Views of equal rank");
1105 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy !=
nullptr) {
1106 Kokkos::Profiling::beginDeepCopy(
1107 Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
1108 dst.label(), dst.data(),
1109 Kokkos::Profiling::make_space_handle(src_memory_space::name()),
1110 src.label(), src.data(),
1111 src.span() *
sizeof(
typename dst_type::value_type));
1114 if (dst.data() ==
nullptr || src.data() ==
nullptr) {
1116 if ((src.extent(0) != dst.extent(0)) || (src.extent(1) != dst.extent(1)) ||
1117 (src.extent(2) != dst.extent(2)) || (src.extent(3) != dst.extent(3)) ||
1118 (src.extent(4) != dst.extent(4)) || (src.extent(5) != dst.extent(5)) ||
1119 (src.extent(6) != dst.extent(6)) || (src.extent(7) != dst.extent(7))) {
1120 std::string message(
1121 "Deprecation Error: Kokkos::deep_copy extents of views don't "
1123 message += dst.label();
1125 message += std::to_string(dst.extent(0));
1126 for (
size_t r = 1; r < dst_type::rank; r++) {
1128 message += std::to_string(dst.extent(r));
1131 message += src.label();
1133 message += std::to_string(src.extent(0));
1134 for (
size_t r = 1; r < src_type::rank; r++) {
1136 message += std::to_string(src.extent(r));
1140 Kokkos::Impl::throw_runtime_exception(message);
1143 "Kokkos::deep_copy: copy between contiguous views, fence due to null "
1145 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
1146 Kokkos::Profiling::endDeepCopy();
1152 dst_ptr_type dst_start = dst.data();
1153 src_ptr_type src_start = src.data();
1154 #ifndef KOKKOS_ENABLE_IMPL_VIEW_LEGACY
1155 dst_ptr_type dst_end = dst.data() + allocation_size_from_mapping_and_accessor(
1156 dst.mapping(), dst.accessor());
1157 src_ptr_type src_end = src.data() + allocation_size_from_mapping_and_accessor(
1158 src.mapping(), src.accessor());
1160 dst_ptr_type dst_end = dst.data() + dst.span();
1161 src_ptr_type src_end = src.data() + src.span();
1163 if (((std::ptrdiff_t)dst_start == (std::ptrdiff_t)src_start) &&
1164 ((std::ptrdiff_t)dst_end == (std::ptrdiff_t)src_end) &&
1165 (dst.span_is_contiguous() && src.span_is_contiguous())) {
1167 "Kokkos::deep_copy: copy between contiguous views, fence due to same "
1169 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
1170 Kokkos::Profiling::endDeepCopy();
1175 if ((((std::ptrdiff_t)dst_start < (std::ptrdiff_t)src_end) &&
1176 ((std::ptrdiff_t)dst_end > (std::ptrdiff_t)src_start)) &&
1177 ((dst.span_is_contiguous() && src.span_is_contiguous()))) {
1178 std::string message(
"Error: Kokkos::deep_copy of overlapping views: ");
1179 message += dst.label();
1181 message += std::to_string((std::ptrdiff_t)dst_start);
1183 message += std::to_string((std::ptrdiff_t)dst_end);
1185 message += src.label();
1187 message += std::to_string((std::ptrdiff_t)src_start);
1189 message += std::to_string((std::ptrdiff_t)src_end);
1191 Kokkos::Impl::throw_runtime_exception(message);
1195 if ((src.extent(0) != dst.extent(0)) || (src.extent(1) != dst.extent(1)) ||
1196 (src.extent(2) != dst.extent(2)) || (src.extent(3) != dst.extent(3)) ||
1197 (src.extent(4) != dst.extent(4)) || (src.extent(5) != dst.extent(5)) ||
1198 (src.extent(6) != dst.extent(6)) || (src.extent(7) != dst.extent(7))) {
1199 std::string message(
1200 "Deprecation Error: Kokkos::deep_copy extents of views don't match: ");
1201 message += dst.label();
1203 message += std::to_string(dst.extent(0));
1204 for (
size_t r = 1; r < dst_type::rank; r++) {
1206 message += std::to_string(dst.extent(r));
1209 message += src.label();
1211 message += std::to_string(src.extent(0));
1212 for (
size_t r = 1; r < src_type::rank; r++) {
1214 message += std::to_string(src.extent(r));
1218 Kokkos::Impl::throw_runtime_exception(message);
1224 if (std::is_same_v<
typename dst_type::value_type,
1225 typename src_type::non_const_value_type> &&
1226 (std::is_same_v<
typename dst_type::array_layout,
1227 typename src_type::array_layout> ||
1228 (dst_type::rank == 1 && src_type::rank == 1)) &&
1229 dst.span_is_contiguous() && src.span_is_contiguous() &&
1230 ((dst_type::rank < 1) || (dst.stride_0() == src.stride_0())) &&
1231 ((dst_type::rank < 2) || (dst.stride_1() == src.stride_1())) &&
1232 ((dst_type::rank < 3) || (dst.stride_2() == src.stride_2())) &&
1233 ((dst_type::rank < 4) || (dst.stride_3() == src.stride_3())) &&
1234 ((dst_type::rank < 5) || (dst.stride_4() == src.stride_4())) &&
1235 ((dst_type::rank < 6) || (dst.stride_5() == src.stride_5())) &&
1236 ((dst_type::rank < 7) || (dst.stride_6() == src.stride_6())) &&
1237 ((dst_type::rank < 8) || (dst.stride_7() == src.stride_7()))) {
1238 #ifndef KOKKOS_ENABLE_IMPL_VIEW_LEGACY
1239 const size_t nbytes = allocation_size_from_mapping_and_accessor(
1240 src.mapping(), src.accessor()) *
1241 sizeof(std::remove_pointer_t<dst_ptr_type>);
1243 const size_t nbytes =
sizeof(
typename dst_type::value_type) * dst.span();
1246 "Kokkos::deep_copy: copy between contiguous views, pre view equality "
1248 if ((
void*)dst.data() != (
void*)src.data() && 0 < nbytes) {
1249 Kokkos::Impl::DeepCopy<dst_memory_space, src_memory_space>(
1250 dst.data(), src.data(), nbytes);
1252 "Kokkos::deep_copy: copy between contiguous views, post deep copy "
1257 "Kokkos::deep_copy: copy between contiguous views, pre copy fence");
1258 Impl::view_copy(dst, src);
1260 "Kokkos::deep_copy: copy between contiguous views, post copy fence");
1262 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
1263 Kokkos::Profiling::endDeepCopy();
1269 namespace Experimental {
1273 template <
class TeamType,
class DT,
class... DP,
class ST,
class... SP>
1274 void KOKKOS_INLINE_FUNCTION
1275 local_deep_copy_contiguous(
const TeamType& team,
const View<DT, DP...>& dst,
1276 const View<ST, SP...>& src) {
1277 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, src.span()),
1278 [&](
const int& i) { dst.data()[i] = src.data()[i]; });
1281 template <
class DT,
class... DP,
class ST,
class... SP>
1282 void KOKKOS_INLINE_FUNCTION local_deep_copy_contiguous(
1283 const View<DT, DP...>& dst,
const View<ST, SP...>& src) {
1284 for (
size_t i = 0; i < src.span(); ++i) {
1285 dst.data()[i] = src.data()[i];
1289 template <
class TeamType,
class DT,
class... DP,
class ST,
class... SP>
1290 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1291 const TeamType& team,
const View<DT, DP...>& dst,
1292 const View<ST, SP...>& src,
1293 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 1 &&
1294 unsigned(ViewTraits<ST, SP...>::rank) == 1)>* =
nullptr) {
1295 if (dst.data() ==
nullptr) {
1299 const size_t N = dst.extent(0);
1301 team.team_barrier();
1302 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N),
1303 [&](
const int& i) { dst(i) = src(i); });
1304 team.team_barrier();
1307 template <
class TeamType,
class DT,
class... DP,
class ST,
class... SP>
1308 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1309 const TeamType& team,
const View<DT, DP...>& dst,
1310 const View<ST, SP...>& src,
1311 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 2 &&
1312 unsigned(ViewTraits<ST, SP...>::rank) == 2)>* =
nullptr) {
1313 if (dst.data() ==
nullptr) {
1317 const size_t N = dst.extent(0) * dst.extent(1);
1319 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1320 team.team_barrier();
1321 local_deep_copy_contiguous(team, dst, src);
1322 team.team_barrier();
1324 team.team_barrier();
1325 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](
const int& i) {
1326 int i0 = i % dst.extent(0);
1327 int i1 = i / dst.extent(0);
1328 dst(i0, i1) = src(i0, i1);
1330 team.team_barrier();
1334 template <
class TeamType,
class DT,
class... DP,
class ST,
class... SP>
1335 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1336 const TeamType& team,
const View<DT, DP...>& dst,
1337 const View<ST, SP...>& src,
1338 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 3 &&
1339 unsigned(ViewTraits<ST, SP...>::rank) == 3)>* =
nullptr) {
1340 if (dst.data() ==
nullptr) {
1344 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2);
1346 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1347 team.team_barrier();
1348 local_deep_copy_contiguous(team, dst, src);
1349 team.team_barrier();
1351 team.team_barrier();
1352 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](
const int& i) {
1353 int i0 = i % dst.extent(0);
1354 int itmp = i / dst.extent(0);
1355 int i1 = itmp % dst.extent(1);
1356 int i2 = itmp / dst.extent(1);
1357 dst(i0, i1, i2) = src(i0, i1, i2);
1359 team.team_barrier();
1363 template <
class TeamType,
class DT,
class... DP,
class ST,
class... SP>
1364 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1365 const TeamType& team,
const View<DT, DP...>& dst,
1366 const View<ST, SP...>& src,
1367 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 4 &&
1368 unsigned(ViewTraits<ST, SP...>::rank) == 4)>* =
nullptr) {
1369 if (dst.data() ==
nullptr) {
1374 dst.extent(0) * dst.extent(1) * dst.extent(2) * dst.extent(3);
1376 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1377 team.team_barrier();
1378 local_deep_copy_contiguous(team, dst, src);
1379 team.team_barrier();
1381 team.team_barrier();
1382 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](
const int& i) {
1383 int i0 = i % dst.extent(0);
1384 int itmp = i / dst.extent(0);
1385 int i1 = itmp % dst.extent(1);
1386 itmp = itmp / dst.extent(1);
1387 int i2 = itmp % dst.extent(2);
1388 int i3 = itmp / dst.extent(2);
1389 dst(i0, i1, i2, i3) = src(i0, i1, i2, i3);
1391 team.team_barrier();
1395 template <
class TeamType,
class DT,
class... DP,
class ST,
class... SP>
1396 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1397 const TeamType& team,
const View<DT, DP...>& dst,
1398 const View<ST, SP...>& src,
1399 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 5 &&
1400 unsigned(ViewTraits<ST, SP...>::rank) == 5)>* =
nullptr) {
1401 if (dst.data() ==
nullptr) {
1405 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1406 dst.extent(3) * dst.extent(4);
1408 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1409 team.team_barrier();
1410 local_deep_copy_contiguous(team, dst, src);
1411 team.team_barrier();
1413 team.team_barrier();
1414 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](
const int& i) {
1415 int i0 = i % dst.extent(0);
1416 int itmp = i / dst.extent(0);
1417 int i1 = itmp % dst.extent(1);
1418 itmp = itmp / dst.extent(1);
1419 int i2 = itmp % dst.extent(2);
1420 itmp = itmp / dst.extent(2);
1421 int i3 = itmp % dst.extent(3);
1422 int i4 = itmp / dst.extent(3);
1423 dst(i0, i1, i2, i3, i4) = src(i0, i1, i2, i3, i4);
1425 team.team_barrier();
1429 template <
class TeamType,
class DT,
class... DP,
class ST,
class... SP>
1430 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1431 const TeamType& team,
const View<DT, DP...>& dst,
1432 const View<ST, SP...>& src,
1433 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 6 &&
1434 unsigned(ViewTraits<ST, SP...>::rank) == 6)>* =
nullptr) {
1435 if (dst.data() ==
nullptr) {
1439 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1440 dst.extent(3) * dst.extent(4) * dst.extent(5);
1442 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1443 team.team_barrier();
1444 local_deep_copy_contiguous(team, dst, src);
1445 team.team_barrier();
1447 team.team_barrier();
1448 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](
const int& i) {
1449 int i0 = i % dst.extent(0);
1450 int itmp = i / dst.extent(0);
1451 int i1 = itmp % dst.extent(1);
1452 itmp = itmp / dst.extent(1);
1453 int i2 = itmp % dst.extent(2);
1454 itmp = itmp / dst.extent(2);
1455 int i3 = itmp % dst.extent(3);
1456 itmp = itmp / dst.extent(3);
1457 int i4 = itmp % dst.extent(4);
1458 int i5 = itmp / dst.extent(4);
1459 dst(i0, i1, i2, i3, i4, i5) = src(i0, i1, i2, i3, i4, i5);
1461 team.team_barrier();
1465 template <
class TeamType,
class DT,
class... DP,
class ST,
class... SP>
1466 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1467 const TeamType& team,
const View<DT, DP...>& dst,
1468 const View<ST, SP...>& src,
1469 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 7 &&
1470 unsigned(ViewTraits<ST, SP...>::rank) == 7)>* =
nullptr) {
1471 if (dst.data() ==
nullptr) {
1475 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1476 dst.extent(3) * dst.extent(4) * dst.extent(5) *
1479 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1480 team.team_barrier();
1481 local_deep_copy_contiguous(team, dst, src);
1482 team.team_barrier();
1484 team.team_barrier();
1485 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](
const int& i) {
1486 int i0 = i % dst.extent(0);
1487 int itmp = i / dst.extent(0);
1488 int i1 = itmp % dst.extent(1);
1489 itmp = itmp / dst.extent(1);
1490 int i2 = itmp % dst.extent(2);
1491 itmp = itmp / dst.extent(2);
1492 int i3 = itmp % dst.extent(3);
1493 itmp = itmp / dst.extent(3);
1494 int i4 = itmp % dst.extent(4);
1495 itmp = itmp / dst.extent(4);
1496 int i5 = itmp % dst.extent(5);
1497 int i6 = itmp / dst.extent(5);
1498 dst(i0, i1, i2, i3, i4, i5, i6) = src(i0, i1, i2, i3, i4, i5, i6);
1500 team.team_barrier();
1504 template <
class DT,
class... DP,
class ST,
class... SP>
1505 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1506 const View<DT, DP...>& dst,
const View<ST, SP...>& src,
1507 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 1 &&
1508 unsigned(ViewTraits<ST, SP...>::rank) == 1)>* =
nullptr) {
1509 if (dst.data() ==
nullptr) {
1513 const size_t N = dst.extent(0);
1515 for (
size_t i = 0; i < N; ++i) {
1520 template <
class DT,
class... DP,
class ST,
class... SP>
1521 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1522 const View<DT, DP...>& dst,
const View<ST, SP...>& src,
1523 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 2 &&
1524 unsigned(ViewTraits<ST, SP...>::rank) == 2)>* =
nullptr) {
1525 if (dst.data() ==
nullptr) {
1529 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1530 local_deep_copy_contiguous(dst, src);
1532 for (
size_t i0 = 0; i0 < dst.extent(0); ++i0)
1533 for (
size_t i1 = 0; i1 < dst.extent(1); ++i1) dst(i0, i1) = src(i0, i1);
1537 template <
class DT,
class... DP,
class ST,
class... SP>
1538 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1539 const View<DT, DP...>& dst,
const View<ST, SP...>& src,
1540 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 3 &&
1541 unsigned(ViewTraits<ST, SP...>::rank) == 3)>* =
nullptr) {
1542 if (dst.data() ==
nullptr) {
1546 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1547 local_deep_copy_contiguous(dst, src);
1549 for (
size_t i0 = 0; i0 < dst.extent(0); ++i0)
1550 for (
size_t i1 = 0; i1 < dst.extent(1); ++i1)
1551 for (
size_t i2 = 0; i2 < dst.extent(2); ++i2)
1552 dst(i0, i1, i2) = src(i0, i1, i2);
1556 template <
class DT,
class... DP,
class ST,
class... SP>
1557 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1558 const View<DT, DP...>& dst,
const View<ST, SP...>& src,
1559 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 4 &&
1560 unsigned(ViewTraits<ST, SP...>::rank) == 4)>* =
nullptr) {
1561 if (dst.data() ==
nullptr) {
1565 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1566 local_deep_copy_contiguous(dst, src);
1568 for (
size_t i0 = 0; i0 < dst.extent(0); ++i0)
1569 for (
size_t i1 = 0; i1 < dst.extent(1); ++i1)
1570 for (
size_t i2 = 0; i2 < dst.extent(2); ++i2)
1571 for (
size_t i3 = 0; i3 < dst.extent(3); ++i3)
1572 dst(i0, i1, i2, i3) = src(i0, i1, i2, i3);
1576 template <
class DT,
class... DP,
class ST,
class... SP>
1577 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1578 const View<DT, DP...>& dst,
const View<ST, SP...>& src,
1579 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 5 &&
1580 unsigned(ViewTraits<ST, SP...>::rank) == 5)>* =
nullptr) {
1581 if (dst.data() ==
nullptr) {
1585 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1586 local_deep_copy_contiguous(dst, src);
1588 for (
size_t i0 = 0; i0 < dst.extent(0); ++i0)
1589 for (
size_t i1 = 0; i1 < dst.extent(1); ++i1)
1590 for (
size_t i2 = 0; i2 < dst.extent(2); ++i2)
1591 for (
size_t i3 = 0; i3 < dst.extent(3); ++i3)
1592 for (
size_t i4 = 0; i4 < dst.extent(4); ++i4)
1593 dst(i0, i1, i2, i3, i4) = src(i0, i1, i2, i3, i4);
1597 template <
class DT,
class... DP,
class ST,
class... SP>
1598 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1599 const View<DT, DP...>& dst,
const View<ST, SP...>& src,
1600 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 6 &&
1601 unsigned(ViewTraits<ST, SP...>::rank) == 6)>* =
nullptr) {
1602 if (dst.data() ==
nullptr) {
1606 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1607 local_deep_copy_contiguous(dst, src);
1609 for (
size_t i0 = 0; i0 < dst.extent(0); ++i0)
1610 for (
size_t i1 = 0; i1 < dst.extent(1); ++i1)
1611 for (
size_t i2 = 0; i2 < dst.extent(2); ++i2)
1612 for (
size_t i3 = 0; i3 < dst.extent(3); ++i3)
1613 for (
size_t i4 = 0; i4 < dst.extent(4); ++i4)
1614 for (
size_t i5 = 0; i5 < dst.extent(5); ++i5)
1615 dst(i0, i1, i2, i3, i4, i5) = src(i0, i1, i2, i3, i4, i5);
1619 template <
class DT,
class... DP,
class ST,
class... SP>
1620 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1621 const View<DT, DP...>& dst,
const View<ST, SP...>& src,
1622 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 7 &&
1623 unsigned(ViewTraits<ST, SP...>::rank) == 7)>* =
nullptr) {
1624 if (dst.data() ==
nullptr) {
1628 if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1629 local_deep_copy_contiguous(dst, src);
1631 for (
size_t i0 = 0; i0 < dst.extent(0); ++i0)
1632 for (
size_t i1 = 0; i1 < dst.extent(1); ++i1)
1633 for (
size_t i2 = 0; i2 < dst.extent(2); ++i2)
1634 for (
size_t i3 = 0; i3 < dst.extent(3); ++i3)
1635 for (
size_t i4 = 0; i4 < dst.extent(4); ++i4)
1636 for (
size_t i5 = 0; i5 < dst.extent(5); ++i5)
1637 for (
size_t i6 = 0; i6 < dst.extent(6); ++i6)
1638 dst(i0, i1, i2, i3, i4, i5, i6) =
1639 src(i0, i1, i2, i3, i4, i5, i6);
1645 template <
class TeamType,
class DT,
class... DP>
1646 void KOKKOS_INLINE_FUNCTION local_deep_copy_contiguous(
1647 const TeamType& team,
const View<DT, DP...>& dst,
1648 typename ViewTraits<DT, DP...>::const_value_type& value,
1649 std::enable_if_t<std::is_same_v<
typename ViewTraits<DT, DP...>::specialize,
1650 void>>* =
nullptr) {
1651 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, dst.span()),
1652 [&](
const int& i) { dst.data()[i] = value; });
1655 template <
class DT,
class... DP>
1656 void KOKKOS_INLINE_FUNCTION local_deep_copy_contiguous(
1657 const View<DT, DP...>& dst,
1658 typename ViewTraits<DT, DP...>::const_value_type& value,
1659 std::enable_if_t<std::is_same_v<
typename ViewTraits<DT, DP...>::specialize,
1660 void>>* =
nullptr) {
1661 for (
size_t i = 0; i < dst.span(); ++i) {
1662 dst.data()[i] = value;
1666 template <
class TeamType,
class DT,
class... DP>
1667 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1668 const TeamType& team,
const View<DT, DP...>& dst,
1669 typename ViewTraits<DT, DP...>::const_value_type& value,
1670 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 1)>* =
nullptr) {
1671 if (dst.data() ==
nullptr) {
1675 const size_t N = dst.extent(0);
1677 team.team_barrier();
1678 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N),
1679 [&](
const int& i) { dst(i) = value; });
1680 team.team_barrier();
1683 template <
class TeamType,
class DT,
class... DP>
1684 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1685 const TeamType& team,
const View<DT, DP...>& dst,
1686 typename ViewTraits<DT, DP...>::const_value_type& value,
1687 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 2)>* =
nullptr) {
1688 if (dst.data() ==
nullptr) {
1692 const size_t N = dst.extent(0) * dst.extent(1);
1694 if (dst.span_is_contiguous()) {
1697 if constexpr (std::is_same_v<decltype(dst.data()),
1698 typename View<DT, DP...>::element_type*>) {
1699 team.team_barrier();
1700 local_deep_copy_contiguous(team, dst, value);
1701 team.team_barrier();
1705 team.team_barrier();
1706 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](
const int& i) {
1707 int i0 = i % dst.extent(0);
1708 int i1 = i / dst.extent(0);
1709 dst(i0, i1) = value;
1711 team.team_barrier();
1714 template <
class TeamType,
class DT,
class... DP>
1715 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1716 const TeamType& team,
const View<DT, DP...>& dst,
1717 typename ViewTraits<DT, DP...>::const_value_type& value,
1718 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 3)>* =
nullptr) {
1719 if (dst.data() ==
nullptr) {
1723 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2);
1725 if (dst.span_is_contiguous()) {
1728 if constexpr (std::is_same_v<decltype(dst.data()),
1729 typename View<DT, DP...>::element_type*>) {
1730 team.team_barrier();
1731 local_deep_copy_contiguous(team, dst, value);
1732 team.team_barrier();
1736 team.team_barrier();
1737 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](
const int& i) {
1738 int i0 = i % dst.extent(0);
1739 int itmp = i / dst.extent(0);
1740 int i1 = itmp % dst.extent(1);
1741 int i2 = itmp / dst.extent(1);
1742 dst(i0, i1, i2) = value;
1744 team.team_barrier();
1747 template <
class TeamType,
class DT,
class... DP>
1748 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1749 const TeamType& team,
const View<DT, DP...>& dst,
1750 typename ViewTraits<DT, DP...>::const_value_type& value,
1751 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 4)>* =
nullptr) {
1752 if (dst.data() ==
nullptr) {
1757 dst.extent(0) * dst.extent(1) * dst.extent(2) * dst.extent(3);
1759 if (dst.span_is_contiguous()) {
1762 if constexpr (std::is_same_v<decltype(dst.data()),
1763 typename View<DT, DP...>::element_type*>) {
1764 team.team_barrier();
1765 local_deep_copy_contiguous(team, dst, value);
1766 team.team_barrier();
1770 team.team_barrier();
1771 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](
const int& i) {
1772 int i0 = i % dst.extent(0);
1773 int itmp = i / dst.extent(0);
1774 int i1 = itmp % dst.extent(1);
1775 itmp = itmp / dst.extent(1);
1776 int i2 = itmp % dst.extent(2);
1777 int i3 = itmp / dst.extent(2);
1778 dst(i0, i1, i2, i3) = value;
1780 team.team_barrier();
1783 template <
class TeamType,
class DT,
class... DP>
1784 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1785 const TeamType& team,
const View<DT, DP...>& dst,
1786 typename ViewTraits<DT, DP...>::const_value_type& value,
1787 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 5)>* =
nullptr) {
1788 if (dst.data() ==
nullptr) {
1792 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1793 dst.extent(3) * dst.extent(4);
1795 if (dst.span_is_contiguous()) {
1798 if constexpr (std::is_same_v<decltype(dst.data()),
1799 typename View<DT, DP...>::element_type*>) {
1800 team.team_barrier();
1801 local_deep_copy_contiguous(team, dst, value);
1802 team.team_barrier();
1806 team.team_barrier();
1807 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](
const int& i) {
1808 int i0 = i % dst.extent(0);
1809 int itmp = i / dst.extent(0);
1810 int i1 = itmp % dst.extent(1);
1811 itmp = itmp / dst.extent(1);
1812 int i2 = itmp % dst.extent(2);
1813 itmp = itmp / dst.extent(2);
1814 int i3 = itmp % dst.extent(3);
1815 int i4 = itmp / dst.extent(3);
1816 dst(i0, i1, i2, i3, i4) = value;
1818 team.team_barrier();
1821 template <
class TeamType,
class DT,
class... DP>
1822 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1823 const TeamType& team,
const View<DT, DP...>& dst,
1824 typename ViewTraits<DT, DP...>::const_value_type& value,
1825 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 6)>* =
nullptr) {
1826 if (dst.data() ==
nullptr) {
1830 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1831 dst.extent(3) * dst.extent(4) * dst.extent(5);
1833 if (dst.span_is_contiguous()) {
1836 if constexpr (std::is_same_v<decltype(dst.data()),
1837 typename View<DT, DP...>::element_type*>) {
1838 team.team_barrier();
1839 local_deep_copy_contiguous(team, dst, value);
1840 team.team_barrier();
1844 team.team_barrier();
1845 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](
const int& i) {
1846 int i0 = i % dst.extent(0);
1847 int itmp = i / dst.extent(0);
1848 int i1 = itmp % dst.extent(1);
1849 itmp = itmp / dst.extent(1);
1850 int i2 = itmp % dst.extent(2);
1851 itmp = itmp / dst.extent(2);
1852 int i3 = itmp % dst.extent(3);
1853 itmp = itmp / dst.extent(3);
1854 int i4 = itmp % dst.extent(4);
1855 int i5 = itmp / dst.extent(4);
1856 dst(i0, i1, i2, i3, i4, i5) = value;
1858 team.team_barrier();
1861 template <
class TeamType,
class DT,
class... DP>
1862 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1863 const TeamType& team,
const View<DT, DP...>& dst,
1864 typename ViewTraits<DT, DP...>::const_value_type& value,
1865 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 7)>* =
nullptr) {
1866 if (dst.data() ==
nullptr) {
1870 const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1871 dst.extent(3) * dst.extent(4) * dst.extent(5) *
1874 if (dst.span_is_contiguous()) {
1877 if constexpr (std::is_same_v<decltype(dst.data()),
1878 typename View<DT, DP...>::element_type*>) {
1879 team.team_barrier();
1880 local_deep_copy_contiguous(team, dst, value);
1881 team.team_barrier();
1885 team.team_barrier();
1886 Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](
const int& i) {
1887 int i0 = i % dst.extent(0);
1888 int itmp = i / dst.extent(0);
1889 int i1 = itmp % dst.extent(1);
1890 itmp = itmp / dst.extent(1);
1891 int i2 = itmp % dst.extent(2);
1892 itmp = itmp / dst.extent(2);
1893 int i3 = itmp % dst.extent(3);
1894 itmp = itmp / dst.extent(3);
1895 int i4 = itmp % dst.extent(4);
1896 itmp = itmp / dst.extent(4);
1897 int i5 = itmp % dst.extent(5);
1898 int i6 = itmp / dst.extent(5);
1899 dst(i0, i1, i2, i3, i4, i5, i6) = value;
1901 team.team_barrier();
1904 template <
class DT,
class... DP>
1905 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1906 const View<DT, DP...>& dst,
1907 typename ViewTraits<DT, DP...>::const_value_type& value,
1908 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 1)>* =
nullptr) {
1909 if (dst.data() ==
nullptr) {
1913 const size_t N = dst.extent(0);
1915 for (
size_t i = 0; i < N; ++i) {
1920 template <
class DT,
class... DP>
1921 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1922 const View<DT, DP...>& dst,
1923 typename ViewTraits<DT, DP...>::const_value_type& value,
1924 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 2)>* =
nullptr) {
1925 if (dst.data() ==
nullptr) {
1929 if (dst.span_is_contiguous()) {
1932 if constexpr (std::is_same_v<decltype(dst.data()),
1933 typename View<DT, DP...>::element_type*>) {
1934 local_deep_copy_contiguous(dst, value);
1938 for (
size_t i0 = 0; i0 < dst.extent(0); ++i0)
1939 for (
size_t i1 = 0; i1 < dst.extent(1); ++i1) dst(i0, i1) = value;
1942 template <
class DT,
class... DP>
1943 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1944 const View<DT, DP...>& dst,
1945 typename ViewTraits<DT, DP...>::const_value_type& value,
1946 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 3)>* =
nullptr) {
1947 if (dst.data() ==
nullptr) {
1951 if (dst.span_is_contiguous()) {
1954 if constexpr (std::is_same_v<decltype(dst.data()),
1955 typename View<DT, DP...>::element_type*>) {
1956 local_deep_copy_contiguous(dst, value);
1960 for (
size_t i0 = 0; i0 < dst.extent(0); ++i0)
1961 for (
size_t i1 = 0; i1 < dst.extent(1); ++i1)
1962 for (
size_t i2 = 0; i2 < dst.extent(2); ++i2) dst(i0, i1, i2) = value;
1965 template <
class DT,
class... DP>
1966 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1967 const View<DT, DP...>& dst,
1968 typename ViewTraits<DT, DP...>::const_value_type& value,
1969 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 4)>* =
nullptr) {
1970 if (dst.data() ==
nullptr) {
1974 if (dst.span_is_contiguous()) {
1977 if constexpr (std::is_same_v<decltype(dst.data()),
1978 typename View<DT, DP...>::element_type*>) {
1979 local_deep_copy_contiguous(dst, value);
1983 for (
size_t i0 = 0; i0 < dst.extent(0); ++i0)
1984 for (
size_t i1 = 0; i1 < dst.extent(1); ++i1)
1985 for (
size_t i2 = 0; i2 < dst.extent(2); ++i2)
1986 for (
size_t i3 = 0; i3 < dst.extent(3); ++i3)
1987 dst(i0, i1, i2, i3) = value;
1990 template <
class DT,
class... DP>
1991 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1992 const View<DT, DP...>& dst,
1993 typename ViewTraits<DT, DP...>::const_value_type& value,
1994 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 5)>* =
nullptr) {
1995 if (dst.data() ==
nullptr) {
1999 if (dst.span_is_contiguous()) {
2002 if constexpr (std::is_same_v<decltype(dst.data()),
2003 typename View<DT, DP...>::element_type*>) {
2004 local_deep_copy_contiguous(dst, value);
2008 for (
size_t i0 = 0; i0 < dst.extent(0); ++i0)
2009 for (
size_t i1 = 0; i1 < dst.extent(1); ++i1)
2010 for (
size_t i2 = 0; i2 < dst.extent(2); ++i2)
2011 for (
size_t i3 = 0; i3 < dst.extent(3); ++i3)
2012 for (
size_t i4 = 0; i4 < dst.extent(4); ++i4)
2013 dst(i0, i1, i2, i3, i4) = value;
2016 template <
class DT,
class... DP>
2017 void KOKKOS_INLINE_FUNCTION local_deep_copy(
2018 const View<DT, DP...>& dst,
2019 typename ViewTraits<DT, DP...>::const_value_type& value,
2020 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 6)>* =
nullptr) {
2021 if (dst.data() ==
nullptr) {
2025 if (dst.span_is_contiguous()) {
2028 if constexpr (std::is_same_v<decltype(dst.data()),
2029 typename View<DT, DP...>::element_type*>) {
2030 local_deep_copy_contiguous(dst, value);
2034 for (
size_t i0 = 0; i0 < dst.extent(0); ++i0)
2035 for (
size_t i1 = 0; i1 < dst.extent(1); ++i1)
2036 for (
size_t i2 = 0; i2 < dst.extent(2); ++i2)
2037 for (
size_t i3 = 0; i3 < dst.extent(3); ++i3)
2038 for (
size_t i4 = 0; i4 < dst.extent(4); ++i4)
2039 for (
size_t i5 = 0; i5 < dst.extent(5); ++i5)
2040 dst(i0, i1, i2, i3, i4, i5) = value;
2043 template <
class DT,
class... DP>
2044 void KOKKOS_INLINE_FUNCTION local_deep_copy(
2045 const View<DT, DP...>& dst,
2046 typename ViewTraits<DT, DP...>::const_value_type& value,
2047 std::enable_if_t<(
unsigned(ViewTraits<DT, DP...>::rank) == 7)>* =
nullptr) {
2048 if (dst.data() ==
nullptr) {
2052 if (dst.span_is_contiguous()) {
2055 if constexpr (std::is_same_v<decltype(dst.data()),
2056 typename View<DT, DP...>::element_type*>) {
2057 local_deep_copy_contiguous(dst, value);
2061 for (
size_t i0 = 0; i0 < dst.extent(0); ++i0)
2062 for (
size_t i1 = 0; i1 < dst.extent(1); ++i1)
2063 for (
size_t i2 = 0; i2 < dst.extent(2); ++i2)
2064 for (
size_t i3 = 0; i3 < dst.extent(3); ++i3)
2065 for (
size_t i4 = 0; i4 < dst.extent(4); ++i4)
2066 for (
size_t i5 = 0; i5 < dst.extent(5); ++i5)
2067 for (
size_t i6 = 0; i6 < dst.extent(6); ++i6)
2068 dst(i0, i1, i2, i3, i4, i5, i6) = value;
2080 template <
class ExecSpace,
class DT,
class... DP>
2081 inline void deep_copy(
2082 const ExecSpace& space,
const View<DT, DP...>& dst,
2083 typename ViewTraits<DT, DP...>::const_value_type& value,
2085 Kokkos::is_execution_space<ExecSpace>::value &&
2086 std::is_void_v<
typename ViewTraits<DT, DP...>::specialize> &&
2088 memory_space>::accessible>* =
2090 using dst_traits = ViewTraits<DT, DP...>;
2091 static_assert(std::is_same_v<
typename dst_traits::non_const_value_type,
2092 typename dst_traits::value_type>,
2093 "deep_copy requires non-const type");
2094 using dst_memory_space =
typename dst_traits::memory_space;
2095 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy !=
nullptr) {
2096 Kokkos::Profiling::beginDeepCopy(
2097 Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
2098 dst.label(), dst.data(),
2100 "(none)", &value, dst.span() *
sizeof(
typename dst_traits::value_type));
2102 if (dst.data() ==
nullptr) {
2103 space.fence(
"Kokkos::deep_copy: scalar copy on space, dst data is null");
2104 }
else if (dst.span_is_contiguous() &&
2105 !ViewTraits<DT, DP...>::impl_is_customized) {
2106 Impl::contiguous_fill_or_memset(space, dst, value);
2108 using ViewType =
View<DT, DP...>;
2110 int64_t strides[ViewType::rank + 1];
2111 dst.stride(strides);
2112 Kokkos::Iterate iterate;
2113 if (std::is_same_v<typename ViewType::array_layout, Kokkos::LayoutRight>) {
2114 iterate = Kokkos::Iterate::Right;
2115 }
else if (std::is_same_v<
typename ViewType::array_layout,
2117 iterate = Kokkos::Iterate::Left;
2118 }
else if (std::is_same_v<
typename ViewType::array_layout,
2120 if (strides[0] > strides[ViewType::rank > 0 ? ViewType::rank - 1 : 0])
2121 iterate = Kokkos::Iterate::Right;
2123 iterate = Kokkos::Iterate::Left;
2125 if (std::is_same_v<
typename ViewType::execution_space::array_layout,
2126 Kokkos::LayoutRight>)
2127 iterate = Kokkos::Iterate::Right;
2129 iterate = Kokkos::Iterate::Left;
2134 using ViewTypeUniform =
2135 std::conditional_t<ViewType::rank == 0,
2136 typename ViewType::uniform_runtime_type,
2137 typename ViewType::uniform_runtime_nomemspace_type>;
2138 if (dst.span() >
static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
2139 if (iterate == Kokkos::Iterate::Right)
2140 Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutRight, ExecSpace,
2141 ViewType::rank, int64_t>(dst, value, space);
2144 ViewType::rank, int64_t>(dst, value, space);
2146 if (iterate == Kokkos::Iterate::Right)
2147 Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutRight, ExecSpace,
2148 ViewType::rank, int32_t>(dst, value, space);
2151 ViewType::rank, int32_t>(dst, value, space);
2154 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
2155 Kokkos::Profiling::endDeepCopy();
2161 template <
class ExecSpace,
class DT,
class... DP>
2162 inline void deep_copy(
2163 const ExecSpace& space,
const View<DT, DP...>& dst,
2164 typename ViewTraits<DT, DP...>::const_value_type& value,
2166 Kokkos::is_execution_space<ExecSpace>::value &&
2167 std::is_void_v<
typename ViewTraits<DT, DP...>::specialize> &&
2169 memory_space>::accessible>* =
2171 using dst_traits = ViewTraits<DT, DP...>;
2172 static_assert(std::is_same_v<
typename dst_traits::non_const_value_type,
2173 typename dst_traits::value_type>,
2174 "deep_copy requires non-const type");
2175 using dst_memory_space =
typename dst_traits::memory_space;
2176 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy !=
nullptr) {
2177 Kokkos::Profiling::beginDeepCopy(
2178 Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
2179 dst.label(), dst.data(),
2181 "(none)", &value, dst.span() *
sizeof(
typename dst_traits::value_type));
2183 if (dst.data() ==
nullptr) {
2185 "Kokkos::deep_copy: scalar-to-view copy on space, dst data is null");
2187 space.fence(
"Kokkos::deep_copy: scalar-to-view copy on space, pre copy");
2188 using fill_exec_space =
typename dst_traits::memory_space::execution_space;
2189 if (dst.span_is_contiguous() &&
2190 !ViewTraits<DT, DP...>::impl_is_customized) {
2191 Impl::contiguous_fill_or_memset(fill_exec_space(), dst, value);
2193 using ViewTypeUniform = std::conditional_t<
2194 View<DT, DP...>::rank == 0,
2195 typename View<DT, DP...>::uniform_runtime_type,
2196 typename View<DT, DP...>::uniform_runtime_nomemspace_type>;
2197 Kokkos::Impl::ViewFill<ViewTypeUniform,
typename dst_traits::array_layout,
2198 fill_exec_space>(dst, value, fill_exec_space());
2200 fill_exec_space().fence(
2201 "Kokkos::deep_copy: scalar-to-view copy on space, fence after fill");
2203 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
2204 Kokkos::Profiling::endDeepCopy();
2209 template <
class ExecSpace,
class ST,
class... SP>
2210 inline void deep_copy(
2211 const ExecSpace& exec_space,
2212 typename ViewTraits<ST, SP...>::non_const_value_type& dst,
2213 const View<ST, SP...>& src,
2214 std::enable_if_t<Kokkos::is_execution_space<ExecSpace>::value &&
2215 std::is_same_v<
typename ViewTraits<ST, SP...>::specialize,
2216 void>>* =
nullptr) {
2217 using src_traits = ViewTraits<ST, SP...>;
2218 using src_memory_space =
typename src_traits::memory_space;
2219 static_assert(src_traits::rank == 0,
2220 "ERROR: Non-rank-zero view in deep_copy( value , View )");
2221 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy !=
nullptr) {
2222 Kokkos::Profiling::beginDeepCopy(
2225 Kokkos::Profiling::make_space_handle(src_memory_space::name()),
2226 src.label(), src.data(),
sizeof(ST));
2229 if (src.data() ==
nullptr) {
2231 "Kokkos::deep_copy: view-to-scalar copy on space, src data is null");
2232 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
2233 Kokkos::Profiling::endDeepCopy();
2238 Kokkos::Impl::DeepCopy<HostSpace, src_memory_space, ExecSpace>(
2239 exec_space, &dst, src.data(),
sizeof(ST));
2240 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
2241 Kokkos::Profiling::endDeepCopy();
2247 template <
class ExecSpace,
class DT,
class... DP,
class ST,
class... SP>
2248 inline void deep_copy(
2249 const ExecSpace& exec_space,
const View<DT, DP...>& dst,
2250 const View<ST, SP...>& src,
2252 (Kokkos::is_execution_space<ExecSpace>::value &&
2253 std::is_void_v<
typename ViewTraits<DT, DP...>::specialize> &&
2254 std::is_void_v<
typename ViewTraits<ST, SP...>::specialize> &&
2255 (
unsigned(ViewTraits<DT, DP...>::rank) ==
unsigned(0) &&
2256 unsigned(ViewTraits<ST, SP...>::rank) ==
unsigned(0)))>* =
nullptr) {
2257 using src_traits = ViewTraits<ST, SP...>;
2258 using dst_traits = ViewTraits<DT, DP...>;
2260 using src_memory_space =
typename src_traits::memory_space;
2261 using dst_memory_space =
typename dst_traits::memory_space;
2262 static_assert(std::is_same_v<
typename dst_traits::value_type,
2263 typename src_traits::non_const_value_type>,
2264 "deep_copy requires matching non-const destination type");
2266 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy !=
nullptr) {
2267 Kokkos::Profiling::beginDeepCopy(
2268 Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
2269 dst.label(), dst.data(),
2270 Kokkos::Profiling::make_space_handle(src_memory_space::name()),
2271 src.label(), src.data(),
sizeof(DT));
2274 if (dst.data() ==
nullptr && src.data() ==
nullptr) {
2276 "Kokkos::deep_copy: view-to-view copy on space, data is null");
2277 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
2278 Kokkos::Profiling::endDeepCopy();
2283 if (dst.data() != src.data()) {
2284 Kokkos::Impl::DeepCopy<dst_memory_space, src_memory_space, ExecSpace>(
2285 exec_space, dst.data(), src.data(),
2286 sizeof(
typename dst_traits::value_type));
2288 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
2289 Kokkos::Profiling::endDeepCopy();
2297 template <
class ExecSpace,
class DT,
class... DP,
class ST,
class... SP>
2298 inline void deep_copy(
2299 const ExecSpace& exec_space,
const View<DT, DP...>& dst,
2300 const View<ST, SP...>& src,
2302 (Kokkos::is_execution_space<ExecSpace>::value &&
2303 std::is_void_v<
typename ViewTraits<DT, DP...>::specialize> &&
2304 std::is_void_v<
typename ViewTraits<ST, SP...>::specialize> &&
2305 (
unsigned(ViewTraits<DT, DP...>::rank) != 0 ||
2306 unsigned(ViewTraits<ST, SP...>::rank) != 0))>* =
nullptr) {
2307 using dst_type =
View<DT, DP...>;
2308 using src_type =
View<ST, SP...>;
2310 static_assert(std::is_same_v<
typename dst_type::value_type,
2311 typename dst_type::non_const_value_type>,
2312 "deep_copy requires non-const destination type");
2314 static_assert((
unsigned(dst_type::rank) ==
unsigned(src_type::rank)),
2315 "deep_copy requires Views of equal rank");
2317 using dst_execution_space =
typename dst_type::execution_space;
2318 using src_execution_space =
typename src_type::execution_space;
2319 using dst_memory_space =
typename dst_type::memory_space;
2320 using src_memory_space =
typename src_type::memory_space;
2321 using dst_value_type =
typename dst_type::value_type;
2322 using src_value_type =
typename src_type::value_type;
2324 if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy !=
nullptr) {
2325 Kokkos::Profiling::beginDeepCopy(
2326 Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
2327 dst.label(), dst.data(),
2328 Kokkos::Profiling::make_space_handle(src_memory_space::name()),
2329 src.label(), src.data(), dst.span() *
sizeof(dst_value_type));
2332 dst_value_type* dst_start = dst.data();
2333 dst_value_type* dst_end = dst.data() + dst.span();
2334 src_value_type* src_start = src.data();
2335 src_value_type* src_end = src.data() + src.span();
2338 if ((dst_start ==
nullptr || src_start ==
nullptr) ||
2339 ((std::ptrdiff_t(dst_start) == std::ptrdiff_t(src_start)) &&
2340 (std::ptrdiff_t(dst_end) == std::ptrdiff_t(src_end)))) {
2342 if ((src.extent(0) != dst.extent(0)) || (src.extent(1) != dst.extent(1)) ||
2343 (src.extent(2) != dst.extent(2)) || (src.extent(3) != dst.extent(3)) ||
2344 (src.extent(4) != dst.extent(4)) || (src.extent(5) != dst.extent(5)) ||
2345 (src.extent(6) != dst.extent(6)) || (src.extent(7) != dst.extent(7))) {
2346 std::string message(
2347 "Deprecation Error: Kokkos::deep_copy extents of views don't "
2349 message += dst.label();
2351 message += std::to_string(dst.extent(0));
2352 for (
size_t r = 1; r < dst_type::rank; r++) {
2354 message += std::to_string(dst.extent(r));
2357 message += src.label();
2359 message += std::to_string(src.extent(0));
2360 for (
size_t r = 1; r < src_type::rank; r++) {
2362 message += std::to_string(src.extent(r));
2366 Kokkos::Impl::throw_runtime_exception(message);
2368 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
2369 Kokkos::Profiling::endDeepCopy();
2375 if ((((std::ptrdiff_t)dst_start < (std::ptrdiff_t)src_end) &&
2376 ((std::ptrdiff_t)dst_end > (std::ptrdiff_t)src_start)) &&
2377 ((dst.span_is_contiguous() && src.span_is_contiguous()))) {
2378 std::string message(
"Error: Kokkos::deep_copy of overlapping views: ");
2379 message += dst.label();
2381 message += std::to_string((std::ptrdiff_t)dst_start);
2383 message += std::to_string((std::ptrdiff_t)dst_end);
2385 message += src.label();
2387 message += std::to_string((std::ptrdiff_t)src_start);
2389 message += std::to_string((std::ptrdiff_t)src_end);
2391 Kokkos::Impl::throw_runtime_exception(message);
2395 if ((src.extent(0) != dst.extent(0)) || (src.extent(1) != dst.extent(1)) ||
2396 (src.extent(2) != dst.extent(2)) || (src.extent(3) != dst.extent(3)) ||
2397 (src.extent(4) != dst.extent(4)) || (src.extent(5) != dst.extent(5)) ||
2398 (src.extent(6) != dst.extent(6)) || (src.extent(7) != dst.extent(7))) {
2399 std::string message(
2400 "Deprecation Error: Kokkos::deep_copy extents of views don't match: ");
2401 message += dst.label();
2403 message += std::to_string(dst.extent(0));
2404 for (
size_t r = 1; r < dst_type::rank; r++) {
2406 message += std::to_string(dst.extent(r));
2409 message += src.label();
2411 message += std::to_string(src.extent(0));
2412 for (
size_t r = 1; r < src_type::rank; r++) {
2414 message += std::to_string(src.extent(r));
2418 Kokkos::Impl::throw_runtime_exception(message);
2424 if (std::is_same_v<
typename dst_type::value_type,
2425 typename src_type::non_const_value_type> &&
2426 (std::is_same_v<
typename dst_type::array_layout,
2427 typename src_type::array_layout> ||
2428 (dst_type::rank == 1 && src_type::rank == 1)) &&
2429 dst.span_is_contiguous() && src.span_is_contiguous() &&
2430 ((dst_type::rank < 1) || (dst.stride_0() == src.stride_0())) &&
2431 ((dst_type::rank < 2) || (dst.stride_1() == src.stride_1())) &&
2432 ((dst_type::rank < 3) || (dst.stride_2() == src.stride_2())) &&
2433 ((dst_type::rank < 4) || (dst.stride_3() == src.stride_3())) &&
2434 ((dst_type::rank < 5) || (dst.stride_4() == src.stride_4())) &&
2435 ((dst_type::rank < 6) || (dst.stride_5() == src.stride_5())) &&
2436 ((dst_type::rank < 7) || (dst.stride_6() == src.stride_6())) &&
2437 ((dst_type::rank < 8) || (dst.stride_7() == src.stride_7()))) {
2438 const size_t nbytes =
sizeof(
typename dst_type::value_type) * dst.span();
2439 if ((
void*)dst.data() != (
void*)src.data() && 0 < nbytes) {
2440 Kokkos::Impl::DeepCopy<dst_memory_space, src_memory_space, ExecSpace>(
2441 exec_space, dst.data(), src.data(), nbytes);
2447 constexpr
bool ExecCanAccessSrcDst =
2450 constexpr
bool DstExecCanAccessSrc =
2452 src_memory_space>::accessible;
2453 constexpr
bool SrcExecCanAccessDst =
2455 dst_memory_space>::accessible;
2457 if constexpr (ExecCanAccessSrcDst) {
2458 Impl::view_copy(exec_space, dst, src);
2459 }
else if constexpr (DstExecCanAccessSrc || SrcExecCanAccessDst) {
2460 using cpy_exec_space =
2461 std::conditional_t<DstExecCanAccessSrc, dst_execution_space,
2462 src_execution_space>;
2464 "Kokkos::deep_copy: view-to-view noncontiguous copy on space, pre "
2466 Impl::view_copy(cpy_exec_space(), dst, src);
2467 cpy_exec_space().fence(
2468 "Kokkos::deep_copy: view-to-view noncontiguous copy on space, post "
2471 Kokkos::Impl::throw_runtime_exception(
2472 "deep_copy given views that would require a temporary allocation");
2475 if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy !=
nullptr) {
2476 Kokkos::Profiling::endDeepCopy();
2488 template <
typename ViewType>
2489 bool size_mismatch(
const ViewType& view,
unsigned int max_extent,
2490 const size_t new_extents[8]) {
2491 for (
unsigned int dim = 0; dim < max_extent; ++dim)
2492 if (new_extents[dim] != view.extent(dim)) {
2495 for (
unsigned int dim = max_extent; dim < 8; ++dim)
2496 if (new_extents[dim] != KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2506 template <
class T,
class... P,
class... ViewCtorArgs>
2507 inline std::enable_if_t<
2508 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2510 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2511 Kokkos::LayoutRight>>
2512 impl_resize(
const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2513 Kokkos::View<T, P...>& v,
const size_t n0,
const size_t n1,
2514 const size_t n2,
const size_t n3,
const size_t n4,
const size_t n5,
2515 const size_t n6,
const size_t n7) {
2516 using view_type = Kokkos::View<T, P...>;
2517 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2519 static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2520 "Can only resize managed views");
2521 static_assert(!alloc_prop_input::has_label,
2522 "The view constructor arguments passed to Kokkos::resize "
2523 "must not include a label!");
2524 static_assert(!alloc_prop_input::has_pointer,
2525 "The view constructor arguments passed to Kokkos::resize must "
2526 "not include a pointer!");
2527 static_assert(!alloc_prop_input::has_memory_space,
2528 "The view constructor arguments passed to Kokkos::resize must "
2529 "not include a memory space instance!");
2538 const size_t new_extents[8] = {n0, n1, n2, n3, n4, n5, n6, n7};
2539 const bool sizeMismatch = Impl::size_mismatch(v, v.rank_dynamic, new_extents);
2542 #ifdef KOKKOS_ENABLE_IMPL_VIEW_LEGACY
2543 auto prop_copy = Impl::with_properties_if_unset(
2544 arg_prop,
typename view_type::execution_space{}, v.label());
2546 view_type v_resized(prop_copy, n0, n1, n2, n3, n4, n5, n6, n7);
2548 auto prop_copy = [&]() {
2549 if constexpr (view_type::traits::impl_is_customized) {
2552 Kokkos::Impl::AccessorArg_t acc_arg{new_extents[view_type::rank()]};
2553 return Impl::with_properties_if_unset(
2554 arg_prop, acc_arg,
typename view_type::execution_space{},
2557 return Impl::with_properties_if_unset(
2558 arg_prop,
typename view_type::execution_space{}, v.label());
2562 view_type v_resized;
2563 if constexpr (view_type::rank() == 0) {
2564 v_resized = view_type(prop_copy);
2565 }
else if constexpr (view_type::rank() == 1) {
2566 v_resized = view_type(prop_copy, n0);
2567 }
else if constexpr (view_type::rank() == 2) {
2568 v_resized = view_type(prop_copy, n0, n1);
2569 }
else if constexpr (view_type::rank() == 3) {
2570 v_resized = view_type(prop_copy, n0, n1, n2);
2571 }
else if constexpr (view_type::rank() == 4) {
2572 v_resized = view_type(prop_copy, n0, n1, n2, n3);
2573 }
else if constexpr (view_type::rank() == 5) {
2574 v_resized = view_type(prop_copy, n0, n1, n2, n3, n4);
2575 }
else if constexpr (view_type::rank() == 6) {
2576 v_resized = view_type(prop_copy, n0, n1, n2, n3, n4, n5);
2577 }
else if constexpr (view_type::rank() == 7) {
2578 v_resized = view_type(prop_copy, n0, n1, n2, n3, n4, n5, n6);
2580 v_resized = view_type(prop_copy, n0, n1, n2, n3, n4, n5, n6, n7);
2584 if constexpr (alloc_prop_input::has_execution_space)
2585 Kokkos::Impl::ViewRemap<view_type, view_type>(
2586 v_resized, v, Impl::get_property<Impl::ExecutionSpaceTag>(prop_copy));
2588 Kokkos::Impl::ViewRemap<view_type, view_type>(v_resized, v);
2589 Kokkos::fence(
"Kokkos::resize(View)");
2596 template <
class T,
class... P,
class... ViewCtorArgs>
2597 inline std::enable_if_t<
2598 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2600 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2601 Kokkos::LayoutRight>>
2602 resize(
const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2603 Kokkos::View<T, P...>& v,
const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2604 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2605 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2606 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2607 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2608 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2609 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2610 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2611 impl_resize(arg_prop, v, n0, n1, n2, n3, n4, n5, n6, n7);
2614 template <
class T,
class... P>
2615 inline std::enable_if_t<
2616 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2618 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2619 Kokkos::LayoutRight>>
2620 resize(Kokkos::View<T, P...>& v,
const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2621 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2622 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2623 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2624 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2625 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2626 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2627 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2628 impl_resize(Impl::ViewCtorProp<>{}, v, n0, n1, n2, n3, n4, n5, n6, n7);
2631 template <
class I,
class T,
class... P>
2632 inline std::enable_if_t<
2633 (Impl::is_view_ctor_property<I>::value ||
2634 Kokkos::is_execution_space<I>::value) &&
2635 (std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2637 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2638 Kokkos::LayoutRight>)>
2639 resize(
const I& arg_prop, Kokkos::View<T, P...>& v,
2640 const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2641 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2642 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2643 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2644 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2645 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2646 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2647 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2648 impl_resize(Kokkos::view_alloc(arg_prop), v, n0, n1, n2, n3, n4, n5, n6, n7);
2651 template <
class T,
class... P,
class... ViewCtorArgs>
2652 inline std::enable_if_t<
2653 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2655 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2656 Kokkos::LayoutRight> ||
2657 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2659 impl_resize(
const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2660 Kokkos::View<T, P...>& v,
2661 const typename Kokkos::View<T, P...>::array_layout& layout) {
2662 using view_type = Kokkos::View<T, P...>;
2663 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2665 static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2666 "Can only resize managed views");
2667 static_assert(!alloc_prop_input::has_label,
2668 "The view constructor arguments passed to Kokkos::resize "
2669 "must not include a label!");
2670 static_assert(!alloc_prop_input::has_pointer,
2671 "The view constructor arguments passed to Kokkos::resize must "
2672 "not include a pointer!");
2673 static_assert(!alloc_prop_input::has_memory_space,
2674 "The view constructor arguments passed to Kokkos::resize must "
2675 "not include a memory space instance!");
2677 if (v.layout() != layout) {
2678 auto prop_copy = Impl::with_properties_if_unset(arg_prop, v.label());
2680 view_type v_resized(prop_copy, layout);
2682 if constexpr (alloc_prop_input::has_execution_space)
2683 Kokkos::Impl::ViewRemap<view_type, view_type>(
2684 v_resized, v, Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop));
2686 Kokkos::Impl::ViewRemap<view_type, view_type>(v_resized, v);
2687 Kokkos::fence(
"Kokkos::resize(View)");
2697 template <
class T,
class... P,
class... ViewCtorArgs>
2698 inline std::enable_if_t<
2699 !(std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2701 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2702 Kokkos::LayoutRight> ||
2703 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2705 impl_resize(
const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2706 Kokkos::View<T, P...>& v,
2707 const typename Kokkos::View<T, P...>::array_layout& layout) {
2708 using view_type = Kokkos::View<T, P...>;
2709 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2711 static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2712 "Can only resize managed views");
2713 static_assert(!alloc_prop_input::has_label,
2714 "The view constructor arguments passed to Kokkos::resize "
2715 "must not include a label!");
2716 static_assert(!alloc_prop_input::has_pointer,
2717 "The view constructor arguments passed to Kokkos::resize must "
2718 "not include a pointer!");
2719 static_assert(!alloc_prop_input::has_memory_space,
2720 "The view constructor arguments passed to Kokkos::resize must "
2721 "not include a memory space instance!");
2723 auto prop_copy = Impl::with_properties_if_unset(arg_prop, v.label());
2725 view_type v_resized(prop_copy, layout);
2727 if constexpr (alloc_prop_input::has_execution_space)
2728 Kokkos::Impl::ViewRemap<view_type, view_type>(
2729 v_resized, v, Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop));
2731 Kokkos::Impl::ViewRemap<view_type, view_type>(v_resized, v);
2732 Kokkos::fence(
"Kokkos::resize(View)");
2738 template <
class T,
class... P,
class... ViewCtorArgs>
2739 inline void resize(
const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2740 Kokkos::View<T, P...>& v,
2741 const typename Kokkos::View<T, P...>::array_layout& layout) {
2742 impl_resize(arg_prop, v, layout);
2745 template <
class I,
class T,
class... P>
2746 inline std::enable_if_t<Impl::is_view_ctor_property<I>::value ||
2747 Kokkos::is_execution_space<I>::value>
2748 resize(
const I& arg_prop, Kokkos::View<T, P...>& v,
2749 const typename Kokkos::View<T, P...>::array_layout& layout) {
2750 impl_resize(arg_prop, v, layout);
2753 template <
class ExecutionSpace,
class T,
class... P>
2754 inline void resize(
const ExecutionSpace& exec_space, Kokkos::View<T, P...>& v,
2755 const typename Kokkos::View<T, P...>::array_layout& layout) {
2756 impl_resize(Impl::ViewCtorProp<>(), exec_space, v, layout);
2759 template <
class T,
class... P>
2760 inline void resize(Kokkos::View<T, P...>& v,
2761 const typename Kokkos::View<T, P...>::array_layout& layout) {
2762 impl_resize(Impl::ViewCtorProp<>{}, v, layout);
2766 template <
class T,
class... P,
class... ViewCtorArgs>
2767 inline std::enable_if_t<
2768 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2770 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2771 Kokkos::LayoutRight>>
2772 impl_realloc(Kokkos::View<T, P...>& v,
const size_t n0,
const size_t n1,
2773 const size_t n2,
const size_t n3,
const size_t n4,
const size_t n5,
2774 const size_t n6,
const size_t n7,
2775 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
2776 using view_type = Kokkos::View<T, P...>;
2777 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2779 static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2780 "Can only realloc managed views");
2781 static_assert(!alloc_prop_input::has_label,
2782 "The view constructor arguments passed to Kokkos::realloc must "
2783 "not include a label!");
2784 static_assert(!alloc_prop_input::has_pointer,
2785 "The view constructor arguments passed to Kokkos::realloc must "
2786 "not include a pointer!");
2787 static_assert(!alloc_prop_input::has_memory_space,
2788 "The view constructor arguments passed to Kokkos::realloc must "
2789 "not include a memory space instance!");
2791 const size_t new_extents[8] = {n0, n1, n2, n3, n4, n5, n6, n7};
2792 const bool sizeMismatch = Impl::size_mismatch(v, v.rank_dynamic, new_extents);
2795 auto arg_prop_copy = Impl::with_properties_if_unset(arg_prop, v.label());
2798 v = view_type(arg_prop_copy, n0, n1, n2, n3, n4, n5, n6, n7);
2802 if constexpr (alloc_prop_input::initialize) {
2803 if constexpr (alloc_prop_input::has_execution_space) {
2804 const auto& exec_space =
2805 Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop);
2806 Kokkos::deep_copy(exec_space, v,
typename view_type::value_type{});
2808 Kokkos::deep_copy(v,
typename view_type::value_type{});
2812 template <
class T,
class... P,
class... ViewCtorArgs>
2813 inline std::enable_if_t<
2814 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2816 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2817 Kokkos::LayoutRight>>
2818 realloc(
const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2819 Kokkos::View<T, P...>& v,
2820 const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2821 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2822 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2823 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2824 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2825 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2826 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2827 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2828 impl_realloc(v, n0, n1, n2, n3, n4, n5, n6, n7, arg_prop);
2831 template <
class T,
class... P>
2832 inline std::enable_if_t<
2833 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2835 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2836 Kokkos::LayoutRight>>
2837 realloc(Kokkos::View<T, P...>& v,
2838 const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2839 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2840 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2841 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2842 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2843 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2844 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2845 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2846 impl_realloc(v, n0, n1, n2, n3, n4, n5, n6, n7, Impl::ViewCtorProp<>{});
2849 template <
class I,
class T,
class... P>
2850 inline std::enable_if_t<
2851 Impl::is_view_ctor_property<I>::value &&
2852 (std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2854 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2855 Kokkos::LayoutRight>)>
2856 realloc(
const I& arg_prop, Kokkos::View<T, P...>& v,
2857 const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2858 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2859 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2860 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2861 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2862 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2863 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2864 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2865 impl_realloc(v, n0, n1, n2, n3, n4, n5, n6, n7, Kokkos::view_alloc(arg_prop));
2868 template <
class T,
class... P,
class... ViewCtorArgs>
2869 inline std::enable_if_t<
2870 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2872 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2873 Kokkos::LayoutRight> ||
2874 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2876 impl_realloc(Kokkos::View<T, P...>& v,
2877 const typename Kokkos::View<T, P...>::array_layout& layout,
2878 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
2879 using view_type = Kokkos::View<T, P...>;
2880 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2882 static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2883 "Can only realloc managed views");
2884 static_assert(!alloc_prop_input::has_label,
2885 "The view constructor arguments passed to Kokkos::realloc must "
2886 "not include a label!");
2887 static_assert(!alloc_prop_input::has_pointer,
2888 "The view constructor arguments passed to Kokkos::realloc must "
2889 "not include a pointer!");
2890 static_assert(!alloc_prop_input::has_memory_space,
2891 "The view constructor arguments passed to Kokkos::realloc must "
2892 "not include a memory space instance!");
2894 if (v.layout() != layout) {
2896 v = view_type(arg_prop, layout);
2900 if constexpr (alloc_prop_input::initialize) {
2901 if constexpr (alloc_prop_input::has_execution_space) {
2902 const auto& exec_space =
2903 Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop);
2904 Kokkos::deep_copy(exec_space, v,
typename view_type::value_type{});
2906 Kokkos::deep_copy(v,
typename view_type::value_type{});
2913 template <
class T,
class... P,
class... ViewCtorArgs>
2914 inline std::enable_if_t<
2915 !(std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2917 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2918 Kokkos::LayoutRight> ||
2919 std::is_same_v<
typename Kokkos::View<T, P...>::array_layout,
2921 impl_realloc(Kokkos::View<T, P...>& v,
2922 const typename Kokkos::View<T, P...>::array_layout& layout,
2923 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
2924 using view_type = Kokkos::View<T, P...>;
2925 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2927 static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2928 "Can only realloc managed views");
2929 static_assert(!alloc_prop_input::has_label,
2930 "The view constructor arguments passed to Kokkos::realloc must "
2931 "not include a label!");
2932 static_assert(!alloc_prop_input::has_pointer,
2933 "The view constructor arguments passed to Kokkos::realloc must "
2934 "not include a pointer!");
2935 static_assert(!alloc_prop_input::has_memory_space,
2936 "The view constructor arguments passed to Kokkos::realloc must "
2937 "not include a memory space instance!");
2939 auto arg_prop_copy = Impl::with_properties_if_unset(arg_prop, v.label());
2942 v = view_type(arg_prop_copy, layout);
2945 template <
class T,
class... P,
class... ViewCtorArgs>
2946 inline void realloc(
2947 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2948 Kokkos::View<T, P...>& v,
2949 const typename Kokkos::View<T, P...>::array_layout& layout) {
2950 impl_realloc(v, layout, arg_prop);
2953 template <
class I,
class T,
class... P>
2954 inline std::enable_if_t<Impl::is_view_ctor_property<I>::value> realloc(
2955 const I& arg_prop, Kokkos::View<T, P...>& v,
2956 const typename Kokkos::View<T, P...>::array_layout& layout) {
2957 impl_realloc(v, layout, Kokkos::view_alloc(arg_prop));
2960 template <
class T,
class... P>
2961 inline void realloc(
2962 Kokkos::View<T, P...>& v,
2963 const typename Kokkos::View<T, P...>::array_layout& layout) {
2964 impl_realloc(v, layout, Impl::ViewCtorProp<>{});
2976 template <
class Space,
class T,
class... P>
2977 struct MirrorViewType {
2979 using src_view_type =
typename Kokkos::View<T, P...>;
2981 using memory_space =
typename Space::memory_space;
2983 static constexpr
bool is_same_memspace =
2984 std::is_same_v<memory_space, typename src_view_type::memory_space>;
2986 using array_layout =
typename src_view_type::array_layout;
2989 using data_type =
typename src_view_type::non_const_data_type;
2991 using dest_view_type = Kokkos::View<data_type, array_layout, Space>;
2995 std::conditional_t<is_same_memspace, src_view_type, dest_view_type>;
2999 template <
class... ViewCtorArgs>
3000 void check_view_ctor_args_create_mirror() {
3001 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
3004 !alloc_prop_input::has_label,
3005 "The view constructor arguments passed to Kokkos::create_mirror[_view] "
3006 "must not include a label!");
3007 static_assert(!alloc_prop_input::has_pointer,
3008 "The view constructor arguments passed to "
3009 "Kokkos::create_mirror[_view] must "
3010 "not include a pointer!");
3011 static_assert(!alloc_prop_input::allow_padding,
3012 "The view constructor arguments passed to "
3013 "Kokkos::create_mirror[_view] must "
3014 "not explicitly allow padding!");
3020 template <
class T,
class... P,
class... ViewCtorArgs>
3021 inline auto create_mirror(
const Kokkos::View<T, P...>& src,
3022 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
3023 check_view_ctor_args_create_mirror<ViewCtorArgs...>();
3025 auto prop_copy = Impl::with_properties_if_unset(
3026 arg_prop, std::string(src.label()).append(
"_mirror"));
3028 if constexpr (Impl::ViewCtorProp<ViewCtorArgs...>::has_memory_space) {
3029 using memory_space =
typename decltype(prop_copy)::memory_space;
3031 typename Impl::MirrorViewType<memory_space, T, P...>::dest_view_type;
3032 #ifndef KOKKOS_ENABLE_IMPL_VIEW_LEGACY
3041 if constexpr (std::is_constructible_v<
3042 typename dst_type::accessor_type,
3043 typename Kokkos::View<T, P...>::accessor_type>)
3044 return dst_type(prop_copy, src.mapping(), src.accessor());
3046 return dst_type(prop_copy, src.layout());
3048 return dst_type(prop_copy, src.layout());
3051 using dst_type =
typename View<T, P...>::HostMirror;
3052 #ifndef KOKKOS_ENABLE_IMPL_VIEW_LEGACY
3055 if constexpr (std::is_constructible_v<
3056 typename dst_type::accessor_type,
3057 typename Kokkos::View<T, P...>::accessor_type>)
3058 return dst_type(prop_copy, src.mapping(), src.accessor());
3060 return dst_type(prop_copy, src.layout());
3062 return dst_type(prop_copy, src.layout());
3065 #if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
3066 !defined(KOKKOS_COMPILER_MSVC)
3067 __builtin_unreachable();
3073 template <
class T,
class... P,
3074 typename = std::enable_if_t<
3075 std::is_void_v<
typename ViewTraits<T, P...>::specialize>>>
3076 auto create_mirror(Kokkos::View<T, P...>
const& src) {
3077 return Impl::create_mirror(src, Impl::ViewCtorProp<>{});
3081 template <
class T,
class... P,
3082 typename = std::enable_if_t<
3083 std::is_void_v<
typename ViewTraits<T, P...>::specialize>>>
3084 auto create_mirror(Kokkos::Impl::WithoutInitializing_t wi,
3085 Kokkos::View<T, P...>
const& src) {
3086 return Impl::create_mirror(src, view_alloc(wi));
3090 template <
class Space,
class T,
class... P,
3091 typename Enable = std::enable_if_t<
3092 Kokkos::is_space<Space>::value &&
3093 std::is_void_v<
typename ViewTraits<T, P...>::specialize>>>
3094 auto create_mirror(Space
const&, Kokkos::View<T, P...>
const& src) {
3095 return Impl::create_mirror(src, view_alloc(
typename Space::memory_space{}));
3100 template <
class T,
class... P,
class... ViewCtorArgs,
3101 typename = std::enable_if_t<
3102 std::is_void_v<
typename ViewTraits<T, P...>::specialize>>>
3103 auto create_mirror(Impl::ViewCtorProp<ViewCtorArgs...>
const& arg_prop,
3104 Kokkos::View<T, P...>
const& src) {
3105 return Impl::create_mirror(src, arg_prop);
3109 template <
class Space,
class T,
class... P,
3110 typename Enable = std::enable_if_t<
3111 Kokkos::is_space<Space>::value &&
3112 std::is_void_v<
typename ViewTraits<T, P...>::specialize>>>
3113 auto create_mirror(Kokkos::Impl::WithoutInitializing_t wi, Space
const&,
3114 Kokkos::View<T, P...>
const& src) {
3115 return Impl::create_mirror(src,
3116 view_alloc(
typename Space::memory_space{}, wi));
3123 template <
class View,
class... ViewCtorArgs>
3124 inline auto choose_create_mirror(
3125 const View& src,
const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
3133 if constexpr (std::is_void_v<typename View::traits::specialize>) {
3137 using namespace Kokkos::Impl;
3139 return create_mirror(src, arg_prop);
3142 using ViewProp = Impl::ViewCtorProp<ViewCtorArgs...>;
3145 using namespace Kokkos;
3147 if constexpr (
sizeof...(ViewCtorArgs) == 0) {
3150 return create_mirror(src);
3151 }
else if constexpr (
sizeof...(ViewCtorArgs) == 1 &&
3152 ViewProp::has_memory_space) {
3155 return create_mirror(
typename ViewProp::memory_space{}, src);
3156 }
else if constexpr (
sizeof...(ViewCtorArgs) == 1 &&
3157 !ViewProp::initialize) {
3160 return create_mirror(
typename Kokkos::Impl::WithoutInitializing_t{}, src);
3161 }
else if constexpr (
sizeof...(ViewCtorArgs) == 2 &&
3162 ViewProp::has_memory_space && !ViewProp::initialize) {
3165 return create_mirror(
typename Kokkos::Impl::WithoutInitializing_t{},
3166 typename ViewProp::memory_space{}, src);
3172 return create_mirror(arg_prop, src);
3176 #if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
3177 !defined(KOKKOS_COMPILER_MSVC)
3178 __builtin_unreachable();
3185 template <
class T,
class... P,
class... ViewCtorArgs>
3186 inline auto create_mirror_view(
3187 const Kokkos::View<T, P...>& src,
3188 [[maybe_unused]]
const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
3189 if constexpr (!Impl::ViewCtorProp<ViewCtorArgs...>::has_memory_space) {
3190 if constexpr (std::is_same_v<
typename Kokkos::View<T, P...>::memory_space,
3191 typename Kokkos::View<
3192 T, P...>::HostMirror::memory_space> &&
3194 typename Kokkos::View<T, P...>::data_type,
3195 typename Kokkos::View<T, P...>::HostMirror::data_type>) {
3196 check_view_ctor_args_create_mirror<ViewCtorArgs...>();
3197 return typename Kokkos::View<T, P...>::HostMirror(src);
3199 return Kokkos::Impl::choose_create_mirror(src, arg_prop);
3202 if constexpr (Impl::MirrorViewType<
typename Impl::ViewCtorProp<
3203 ViewCtorArgs...>::memory_space,
3204 T, P...>::is_same_memspace) {
3205 check_view_ctor_args_create_mirror<ViewCtorArgs...>();
3206 return typename Impl::MirrorViewType<
3207 typename Impl::ViewCtorProp<ViewCtorArgs...>::memory_space, T,
3208 P...>::view_type(src);
3210 return Kokkos::Impl::choose_create_mirror(src, arg_prop);
3213 #if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
3214 !defined(KOKKOS_COMPILER_MSVC)
3215 __builtin_unreachable();
3221 template <
class T,
class... P>
3222 auto create_mirror_view(
const Kokkos::View<T, P...>& src) {
3223 return Impl::create_mirror_view(src, view_alloc());
3227 template <
class T,
class... P>
3228 auto create_mirror_view(Kokkos::Impl::WithoutInitializing_t wi,
3229 Kokkos::View<T, P...>
const& src) {
3230 return Impl::create_mirror_view(src, view_alloc(wi));
3234 template <
class Space,
class T,
class... P,
3235 class Enable = std::enable_if_t<Kokkos::is_space<Space>::value>>
3236 auto create_mirror_view(
const Space&,
const Kokkos::View<T, P...>& src) {
3237 return Impl::create_mirror_view(src,
3238 view_alloc(
typename Space::memory_space()));
3242 template <
class Space,
class T,
class... P,
3243 typename Enable = std::enable_if_t<Kokkos::is_space<Space>::value>>
3244 auto create_mirror_view(Kokkos::Impl::WithoutInitializing_t wi, Space
const&,
3245 Kokkos::View<T, P...>
const& src) {
3246 return Impl::create_mirror_view(
3247 src, view_alloc(
typename Space::memory_space{}, wi));
3252 template <
class T,
class... P,
class... ViewCtorArgs,
3253 typename = std::enable_if_t<
3254 std::is_void_v<
typename ViewTraits<T, P...>::specialize>>>
3255 auto create_mirror_view(
const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
3256 const Kokkos::View<T, P...>& src) {
3257 return Impl::create_mirror_view(src, arg_prop);
3263 template <
class... ViewCtorArgs>
3264 void check_view_ctor_args_create_mirror_view_and_copy() {
3265 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
3268 alloc_prop_input::has_memory_space,
3269 "The view constructor arguments passed to "
3270 "Kokkos::create_mirror_view_and_copy must include a memory space!");
3271 static_assert(!alloc_prop_input::has_pointer,
3272 "The view constructor arguments passed to "
3273 "Kokkos::create_mirror_view_and_copy must "
3274 "not include a pointer!");
3275 static_assert(!alloc_prop_input::allow_padding,
3276 "The view constructor arguments passed to "
3277 "Kokkos::create_mirror_view_and_copy must "
3278 "not explicitly allow padding!");
3286 template <
class... ViewCtorArgs,
class T,
class... P,
3287 class Enable = std::enable_if_t<
3288 std::is_void_v<
typename ViewTraits<T, P...>::specialize>>>
3289 auto create_mirror_view_and_copy(
3290 [[maybe_unused]]
const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
3291 const Kokkos::View<T, P...>& src) {
3292 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
3294 Impl::check_view_ctor_args_create_mirror_view_and_copy<ViewCtorArgs...>();
3296 if constexpr (Impl::MirrorViewType<
typename alloc_prop_input::memory_space, T,
3297 P...>::is_same_memspace) {
3299 if constexpr (!alloc_prop_input::has_execution_space)
3301 "Kokkos::create_mirror_view_and_copy: fence before returning src "
3305 using Space =
typename alloc_prop_input::memory_space;
3306 using Mirror =
typename Impl::MirrorViewType<Space, T, P...>::view_type;
3308 auto arg_prop_copy = Impl::with_properties_if_unset(
3309 arg_prop, std::string{}, WithoutInitializing,
3310 typename Space::execution_space{});
3312 std::string& label = Impl::get_property<Impl::LabelTag>(arg_prop_copy);
3313 if (label.empty()) label = src.label();
3314 auto mirror =
typename Mirror::non_const_type{arg_prop_copy, src.layout()};
3315 if constexpr (alloc_prop_input::has_execution_space) {
3316 deep_copy(Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop_copy),
3319 deep_copy(mirror, src);
3322 #if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
3323 !defined(KOKKOS_COMPILER_MSVC)
3324 __builtin_unreachable();
3331 template <
class Space,
class T,
class... P,
3332 typename Enable = std::enable_if_t<Kokkos::is_space<Space>::value>>
3333 typename Impl::MirrorViewType<Space, T, P...>::view_type
3334 create_mirror_view_and_copy(
3335 const Space&,
const Kokkos::View<T, P...>& src,
3336 std::string
const& name =
"",
3338 std::is_void_v<
typename ViewTraits<T, P...>::specialize>>* =
nullptr) {
3339 return create_mirror_view_and_copy(
3340 Kokkos::view_alloc(
typename Space::memory_space{}, name), src);
Memory layout tag indicating left-to-right (Fortran scheme) striding of multi-indices.
Can AccessSpace access MemorySpace ?
Replacement for std::pair that works on CUDA devices.
Memory layout tag indicated arbitrarily strided multi-index mapping into contiguous memory...
Memory management for host memory.
Memory layout tag indicating right-to-left (C or lexigraphical scheme) striding of multi-indices...
static constexpr const char * name()
Return Name of the MemorySpace.
Declaration of various MemoryLayout options.
Declaration of parallel operators.
Execution policy for work over a range of an integral type.