Kokkos Core Kernels Package  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Groups Pages
Kokkos_CopyViews.hpp
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 
17 #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18 #include <Kokkos_Macros.hpp>
19 static_assert(false,
20  "Including non-public Kokkos header files is not allowed.");
21 #endif
22 #ifndef KOKKOS_COPYVIEWS_HPP_
23 #define KOKKOS_COPYVIEWS_HPP_
24 #include <string>
25 #include <sstream>
26 #include <Kokkos_Parallel.hpp>
27 #include <KokkosExp_MDRangePolicy.hpp>
28 #include <Kokkos_Layout.hpp>
29 #include <impl/Kokkos_HostSpace_ZeroMemset.hpp>
30 
31 //----------------------------------------------------------------------------
32 //----------------------------------------------------------------------------
33 
34 namespace Kokkos {
35 
36 namespace Impl {
37 
38 template <class Layout>
39 struct ViewFillLayoutSelector {};
40 
41 template <>
42 struct ViewFillLayoutSelector<Kokkos::LayoutLeft> {
43  static const Kokkos::Iterate iterate = Kokkos::Iterate::Left;
44 };
45 
46 template <>
47 struct ViewFillLayoutSelector<Kokkos::LayoutRight> {
48  static const Kokkos::Iterate iterate = Kokkos::Iterate::Right;
49 };
50 
51 } // namespace Impl
52 } // namespace Kokkos
53 
54 namespace Kokkos {
55 namespace Impl {
56 
57 template <class ViewType, class Layout, class ExecSpace, typename iType>
58 struct ViewFill<ViewType, Layout, ExecSpace, 0, iType> {
59  using ST = typename ViewType::non_const_value_type;
60  ViewFill(const ViewType& a, const ST& val, const ExecSpace& space) {
61  Kokkos::Impl::DeepCopy<typename ViewType::memory_space, Kokkos::HostSpace,
62  ExecSpace>(space, a.data(), &val, sizeof(ST));
63  }
64 };
65 
66 template <class ViewType, class Layout, class ExecSpace, typename iType>
67 struct ViewFill<ViewType, Layout, ExecSpace, 1, iType> {
68  ViewType a;
69  typename ViewType::const_value_type val;
70  using policy_type = Kokkos::RangePolicy<ExecSpace, Kokkos::IndexType<iType>>;
71 
72  ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
73  const ExecSpace& space)
74  : a(a_), val(val_) {
75  Kokkos::parallel_for("Kokkos::ViewFill-1D",
76  policy_type(space, 0, a.extent(0)), *this);
77  }
78 
79  KOKKOS_INLINE_FUNCTION
80  void operator()(const iType& i) const { a(i) = val; };
81 };
82 
83 template <class ViewType, class Layout, class ExecSpace, typename iType>
84 struct ViewFill<ViewType, Layout, ExecSpace, 2, iType> {
85  ViewType a;
86  typename ViewType::const_value_type val;
87 
88  using iterate_type = Kokkos::Rank<2, ViewFillLayoutSelector<Layout>::iterate,
89  ViewFillLayoutSelector<Layout>::iterate>;
90  using policy_type =
91  Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
92 
93  ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
94  const ExecSpace& space)
95  : a(a_), val(val_) {
96  Kokkos::parallel_for("Kokkos::ViewFill-2D",
97  policy_type(space, {0, 0}, {a.extent(0), a.extent(1)}),
98  *this);
99  }
100 
101  KOKKOS_INLINE_FUNCTION
102  void operator()(const iType& i0, const iType& i1) const { a(i0, i1) = val; };
103 };
104 
105 template <class ViewType, class Layout, class ExecSpace, typename iType>
106 struct ViewFill<ViewType, Layout, ExecSpace, 3, iType> {
107  ViewType a;
108  typename ViewType::const_value_type val;
109 
110  using iterate_type = Kokkos::Rank<3, ViewFillLayoutSelector<Layout>::iterate,
111  ViewFillLayoutSelector<Layout>::iterate>;
112  using policy_type =
113  Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
114 
115  ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
116  const ExecSpace& space)
117  : a(a_), val(val_) {
118  Kokkos::parallel_for(
119  "Kokkos::ViewFill-3D",
120  policy_type(space, {0, 0, 0}, {a.extent(0), a.extent(1), a.extent(2)}),
121  *this);
122  }
123 
124  KOKKOS_INLINE_FUNCTION
125  void operator()(const iType& i0, const iType& i1, const iType& i2) const {
126  a(i0, i1, i2) = val;
127  };
128 };
129 
130 template <class ViewType, class Layout, class ExecSpace, typename iType>
131 struct ViewFill<ViewType, Layout, ExecSpace, 4, iType> {
132  ViewType a;
133  typename ViewType::const_value_type val;
134 
135  using iterate_type = Kokkos::Rank<4, ViewFillLayoutSelector<Layout>::iterate,
136  ViewFillLayoutSelector<Layout>::iterate>;
137  using policy_type =
138  Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
139 
140  ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
141  const ExecSpace& space)
142  : a(a_), val(val_) {
143  Kokkos::parallel_for(
144  "Kokkos::ViewFill-4D",
145  policy_type(space, {0, 0, 0, 0},
146  {a.extent(0), a.extent(1), a.extent(2), a.extent(3)}),
147  *this);
148  }
149 
150  KOKKOS_INLINE_FUNCTION
151  void operator()(const iType& i0, const iType& i1, const iType& i2,
152  const iType& i3) const {
153  a(i0, i1, i2, i3) = val;
154  };
155 };
156 
157 template <class ViewType, class Layout, class ExecSpace, typename iType>
158 struct ViewFill<ViewType, Layout, ExecSpace, 5, iType> {
159  ViewType a;
160  typename ViewType::const_value_type val;
161 
162  using iterate_type = Kokkos::Rank<5, ViewFillLayoutSelector<Layout>::iterate,
163  ViewFillLayoutSelector<Layout>::iterate>;
164  using policy_type =
165  Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
166 
167  ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
168  const ExecSpace& space)
169  : a(a_), val(val_) {
170  Kokkos::parallel_for("Kokkos::ViewFill-5D",
171  policy_type(space, {0, 0, 0, 0, 0},
172  {a.extent(0), a.extent(1), a.extent(2),
173  a.extent(3), a.extent(4)}),
174  *this);
175  }
176 
177  KOKKOS_INLINE_FUNCTION
178  void operator()(const iType& i0, const iType& i1, const iType& i2,
179  const iType& i3, const iType& i4) const {
180  a(i0, i1, i2, i3, i4) = val;
181  };
182 };
183 
184 template <class ViewType, class Layout, class ExecSpace, typename iType>
185 struct ViewFill<ViewType, Layout, ExecSpace, 6, iType> {
186  ViewType a;
187  typename ViewType::const_value_type val;
188 
189  using iterate_type = Kokkos::Rank<6, ViewFillLayoutSelector<Layout>::iterate,
190  ViewFillLayoutSelector<Layout>::iterate>;
191  using policy_type =
192  Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
193 
194  ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
195  const ExecSpace& space)
196  : a(a_), val(val_) {
197  Kokkos::parallel_for("Kokkos::ViewFill-6D",
198  policy_type(space, {0, 0, 0, 0, 0, 0},
199  {a.extent(0), a.extent(1), a.extent(2),
200  a.extent(3), a.extent(4), a.extent(5)}),
201  *this);
202  }
203 
204  KOKKOS_INLINE_FUNCTION
205  void operator()(const iType& i0, const iType& i1, const iType& i2,
206  const iType& i3, const iType& i4, const iType& i5) const {
207  a(i0, i1, i2, i3, i4, i5) = val;
208  };
209 };
210 
211 template <class ViewType, class Layout, class ExecSpace, typename iType>
212 struct ViewFill<ViewType, Layout, ExecSpace, 7, iType> {
213  ViewType a;
214  typename ViewType::const_value_type val;
215 
216  using iterate_type = Kokkos::Rank<6, ViewFillLayoutSelector<Layout>::iterate,
217  ViewFillLayoutSelector<Layout>::iterate>;
218  using policy_type =
219  Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
220 
221  ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
222  const ExecSpace& space)
223  : a(a_), val(val_) {
224  // MDRangePolicy is not supported for 7D views
225  // Iterate separately over extent(2)
226  Kokkos::parallel_for("Kokkos::ViewFill-7D",
227  policy_type(space, {0, 0, 0, 0, 0, 0},
228  {a.extent(0), a.extent(1), a.extent(3),
229  a.extent(4), a.extent(5), a.extent(6)}),
230  *this);
231  }
232 
233  KOKKOS_INLINE_FUNCTION
234  void operator()(const iType& i0, const iType& i1, const iType& i3,
235  const iType& i4, const iType& i5, const iType& i6) const {
236  for (iType i2 = 0; i2 < iType(a.extent(2)); i2++)
237  a(i0, i1, i2, i3, i4, i5, i6) = val;
238  };
239 };
240 
241 template <class ViewType, class Layout, class ExecSpace, typename iType>
242 struct ViewFill<ViewType, Layout, ExecSpace, 8, iType> {
243  ViewType a;
244  typename ViewType::const_value_type val;
245 
246  using iterate_type = Kokkos::Rank<6, ViewFillLayoutSelector<Layout>::iterate,
247  ViewFillLayoutSelector<Layout>::iterate>;
248  using policy_type =
249  Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
250 
251  ViewFill(const ViewType& a_, typename ViewType::const_value_type& val_,
252  const ExecSpace& space)
253  : a(a_), val(val_) {
254  // MDRangePolicy is not supported for 8D views
255  // Iterate separately over extent(2) and extent(4)
256  Kokkos::parallel_for("Kokkos::ViewFill-8D",
257  policy_type(space, {0, 0, 0, 0, 0, 0},
258  {a.extent(0), a.extent(1), a.extent(3),
259  a.extent(5), a.extent(6), a.extent(7)}),
260  *this);
261  }
262 
263  KOKKOS_INLINE_FUNCTION
264  void operator()(const iType& i0, const iType& i1, const iType& i3,
265  const iType& i5, const iType& i6, const iType& i7) const {
266  for (iType i2 = 0; i2 < iType(a.extent(2)); i2++)
267  for (iType i4 = 0; i4 < iType(a.extent(4)); i4++)
268  a(i0, i1, i2, i3, i4, i5, i6, i7) = val;
269  };
270 };
271 
272 template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
273  typename iType>
274 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 1, iType> {
275  ViewTypeA a;
276  ViewTypeB b;
277 
278  using policy_type = Kokkos::RangePolicy<ExecSpace, Kokkos::IndexType<iType>>;
279  using value_type = typename ViewTypeA::value_type;
280 
281  ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
282  const ExecSpace space = ExecSpace())
283  : a(a_), b(b_) {
284  Kokkos::parallel_for("Kokkos::ViewCopy-1D",
285  policy_type(space, 0, a.extent(0)), *this);
286  }
287 
288  KOKKOS_INLINE_FUNCTION
289  void operator()(const iType& i0) const {
290  a(i0) = static_cast<value_type>(b(i0));
291  };
292 };
293 
294 template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
295  typename iType>
296 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 2, iType> {
297  ViewTypeA a;
298  ViewTypeB b;
299  static const Kokkos::Iterate outer_iteration_pattern =
300  Kokkos::Impl::layout_iterate_type_selector<
301  Layout>::outer_iteration_pattern;
302  static const Kokkos::Iterate inner_iteration_pattern =
303  Kokkos::Impl::layout_iterate_type_selector<
304  Layout>::inner_iteration_pattern;
305  using iterate_type =
306  Kokkos::Rank<2, outer_iteration_pattern, inner_iteration_pattern>;
307  using policy_type =
308  Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
309  using value_type = typename ViewTypeA::value_type;
310 
311  ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
312  const ExecSpace space = ExecSpace())
313  : a(a_), b(b_) {
314  Kokkos::parallel_for("Kokkos::ViewCopy-2D",
315  policy_type(space, {0, 0}, {a.extent(0), a.extent(1)}),
316  *this);
317  }
318 
319  KOKKOS_INLINE_FUNCTION
320  void operator()(const iType& i0, const iType& i1) const {
321  a(i0, i1) = static_cast<value_type>(b(i0, i1));
322  };
323 };
324 
325 template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
326  typename iType>
327 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 3, iType> {
328  ViewTypeA a;
329  ViewTypeB b;
330 
331  static const Kokkos::Iterate outer_iteration_pattern =
332  Kokkos::Impl::layout_iterate_type_selector<
333  Layout>::outer_iteration_pattern;
334  static const Kokkos::Iterate inner_iteration_pattern =
335  Kokkos::Impl::layout_iterate_type_selector<
336  Layout>::inner_iteration_pattern;
337  using iterate_type =
338  Kokkos::Rank<3, outer_iteration_pattern, inner_iteration_pattern>;
339  using policy_type =
340  Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
341  using value_type = typename ViewTypeA::value_type;
342 
343  ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
344  const ExecSpace space = ExecSpace())
345  : a(a_), b(b_) {
346  Kokkos::parallel_for(
347  "Kokkos::ViewCopy-3D",
348  policy_type(space, {0, 0, 0}, {a.extent(0), a.extent(1), a.extent(2)}),
349  *this);
350  }
351 
352  KOKKOS_INLINE_FUNCTION
353  void operator()(const iType& i0, const iType& i1, const iType& i2) const {
354  a(i0, i1, i2) = static_cast<value_type>(b(i0, i1, i2));
355  };
356 };
357 
358 template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
359  typename iType>
360 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 4, iType> {
361  ViewTypeA a;
362  ViewTypeB b;
363 
364  static const Kokkos::Iterate outer_iteration_pattern =
365  Kokkos::Impl::layout_iterate_type_selector<
366  Layout>::outer_iteration_pattern;
367  static const Kokkos::Iterate inner_iteration_pattern =
368  Kokkos::Impl::layout_iterate_type_selector<
369  Layout>::inner_iteration_pattern;
370  using iterate_type =
371  Kokkos::Rank<4, outer_iteration_pattern, inner_iteration_pattern>;
372  using policy_type =
373  Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
374 
375  ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
376  const ExecSpace space = ExecSpace())
377  : a(a_), b(b_) {
378  Kokkos::parallel_for(
379  "Kokkos::ViewCopy-4D",
380  policy_type(space, {0, 0, 0, 0},
381  {a.extent(0), a.extent(1), a.extent(2), a.extent(3)}),
382  *this);
383  }
384 
385  KOKKOS_INLINE_FUNCTION
386  void operator()(const iType& i0, const iType& i1, const iType& i2,
387  const iType& i3) const {
388  a(i0, i1, i2, i3) = b(i0, i1, i2, i3);
389  };
390 };
391 
392 template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
393  typename iType>
394 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 5, iType> {
395  ViewTypeA a;
396  ViewTypeB b;
397 
398  static const Kokkos::Iterate outer_iteration_pattern =
399  Kokkos::Impl::layout_iterate_type_selector<
400  Layout>::outer_iteration_pattern;
401  static const Kokkos::Iterate inner_iteration_pattern =
402  Kokkos::Impl::layout_iterate_type_selector<
403  Layout>::inner_iteration_pattern;
404  using iterate_type =
405  Kokkos::Rank<5, outer_iteration_pattern, inner_iteration_pattern>;
406  using policy_type =
407  Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
408 
409  ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
410  const ExecSpace space = ExecSpace())
411  : a(a_), b(b_) {
412  Kokkos::parallel_for("Kokkos::ViewCopy-5D",
413  policy_type(space, {0, 0, 0, 0, 0},
414  {a.extent(0), a.extent(1), a.extent(2),
415  a.extent(3), a.extent(4)}),
416  *this);
417  }
418 
419  KOKKOS_INLINE_FUNCTION
420  void operator()(const iType& i0, const iType& i1, const iType& i2,
421  const iType& i3, const iType& i4) const {
422  a(i0, i1, i2, i3, i4) = b(i0, i1, i2, i3, i4);
423  };
424 };
425 
426 template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
427  typename iType>
428 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 6, iType> {
429  ViewTypeA a;
430  ViewTypeB b;
431 
432  static const Kokkos::Iterate outer_iteration_pattern =
433  Kokkos::Impl::layout_iterate_type_selector<
434  Layout>::outer_iteration_pattern;
435  static const Kokkos::Iterate inner_iteration_pattern =
436  Kokkos::Impl::layout_iterate_type_selector<
437  Layout>::inner_iteration_pattern;
438  using iterate_type =
439  Kokkos::Rank<6, outer_iteration_pattern, inner_iteration_pattern>;
440  using policy_type =
441  Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
442 
443  ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
444  const ExecSpace space = ExecSpace())
445  : a(a_), b(b_) {
446  Kokkos::parallel_for("Kokkos::ViewCopy-6D",
447  policy_type(space, {0, 0, 0, 0, 0, 0},
448  {a.extent(0), a.extent(1), a.extent(2),
449  a.extent(3), a.extent(4), a.extent(5)}),
450  *this);
451  }
452 
453  KOKKOS_INLINE_FUNCTION
454  void operator()(const iType& i0, const iType& i1, const iType& i2,
455  const iType& i3, const iType& i4, const iType& i5) const {
456  a(i0, i1, i2, i3, i4, i5) = b(i0, i1, i2, i3, i4, i5);
457  };
458 };
459 
460 template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
461  typename iType>
462 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 7, iType> {
463  ViewTypeA a;
464  ViewTypeB b;
465 
466  static const Kokkos::Iterate outer_iteration_pattern =
467  Kokkos::Impl::layout_iterate_type_selector<
468  Layout>::outer_iteration_pattern;
469  static const Kokkos::Iterate inner_iteration_pattern =
470  Kokkos::Impl::layout_iterate_type_selector<
471  Layout>::inner_iteration_pattern;
472  using iterate_type =
473  Kokkos::Rank<6, outer_iteration_pattern, inner_iteration_pattern>;
474  using policy_type =
475  Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
476 
477  ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
478  const ExecSpace space = ExecSpace())
479  : a(a_), b(b_) {
480  // MDRangePolicy is not supported for 7D views
481  // Iterate separately over extent(2)
482  Kokkos::parallel_for("Kokkos::ViewCopy-7D",
483  policy_type(space, {0, 0, 0, 0, 0, 0},
484  {a.extent(0), a.extent(1), a.extent(3),
485  a.extent(4), a.extent(5), a.extent(6)}),
486  *this);
487  }
488 
489  KOKKOS_INLINE_FUNCTION
490  void operator()(const iType& i0, const iType& i1, const iType& i3,
491  const iType& i4, const iType& i5, const iType& i6) const {
492  for (iType i2 = 0; i2 < iType(a.extent(2)); i2++)
493  a(i0, i1, i2, i3, i4, i5, i6) = b(i0, i1, i2, i3, i4, i5, i6);
494  };
495 };
496 
497 template <class ViewTypeA, class ViewTypeB, class Layout, class ExecSpace,
498  typename iType>
499 struct ViewCopy<ViewTypeA, ViewTypeB, Layout, ExecSpace, 8, iType> {
500  ViewTypeA a;
501  ViewTypeB b;
502 
503  static const Kokkos::Iterate outer_iteration_pattern =
504  Kokkos::Impl::layout_iterate_type_selector<
505  Layout>::outer_iteration_pattern;
506  static const Kokkos::Iterate inner_iteration_pattern =
507  Kokkos::Impl::layout_iterate_type_selector<
508  Layout>::inner_iteration_pattern;
509  using iterate_type =
510  Kokkos::Rank<6, outer_iteration_pattern, inner_iteration_pattern>;
511  using policy_type =
512  Kokkos::MDRangePolicy<ExecSpace, iterate_type, Kokkos::IndexType<iType>>;
513 
514  ViewCopy(const ViewTypeA& a_, const ViewTypeB& b_,
515  const ExecSpace space = ExecSpace())
516  : a(a_), b(b_) {
517  // MDRangePolicy is not supported for 8D views
518  // Iterate separately over extent(2) and extent(4)
519  Kokkos::parallel_for("Kokkos::ViewCopy-8D",
520  policy_type(space, {0, 0, 0, 0, 0, 0},
521  {a.extent(0), a.extent(1), a.extent(3),
522  a.extent(5), a.extent(6), a.extent(7)}),
523  *this);
524  }
525 
526  KOKKOS_INLINE_FUNCTION
527  void operator()(const iType& i0, const iType& i1, const iType& i3,
528  const iType& i5, const iType& i6, const iType& i7) const {
529  for (iType i2 = 0; i2 < iType(a.extent(2)); i2++)
530  for (iType i4 = 0; i4 < iType(a.extent(4)); i4++)
531  a(i0, i1, i2, i3, i4, i5, i6, i7) = b(i0, i1, i2, i3, i4, i5, i6, i7);
532  };
533 };
534 
535 } // namespace Impl
536 } // namespace Kokkos
537 
538 namespace Kokkos {
539 namespace Impl {
540 
541 template <class DstType>
542 Kokkos::Iterate get_iteration_order(const DstType& dst) {
543  int64_t strides[DstType::rank + 1];
544  dst.stride(strides);
545  Kokkos::Iterate iterate;
546  if (std::is_same_v<typename DstType::array_layout, Kokkos::LayoutRight>) {
547  iterate = Kokkos::Iterate::Right;
548  } else if (std::is_same_v<typename DstType::array_layout,
549  Kokkos::LayoutLeft>) {
550  iterate = Kokkos::Iterate::Left;
551  } else if (std::is_same_v<typename DstType::array_layout,
552  Kokkos::LayoutStride>) {
553  if (strides[0] > strides[DstType::rank - 1])
554  iterate = Kokkos::Iterate::Right;
555  else
556  iterate = Kokkos::Iterate::Left;
557  } else {
558  if (std::is_same_v<typename DstType::execution_space::array_layout,
559  Kokkos::LayoutRight>)
560  iterate = Kokkos::Iterate::Right;
561  else
562  iterate = Kokkos::Iterate::Left;
563  }
564  return iterate;
565 }
566 
567 template <class ExecutionSpace, class DstType, class SrcType>
568 void view_copy(const ExecutionSpace& space, const DstType& dst,
569  const SrcType& src) {
570  using dst_memory_space = typename DstType::memory_space;
571  using src_memory_space = typename SrcType::memory_space;
572 
573  constexpr bool ExecCanAccessSrc =
574  Kokkos::SpaceAccessibility<ExecutionSpace, src_memory_space>::accessible;
575  constexpr bool ExecCanAccessDst =
576  Kokkos::SpaceAccessibility<ExecutionSpace, dst_memory_space>::accessible;
577 
578  if (!(ExecCanAccessSrc && ExecCanAccessDst)) {
579  Kokkos::Impl::throw_runtime_exception(
580  "Kokkos::Impl::view_copy called with invalid execution space");
581  } else {
582  // Figure out iteration order in case we need it
583  Kokkos::Iterate iterate = get_iteration_order(dst);
584 
585  if ((dst.span() >= size_t(std::numeric_limits<int>::max())) ||
586  (src.span() >= size_t(std::numeric_limits<int>::max()))) {
587  if (iterate == Kokkos::Iterate::Right)
588  Kokkos::Impl::ViewCopy<
589  typename DstType::uniform_runtime_nomemspace_type,
590  typename SrcType::uniform_runtime_const_nomemspace_type,
591  Kokkos::LayoutRight, ExecutionSpace, DstType::rank, int64_t>(
592  dst, src, space);
593  else
594  Kokkos::Impl::ViewCopy<
595  typename DstType::uniform_runtime_nomemspace_type,
596  typename SrcType::uniform_runtime_const_nomemspace_type,
597  Kokkos::LayoutLeft, ExecutionSpace, DstType::rank, int64_t>(
598  dst, src, space);
599  } else {
600  if (iterate == Kokkos::Iterate::Right)
601  Kokkos::Impl::ViewCopy<
602  typename DstType::uniform_runtime_nomemspace_type,
603  typename SrcType::uniform_runtime_const_nomemspace_type,
604  Kokkos::LayoutRight, ExecutionSpace, DstType::rank, int>(dst, src,
605  space);
606  else
607  Kokkos::Impl::ViewCopy<
608  typename DstType::uniform_runtime_nomemspace_type,
609  typename SrcType::uniform_runtime_const_nomemspace_type,
610  Kokkos::LayoutLeft, ExecutionSpace, DstType::rank, int>(dst, src,
611  space);
612  }
613  }
614 }
615 
616 template <class DstType, class SrcType>
617 void view_copy(const DstType& dst, const SrcType& src) {
618  using dst_execution_space = typename DstType::execution_space;
619  using src_execution_space = typename SrcType::execution_space;
620  using dst_memory_space = typename DstType::memory_space;
621  using src_memory_space = typename SrcType::memory_space;
622 
623  constexpr bool DstExecCanAccessSrc =
624  Kokkos::SpaceAccessibility<dst_execution_space,
625  src_memory_space>::accessible;
626 
627  constexpr bool SrcExecCanAccessDst =
628  Kokkos::SpaceAccessibility<src_execution_space,
629  dst_memory_space>::accessible;
630 
631  if (!DstExecCanAccessSrc && !SrcExecCanAccessDst) {
632  std::ostringstream ss;
633  ss << "Error: Kokkos::deep_copy with no available copy mechanism: "
634  << "from source view (\"" << src.label() << "\") to destination view (\""
635  << dst.label() << "\").\n"
636  << "There is no common execution space that can access both source's "
637  "space\n"
638  << "(" << src_memory_space().name() << ") and destination's space ("
639  << dst_memory_space().name() << "), "
640  << "so source and destination\n"
641  << "must be contiguous and have the same layout.\n";
642  Kokkos::Impl::throw_runtime_exception(ss.str());
643  }
644 
645  using ExecutionSpace =
646  std::conditional_t<DstExecCanAccessSrc, dst_execution_space,
647  src_execution_space>;
648 
649  // Figure out iteration order in case we need it
650  Kokkos::Iterate iterate = get_iteration_order(dst);
651 
652  if ((dst.span() >= size_t(std::numeric_limits<int>::max())) ||
653  (src.span() >= size_t(std::numeric_limits<int>::max()))) {
654  if (iterate == Kokkos::Iterate::Right)
655  Kokkos::Impl::ViewCopy<
656  typename DstType::uniform_runtime_nomemspace_type,
657  typename SrcType::uniform_runtime_const_nomemspace_type,
658  Kokkos::LayoutRight, ExecutionSpace, DstType::rank, int64_t>(dst,
659  src);
660  else
661  Kokkos::Impl::ViewCopy<
662  typename DstType::uniform_runtime_nomemspace_type,
663  typename SrcType::uniform_runtime_const_nomemspace_type,
664  Kokkos::LayoutLeft, ExecutionSpace, DstType::rank, int64_t>(dst, src);
665  } else {
666  if (iterate == Kokkos::Iterate::Right)
667  Kokkos::Impl::ViewCopy<
668  typename DstType::uniform_runtime_nomemspace_type,
669  typename SrcType::uniform_runtime_const_nomemspace_type,
670  Kokkos::LayoutRight, ExecutionSpace, DstType::rank, int>(dst, src);
671  else
672  Kokkos::Impl::ViewCopy<
673  typename DstType::uniform_runtime_nomemspace_type,
674  typename SrcType::uniform_runtime_const_nomemspace_type,
675  Kokkos::LayoutLeft, ExecutionSpace, DstType::rank, int>(dst, src);
676  }
677 }
678 
679 template <class DstType, class SrcType, class... Args>
680 struct CommonSubview {
681  using dst_subview_type = typename Kokkos::Subview<DstType, Args...>;
682  using src_subview_type = typename Kokkos::Subview<SrcType, Args...>;
683  dst_subview_type dst_sub;
684  src_subview_type src_sub;
685  CommonSubview(const DstType& dst, const SrcType& src, const Args&... args)
686  : dst_sub(dst, args...), src_sub(src, args...) {}
687 };
688 
689 template <class DstType, class SrcType, int Rank = DstType::rank>
690 struct ViewRemap;
691 
692 template <class DstType, class SrcType>
693 struct ViewRemap<DstType, SrcType, 1> {
694  using p_type = Kokkos::pair<int64_t, int64_t>;
695 
696  template <typename... OptExecSpace>
697  ViewRemap(const DstType& dst, const SrcType& src,
698  const OptExecSpace&... exec_space) {
699  static_assert(
700  sizeof...(OptExecSpace) <= 1,
701  "OptExecSpace must be either empty or be an execution space!");
702 
703  if (dst.extent(0) == src.extent(0)) {
704  view_copy(exec_space..., dst, src);
705  } else {
706  p_type ext0(0, std::min(dst.extent(0), src.extent(0)));
707  CommonSubview common_subview(dst, src, ext0);
708  view_copy(exec_space..., common_subview.dst_sub, common_subview.src_sub);
709  }
710  }
711 };
712 
713 template <class DstType, class SrcType, std::size_t... I>
714 auto create_common_subview_first_and_last_match(const DstType& dst,
715  const SrcType& src,
716  std::index_sequence<I...>) {
717  using p_type = Kokkos::pair<int64_t, int64_t>;
718  CommonSubview common_subview(
719  dst, src, Kokkos::ALL,
720  (p_type(0, std::min(dst.extent(I + 1), src.extent(I + 1))))...,
721  Kokkos::ALL);
722  return common_subview;
723 }
724 
725 template <class DstType, class SrcType, std::size_t... I>
726 auto create_common_subview_first_match(const DstType& dst, const SrcType& src,
727  std::index_sequence<I...>) {
728  using p_type = Kokkos::pair<int64_t, int64_t>;
729  CommonSubview common_subview(
730  dst, src, Kokkos::ALL,
731  (p_type(0, std::min(dst.extent(I + 1), src.extent(I + 1))))...);
732  return common_subview;
733 }
734 
735 template <class DstType, class SrcType, std::size_t... I>
736 auto create_common_subview_last_match(const DstType& dst, const SrcType& src,
737  std::index_sequence<I...>) {
738  using p_type = Kokkos::pair<int64_t, int64_t>;
739  CommonSubview common_subview(
740  dst, src, (p_type(0, std::min(dst.extent(I), src.extent(I))))...,
741  Kokkos::ALL);
742  return common_subview;
743 }
744 
745 template <class DstType, class SrcType, std::size_t... I>
746 auto create_common_subview_no_match(const DstType& dst, const SrcType& src,
747  std::index_sequence<I...>) {
748  using p_type = Kokkos::pair<int64_t, int64_t>;
749  CommonSubview common_subview(
750  dst, src, (p_type(0, std::min(dst.extent(I), src.extent(I))))...);
751  return common_subview;
752 }
753 
754 template <class DstType, class SrcType, int Rank>
755 struct ViewRemap {
756  using p_type = Kokkos::pair<int64_t, int64_t>;
757 
758  template <typename... OptExecSpace>
759  ViewRemap(const DstType& dst, const SrcType& src,
760  const OptExecSpace&... exec_space) {
761  static_assert(
762  sizeof...(OptExecSpace) <= 1,
763  "OptExecSpace must be either empty or be an execution space!");
764 
765  if (dst.extent(0) == src.extent(0)) {
766  if (dst.extent(Rank - 1) == src.extent(Rank - 1)) {
767  if constexpr (Rank < 3)
768  view_copy(exec_space..., dst, src);
769  else {
770  auto common_subview = create_common_subview_first_and_last_match(
771  dst, src, std::make_index_sequence<Rank - 2>{});
772  view_copy(exec_space..., common_subview.dst_sub,
773  common_subview.src_sub);
774  }
775  } else {
776  auto common_subview = create_common_subview_first_match(
777  dst, src, std::make_index_sequence<Rank - 1>{});
778  view_copy(exec_space..., common_subview.dst_sub,
779  common_subview.src_sub);
780  }
781  } else {
782  if (dst.extent(Rank - 1) == src.extent(Rank - 1)) {
783  auto common_subview = create_common_subview_last_match(
784  dst, src, std::make_index_sequence<Rank - 1>{});
785  view_copy(exec_space..., common_subview.dst_sub,
786  common_subview.src_sub);
787  } else {
788  auto common_subview = create_common_subview_no_match(
789  dst, src, std::make_index_sequence<Rank>{});
790  view_copy(exec_space..., common_subview.dst_sub,
791  common_subview.src_sub);
792  }
793  }
794  }
795 };
796 
797 template <typename ExecutionSpace, class DT, class... DP>
798 inline void contiguous_fill(
799  const ExecutionSpace& exec_space, const View<DT, DP...>& dst,
800  typename ViewTraits<DT, DP...>::const_value_type& value) {
801  using ViewType = View<DT, DP...>;
802  using ViewTypeFlat = Kokkos::View<
803  typename ViewType::value_type*, Kokkos::LayoutRight,
804  Kokkos::Device<typename ViewType::execution_space,
805  std::conditional_t<ViewType::rank == 0,
806  typename ViewType::memory_space,
807  Kokkos::AnonymousSpace>>,
808  Kokkos::MemoryTraits<0>>;
809 
810  ViewTypeFlat dst_flat(dst.data(), dst.size());
811  if (dst.span() < static_cast<size_t>(std::numeric_limits<int>::max())) {
812  Kokkos::Impl::ViewFill<ViewTypeFlat, Kokkos::LayoutRight, ExecutionSpace,
813  ViewTypeFlat::rank, int>(dst_flat, value,
814  exec_space);
815  } else
816  Kokkos::Impl::ViewFill<ViewTypeFlat, Kokkos::LayoutRight, ExecutionSpace,
817  ViewTypeFlat::rank, int64_t>(dst_flat, value,
818  exec_space);
819 }
820 
821 // Default implementation for execution spaces that don't provide a definition
822 template <typename ExecutionSpace>
823 struct ZeroMemset {
824  ZeroMemset(const ExecutionSpace& exec_space, void* dst, size_t cnt) {
825  contiguous_fill(
826  exec_space,
827  Kokkos::View<std::byte*, ExecutionSpace, Kokkos::MemoryUnmanaged>(
828  static_cast<std::byte*>(dst), cnt),
829  std::byte{});
830  }
831 };
832 
833 template <typename ExecutionSpace, class DT, class... DP>
834 inline std::enable_if_t<
835  std::is_trivial_v<typename ViewTraits<DT, DP...>::value_type>>
836 contiguous_fill_or_memset(
837  const ExecutionSpace& exec_space, const View<DT, DP...>& dst,
838  typename ViewTraits<DT, DP...>::const_value_type& value) {
839  // With OpenMP, using memset has significant performance issues.
840  if (Impl::is_zero_byte(value)
841 #ifdef KOKKOS_ENABLE_OPENMP
842  && !std::is_same_v<ExecutionSpace, Kokkos::OpenMP>
843 #endif
844  )
845  // FIXME intel/19 icpc fails to deduce template parameter here,
846  // resulting in compilation errors; explicitly passing the template
847  // parameter to ZeroMemset helps workaround the issue.
848  // See https://github.com/kokkos/kokkos/issues/7273.
849  ZeroMemset<ExecutionSpace>(
850  exec_space, dst.data(),
851  dst.size() * sizeof(typename ViewTraits<DT, DP...>::value_type));
852  else
853  contiguous_fill(exec_space, dst, value);
854 }
855 
856 template <typename ExecutionSpace, class DT, class... DP>
857 inline std::enable_if_t<
858  !std::is_trivial_v<typename ViewTraits<DT, DP...>::value_type>>
859 contiguous_fill_or_memset(
860  const ExecutionSpace& exec_space, const View<DT, DP...>& dst,
861  typename ViewTraits<DT, DP...>::const_value_type& value) {
862  contiguous_fill(exec_space, dst, value);
863 }
864 
865 template <class DT, class... DP>
866 inline std::enable_if_t<
867  std::is_trivial_v<typename ViewTraits<DT, DP...>::value_type>>
868 contiguous_fill_or_memset(
869  const View<DT, DP...>& dst,
870  typename ViewTraits<DT, DP...>::const_value_type& value) {
871  using ViewType = View<DT, DP...>;
872  using exec_space_type = typename ViewType::execution_space;
873  exec_space_type exec;
874 
875 // On A64FX memset seems to do the wrong thing with regards to first touch
876 // leading to the significant performance issues
877 #ifndef KOKKOS_ARCH_A64FX
878  if (Impl::is_zero_byte(value))
879  // FIXME intel/19 icpc fails to deduce template parameter here,
880  // resulting in compilation errors; explicitly passing the template
881  // parameter to ZeroMemset helps workaround the issue.
882  // See https://github.com/kokkos/kokkos/issues/7273.
883  ZeroMemset<exec_space_type>(
884  exec, dst.data(), dst.size() * sizeof(typename ViewType::value_type));
885  else
886 #endif
887  contiguous_fill(exec, dst, value);
888 }
889 
890 template <class DT, class... DP>
891 inline std::enable_if_t<
892  !std::is_trivial_v<typename ViewTraits<DT, DP...>::value_type>>
893 contiguous_fill_or_memset(
894  const View<DT, DP...>& dst,
895  typename ViewTraits<DT, DP...>::const_value_type& value) {
896  using ViewType = View<DT, DP...>;
897  using exec_space_type = typename ViewType::execution_space;
898 
899  contiguous_fill(exec_space_type(), dst, value);
900 }
901 } // namespace Impl
902 
904 template <class DT, class... DP>
905 inline void deep_copy(
906  const View<DT, DP...>& dst,
907  typename ViewTraits<DT, DP...>::const_value_type& value,
908  std::enable_if_t<std::is_same_v<typename ViewTraits<DT, DP...>::specialize,
909  void>>* = nullptr) {
910  using ViewType = View<DT, DP...>;
911  using exec_space_type = typename ViewType::execution_space;
912 
913  if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
914  Kokkos::Profiling::beginDeepCopy(
915  Kokkos::Profiling::make_space_handle(ViewType::memory_space::name()),
916  dst.label(), dst.data(),
917  Kokkos::Profiling::make_space_handle(Kokkos::HostSpace::name()),
918  "Scalar", &value, dst.span() * sizeof(typename ViewType::value_type));
919  }
920 
921  if (dst.data() == nullptr) {
922  Kokkos::fence(
923  "Kokkos::deep_copy: scalar copy, fence because destination is null");
924  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
925  Kokkos::Profiling::endDeepCopy();
926  }
927  return;
928  }
929 
930  Kokkos::fence("Kokkos::deep_copy: scalar copy, pre copy fence");
931  static_assert(std::is_same_v<typename ViewType::non_const_value_type,
932  typename ViewType::value_type>,
933  "deep_copy requires non-const type");
934 
935  // If contiguous we can simply do a 1D flat loop or use memset
936  if (dst.span_is_contiguous()) {
937  Impl::contiguous_fill_or_memset(dst, value);
938  Kokkos::fence("Kokkos::deep_copy: scalar copy, post copy fence");
939  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
940  Kokkos::Profiling::endDeepCopy();
941  }
942  return;
943  }
944 
945  // Figure out iteration order to do the ViewFill
946  int64_t strides[ViewType::rank + 1];
947  dst.stride(strides);
948  Kokkos::Iterate iterate;
949  if (std::is_same_v<typename ViewType::array_layout, Kokkos::LayoutRight>) {
950  iterate = Kokkos::Iterate::Right;
951  } else if (std::is_same_v<typename ViewType::array_layout,
952  Kokkos::LayoutLeft>) {
953  iterate = Kokkos::Iterate::Left;
954  } else if (std::is_same_v<typename ViewType::array_layout,
955  Kokkos::LayoutStride>) {
956  if (strides[0] > strides[ViewType::rank > 0 ? ViewType::rank - 1 : 0])
957  iterate = Kokkos::Iterate::Right;
958  else
959  iterate = Kokkos::Iterate::Left;
960  } else {
961  if (std::is_same_v<typename ViewType::execution_space::array_layout,
962  Kokkos::LayoutRight>)
963  iterate = Kokkos::Iterate::Right;
964  else
965  iterate = Kokkos::Iterate::Left;
966  }
967 
968  // Lets call the right ViewFill functor based on integer space needed and
969  // iteration type
970  using ViewTypeUniform =
971  std::conditional_t<ViewType::rank == 0,
972  typename ViewType::uniform_runtime_type,
973  typename ViewType::uniform_runtime_nomemspace_type>;
974  if (dst.span() > static_cast<size_t>(std::numeric_limits<int>::max())) {
975  if (iterate == Kokkos::Iterate::Right)
976  Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutRight,
977  exec_space_type, ViewType::rank, int64_t>(
978  dst, value, exec_space_type());
979  else
980  Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutLeft,
981  exec_space_type, ViewType::rank, int64_t>(
982  dst, value, exec_space_type());
983  } else {
984  if (iterate == Kokkos::Iterate::Right)
985  Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutRight,
986  exec_space_type, ViewType::rank, int>(
987  dst, value, exec_space_type());
988  else
989  Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutLeft,
990  exec_space_type, ViewType::rank, int>(
991  dst, value, exec_space_type());
992  }
993  Kokkos::fence("Kokkos::deep_copy: scalar copy, post copy fence");
994 
995  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
996  Kokkos::Profiling::endDeepCopy();
997  }
998 }
999 
1001 template <class ST, class... SP>
1002 inline void deep_copy(
1003  typename ViewTraits<ST, SP...>::non_const_value_type& dst,
1004  const View<ST, SP...>& src,
1005  std::enable_if_t<std::is_same_v<typename ViewTraits<ST, SP...>::specialize,
1006  void>>* = nullptr) {
1007  using src_traits = ViewTraits<ST, SP...>;
1008  using src_memory_space = typename src_traits::memory_space;
1009 
1010  static_assert(src_traits::rank == 0,
1011  "ERROR: Non-rank-zero view in deep_copy( value , View )");
1012 
1013  if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
1014  Kokkos::Profiling::beginDeepCopy(
1015  Kokkos::Profiling::make_space_handle(Kokkos::HostSpace::name()),
1016  "Scalar", &dst,
1017  Kokkos::Profiling::make_space_handle(src_memory_space::name()),
1018  src.label(), src.data(),
1019  src.span() * sizeof(typename src_traits::value_type));
1020  }
1021 
1022  if (src.data() == nullptr) {
1023  Kokkos::fence("Kokkos::deep_copy: copy into scalar, src is null");
1024  } else {
1025  Kokkos::fence("Kokkos::deep_copy: copy into scalar, pre copy fence");
1026  Kokkos::Impl::DeepCopy<HostSpace, src_memory_space>(&dst, src.data(),
1027  sizeof(ST));
1028  Kokkos::fence("Kokkos::deep_copy: copy into scalar, post copy fence");
1029  }
1030 
1031  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
1032  Kokkos::Profiling::endDeepCopy();
1033  }
1034 }
1035 
1036 //----------------------------------------------------------------------------
1038 template <class DT, class... DP, class ST, class... SP>
1039 inline void deep_copy(
1040  const View<DT, DP...>& dst, const View<ST, SP...>& src,
1041  std::enable_if_t<
1042  (std::is_void_v<typename ViewTraits<DT, DP...>::specialize> &&
1043  std::is_void_v<typename ViewTraits<ST, SP...>::specialize> &&
1044  (unsigned(ViewTraits<DT, DP...>::rank) == unsigned(0) &&
1045  unsigned(ViewTraits<ST, SP...>::rank) == unsigned(0)))>* = nullptr) {
1046  using dst_type = View<DT, DP...>;
1047  using src_type = View<ST, SP...>;
1048 
1049  using value_type = typename dst_type::value_type;
1050  using dst_memory_space = typename dst_type::memory_space;
1051  using src_memory_space = typename src_type::memory_space;
1052 
1053  static_assert(std::is_same_v<typename dst_type::value_type,
1054  typename src_type::non_const_value_type>,
1055  "deep_copy requires matching non-const destination type");
1056 
1057  if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
1058  Kokkos::Profiling::beginDeepCopy(
1059  Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
1060  dst.label(), dst.data(),
1061  Kokkos::Profiling::make_space_handle(src_memory_space::name()),
1062  src.label(), src.data(),
1063  src.span() * sizeof(typename dst_type::value_type));
1064  }
1065 
1066  if (dst.data() == nullptr && src.data() == nullptr) {
1067  Kokkos::fence(
1068  "Kokkos::deep_copy: scalar to scalar copy, both pointers null");
1069  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
1070  Kokkos::Profiling::endDeepCopy();
1071  }
1072  return;
1073  }
1074 
1075  Kokkos::fence("Kokkos::deep_copy: scalar to scalar copy, pre copy fence");
1076  if (dst.data() != src.data()) {
1077  Kokkos::Impl::DeepCopy<dst_memory_space, src_memory_space>(
1078  dst.data(), src.data(), sizeof(value_type));
1079  Kokkos::fence("Kokkos::deep_copy: scalar to scalar copy, post copy fence");
1080  }
1081  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
1082  Kokkos::Profiling::endDeepCopy();
1083  }
1084 }
1085 
1086 //----------------------------------------------------------------------------
1090 template <class DT, class... DP, class ST, class... SP>
1091 inline void deep_copy(
1092  const View<DT, DP...>& dst, const View<ST, SP...>& src,
1093  std::enable_if_t<
1094  (std::is_void_v<typename ViewTraits<DT, DP...>::specialize> &&
1095  std::is_void_v<typename ViewTraits<ST, SP...>::specialize> &&
1096  (unsigned(ViewTraits<DT, DP...>::rank) != 0 ||
1097  unsigned(ViewTraits<ST, SP...>::rank) != 0))>* = nullptr) {
1098  using dst_type = View<DT, DP...>;
1099  using src_type = View<ST, SP...>;
1100  using dst_memory_space = typename dst_type::memory_space;
1101  using src_memory_space = typename src_type::memory_space;
1102  using dst_value_type = typename dst_type::value_type;
1103  using src_value_type = typename src_type::value_type;
1104 
1105  static_assert(std::is_same_v<typename dst_type::value_type,
1106  typename dst_type::non_const_value_type>,
1107  "deep_copy requires non-const destination type");
1108 
1109  static_assert((unsigned(dst_type::rank) == unsigned(src_type::rank)),
1110  "deep_copy requires Views of equal rank");
1111 
1112  if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
1113  Kokkos::Profiling::beginDeepCopy(
1114  Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
1115  dst.label(), dst.data(),
1116  Kokkos::Profiling::make_space_handle(src_memory_space::name()),
1117  src.label(), src.data(),
1118  src.span() * sizeof(typename dst_type::value_type));
1119  }
1120 
1121  if (dst.data() == nullptr || src.data() == nullptr) {
1122  // throw if dimension mismatch
1123  if ((src.extent(0) != dst.extent(0)) || (src.extent(1) != dst.extent(1)) ||
1124  (src.extent(2) != dst.extent(2)) || (src.extent(3) != dst.extent(3)) ||
1125  (src.extent(4) != dst.extent(4)) || (src.extent(5) != dst.extent(5)) ||
1126  (src.extent(6) != dst.extent(6)) || (src.extent(7) != dst.extent(7))) {
1127  std::string message(
1128  "Deprecation Error: Kokkos::deep_copy extents of views don't "
1129  "match: ");
1130  message += dst.label();
1131  message += "(";
1132  message += std::to_string(dst.extent(0));
1133  for (size_t r = 1; r < dst_type::rank; r++) {
1134  message += ",";
1135  message += std::to_string(dst.extent(r));
1136  }
1137  message += ") ";
1138  message += src.label();
1139  message += "(";
1140  message += std::to_string(src.extent(0));
1141  for (size_t r = 1; r < src_type::rank; r++) {
1142  message += ",";
1143  message += std::to_string(src.extent(r));
1144  }
1145  message += ") ";
1146 
1147  Kokkos::Impl::throw_runtime_exception(message);
1148  }
1149  Kokkos::fence(
1150  "Kokkos::deep_copy: copy between contiguous views, fence due to null "
1151  "argument");
1152  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
1153  Kokkos::Profiling::endDeepCopy();
1154  }
1155  return;
1156  }
1157 
1158  // Checking for Overlapping Views.
1159  dst_value_type* dst_start = dst.data();
1160  dst_value_type* dst_end = dst.data() + dst.span();
1161  src_value_type* src_start = src.data();
1162  src_value_type* src_end = src.data() + src.span();
1163  if (((std::ptrdiff_t)dst_start == (std::ptrdiff_t)src_start) &&
1164  ((std::ptrdiff_t)dst_end == (std::ptrdiff_t)src_end) &&
1165  (dst.span_is_contiguous() && src.span_is_contiguous())) {
1166  Kokkos::fence(
1167  "Kokkos::deep_copy: copy between contiguous views, fence due to same "
1168  "spans");
1169  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
1170  Kokkos::Profiling::endDeepCopy();
1171  }
1172  return;
1173  }
1174 
1175  if ((((std::ptrdiff_t)dst_start < (std::ptrdiff_t)src_end) &&
1176  ((std::ptrdiff_t)dst_end > (std::ptrdiff_t)src_start)) &&
1177  ((dst.span_is_contiguous() && src.span_is_contiguous()))) {
1178  std::string message("Error: Kokkos::deep_copy of overlapping views: ");
1179  message += dst.label();
1180  message += "(";
1181  message += std::to_string((std::ptrdiff_t)dst_start);
1182  message += ",";
1183  message += std::to_string((std::ptrdiff_t)dst_end);
1184  message += ") ";
1185  message += src.label();
1186  message += "(";
1187  message += std::to_string((std::ptrdiff_t)src_start);
1188  message += ",";
1189  message += std::to_string((std::ptrdiff_t)src_end);
1190  message += ") ";
1191  Kokkos::Impl::throw_runtime_exception(message);
1192  }
1193 
1194  // Check for same extents
1195  if ((src.extent(0) != dst.extent(0)) || (src.extent(1) != dst.extent(1)) ||
1196  (src.extent(2) != dst.extent(2)) || (src.extent(3) != dst.extent(3)) ||
1197  (src.extent(4) != dst.extent(4)) || (src.extent(5) != dst.extent(5)) ||
1198  (src.extent(6) != dst.extent(6)) || (src.extent(7) != dst.extent(7))) {
1199  std::string message(
1200  "Deprecation Error: Kokkos::deep_copy extents of views don't match: ");
1201  message += dst.label();
1202  message += "(";
1203  message += std::to_string(dst.extent(0));
1204  for (size_t r = 1; r < dst_type::rank; r++) {
1205  message += ",";
1206  message += std::to_string(dst.extent(r));
1207  }
1208  message += ") ";
1209  message += src.label();
1210  message += "(";
1211  message += std::to_string(src.extent(0));
1212  for (size_t r = 1; r < src_type::rank; r++) {
1213  message += ",";
1214  message += std::to_string(src.extent(r));
1215  }
1216  message += ") ";
1217 
1218  Kokkos::Impl::throw_runtime_exception(message);
1219  }
1220 
1221  // If same type, equal layout, equal dimensions, equal span, and contiguous
1222  // memory then can byte-wise copy
1223 
1224  if (std::is_same_v<typename dst_type::value_type,
1225  typename src_type::non_const_value_type> &&
1226  (std::is_same_v<typename dst_type::array_layout,
1227  typename src_type::array_layout> ||
1228  (dst_type::rank == 1 && src_type::rank == 1)) &&
1229  dst.span_is_contiguous() && src.span_is_contiguous() &&
1230  ((dst_type::rank < 1) || (dst.stride_0() == src.stride_0())) &&
1231  ((dst_type::rank < 2) || (dst.stride_1() == src.stride_1())) &&
1232  ((dst_type::rank < 3) || (dst.stride_2() == src.stride_2())) &&
1233  ((dst_type::rank < 4) || (dst.stride_3() == src.stride_3())) &&
1234  ((dst_type::rank < 5) || (dst.stride_4() == src.stride_4())) &&
1235  ((dst_type::rank < 6) || (dst.stride_5() == src.stride_5())) &&
1236  ((dst_type::rank < 7) || (dst.stride_6() == src.stride_6())) &&
1237  ((dst_type::rank < 8) || (dst.stride_7() == src.stride_7()))) {
1238  const size_t nbytes = sizeof(typename dst_type::value_type) * dst.span();
1239  Kokkos::fence(
1240  "Kokkos::deep_copy: copy between contiguous views, pre view equality "
1241  "check");
1242  if ((void*)dst.data() != (void*)src.data() && 0 < nbytes) {
1243  Kokkos::Impl::DeepCopy<dst_memory_space, src_memory_space>(
1244  dst.data(), src.data(), nbytes);
1245  Kokkos::fence(
1246  "Kokkos::deep_copy: copy between contiguous views, post deep copy "
1247  "fence");
1248  }
1249  } else {
1250  Kokkos::fence(
1251  "Kokkos::deep_copy: copy between contiguous views, pre copy fence");
1252  Impl::view_copy(dst, src);
1253  Kokkos::fence(
1254  "Kokkos::deep_copy: copy between contiguous views, post copy fence");
1255  }
1256  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
1257  Kokkos::Profiling::endDeepCopy();
1258  }
1259 }
1260 
1261 //----------------------------------------------------------------------------
1262 //----------------------------------------------------------------------------
1263 namespace Experimental {
1267 template <class TeamType, class DT, class... DP, class ST, class... SP>
1268 void KOKKOS_INLINE_FUNCTION
1269 local_deep_copy_contiguous(const TeamType& team, const View<DT, DP...>& dst,
1270  const View<ST, SP...>& src) {
1271  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, src.span()),
1272  [&](const int& i) { dst.data()[i] = src.data()[i]; });
1273 }
1274 //----------------------------------------------------------------------------
1275 template <class DT, class... DP, class ST, class... SP>
1276 void KOKKOS_INLINE_FUNCTION local_deep_copy_contiguous(
1277  const View<DT, DP...>& dst, const View<ST, SP...>& src) {
1278  for (size_t i = 0; i < src.span(); ++i) {
1279  dst.data()[i] = src.data()[i];
1280  }
1281 }
1282 //----------------------------------------------------------------------------
1283 template <class TeamType, class DT, class... DP, class ST, class... SP>
1284 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1285  const TeamType& team, const View<DT, DP...>& dst,
1286  const View<ST, SP...>& src,
1287  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 1 &&
1288  unsigned(ViewTraits<ST, SP...>::rank) == 1)>* = nullptr) {
1289  if (dst.data() == nullptr) {
1290  return;
1291  }
1292 
1293  const size_t N = dst.extent(0);
1294 
1295  team.team_barrier();
1296  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N),
1297  [&](const int& i) { dst(i) = src(i); });
1298  team.team_barrier();
1299 }
1300 //----------------------------------------------------------------------------
1301 template <class TeamType, class DT, class... DP, class ST, class... SP>
1302 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1303  const TeamType& team, const View<DT, DP...>& dst,
1304  const View<ST, SP...>& src,
1305  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 2 &&
1306  unsigned(ViewTraits<ST, SP...>::rank) == 2)>* = nullptr) {
1307  if (dst.data() == nullptr) {
1308  return;
1309  }
1310 
1311  const size_t N = dst.extent(0) * dst.extent(1);
1312 
1313  if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1314  team.team_barrier();
1315  local_deep_copy_contiguous(team, dst, src);
1316  team.team_barrier();
1317  } else {
1318  team.team_barrier();
1319  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1320  int i0 = i % dst.extent(0);
1321  int i1 = i / dst.extent(0);
1322  dst(i0, i1) = src(i0, i1);
1323  });
1324  team.team_barrier();
1325  }
1326 }
1327 //----------------------------------------------------------------------------
1328 template <class TeamType, class DT, class... DP, class ST, class... SP>
1329 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1330  const TeamType& team, const View<DT, DP...>& dst,
1331  const View<ST, SP...>& src,
1332  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 3 &&
1333  unsigned(ViewTraits<ST, SP...>::rank) == 3)>* = nullptr) {
1334  if (dst.data() == nullptr) {
1335  return;
1336  }
1337 
1338  const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2);
1339 
1340  if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1341  team.team_barrier();
1342  local_deep_copy_contiguous(team, dst, src);
1343  team.team_barrier();
1344  } else {
1345  team.team_barrier();
1346  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1347  int i0 = i % dst.extent(0);
1348  int itmp = i / dst.extent(0);
1349  int i1 = itmp % dst.extent(1);
1350  int i2 = itmp / dst.extent(1);
1351  dst(i0, i1, i2) = src(i0, i1, i2);
1352  });
1353  team.team_barrier();
1354  }
1355 }
1356 //----------------------------------------------------------------------------
1357 template <class TeamType, class DT, class... DP, class ST, class... SP>
1358 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1359  const TeamType& team, const View<DT, DP...>& dst,
1360  const View<ST, SP...>& src,
1361  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 4 &&
1362  unsigned(ViewTraits<ST, SP...>::rank) == 4)>* = nullptr) {
1363  if (dst.data() == nullptr) {
1364  return;
1365  }
1366 
1367  const size_t N =
1368  dst.extent(0) * dst.extent(1) * dst.extent(2) * dst.extent(3);
1369 
1370  if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1371  team.team_barrier();
1372  local_deep_copy_contiguous(team, dst, src);
1373  team.team_barrier();
1374  } else {
1375  team.team_barrier();
1376  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1377  int i0 = i % dst.extent(0);
1378  int itmp = i / dst.extent(0);
1379  int i1 = itmp % dst.extent(1);
1380  itmp = itmp / dst.extent(1);
1381  int i2 = itmp % dst.extent(2);
1382  int i3 = itmp / dst.extent(2);
1383  dst(i0, i1, i2, i3) = src(i0, i1, i2, i3);
1384  });
1385  team.team_barrier();
1386  }
1387 }
1388 //----------------------------------------------------------------------------
1389 template <class TeamType, class DT, class... DP, class ST, class... SP>
1390 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1391  const TeamType& team, const View<DT, DP...>& dst,
1392  const View<ST, SP...>& src,
1393  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 5 &&
1394  unsigned(ViewTraits<ST, SP...>::rank) == 5)>* = nullptr) {
1395  if (dst.data() == nullptr) {
1396  return;
1397  }
1398 
1399  const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1400  dst.extent(3) * dst.extent(4);
1401 
1402  if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1403  team.team_barrier();
1404  local_deep_copy_contiguous(team, dst, src);
1405  team.team_barrier();
1406  } else {
1407  team.team_barrier();
1408  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1409  int i0 = i % dst.extent(0);
1410  int itmp = i / dst.extent(0);
1411  int i1 = itmp % dst.extent(1);
1412  itmp = itmp / dst.extent(1);
1413  int i2 = itmp % dst.extent(2);
1414  itmp = itmp / dst.extent(2);
1415  int i3 = itmp % dst.extent(3);
1416  int i4 = itmp / dst.extent(3);
1417  dst(i0, i1, i2, i3, i4) = src(i0, i1, i2, i3, i4);
1418  });
1419  team.team_barrier();
1420  }
1421 }
1422 //----------------------------------------------------------------------------
1423 template <class TeamType, class DT, class... DP, class ST, class... SP>
1424 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1425  const TeamType& team, const View<DT, DP...>& dst,
1426  const View<ST, SP...>& src,
1427  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 6 &&
1428  unsigned(ViewTraits<ST, SP...>::rank) == 6)>* = nullptr) {
1429  if (dst.data() == nullptr) {
1430  return;
1431  }
1432 
1433  const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1434  dst.extent(3) * dst.extent(4) * dst.extent(5);
1435 
1436  if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1437  team.team_barrier();
1438  local_deep_copy_contiguous(team, dst, src);
1439  team.team_barrier();
1440  } else {
1441  team.team_barrier();
1442  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1443  int i0 = i % dst.extent(0);
1444  int itmp = i / dst.extent(0);
1445  int i1 = itmp % dst.extent(1);
1446  itmp = itmp / dst.extent(1);
1447  int i2 = itmp % dst.extent(2);
1448  itmp = itmp / dst.extent(2);
1449  int i3 = itmp % dst.extent(3);
1450  itmp = itmp / dst.extent(3);
1451  int i4 = itmp % dst.extent(4);
1452  int i5 = itmp / dst.extent(4);
1453  dst(i0, i1, i2, i3, i4, i5) = src(i0, i1, i2, i3, i4, i5);
1454  });
1455  team.team_barrier();
1456  }
1457 }
1458 //----------------------------------------------------------------------------
1459 template <class TeamType, class DT, class... DP, class ST, class... SP>
1460 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1461  const TeamType& team, const View<DT, DP...>& dst,
1462  const View<ST, SP...>& src,
1463  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 7 &&
1464  unsigned(ViewTraits<ST, SP...>::rank) == 7)>* = nullptr) {
1465  if (dst.data() == nullptr) {
1466  return;
1467  }
1468 
1469  const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1470  dst.extent(3) * dst.extent(4) * dst.extent(5) *
1471  dst.extent(6);
1472 
1473  if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1474  team.team_barrier();
1475  local_deep_copy_contiguous(team, dst, src);
1476  team.team_barrier();
1477  } else {
1478  team.team_barrier();
1479  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1480  int i0 = i % dst.extent(0);
1481  int itmp = i / dst.extent(0);
1482  int i1 = itmp % dst.extent(1);
1483  itmp = itmp / dst.extent(1);
1484  int i2 = itmp % dst.extent(2);
1485  itmp = itmp / dst.extent(2);
1486  int i3 = itmp % dst.extent(3);
1487  itmp = itmp / dst.extent(3);
1488  int i4 = itmp % dst.extent(4);
1489  itmp = itmp / dst.extent(4);
1490  int i5 = itmp % dst.extent(5);
1491  int i6 = itmp / dst.extent(5);
1492  dst(i0, i1, i2, i3, i4, i5, i6) = src(i0, i1, i2, i3, i4, i5, i6);
1493  });
1494  team.team_barrier();
1495  }
1496 }
1497 //----------------------------------------------------------------------------
1498 template <class DT, class... DP, class ST, class... SP>
1499 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1500  const View<DT, DP...>& dst, const View<ST, SP...>& src,
1501  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 1 &&
1502  unsigned(ViewTraits<ST, SP...>::rank) == 1)>* = nullptr) {
1503  if (dst.data() == nullptr) {
1504  return;
1505  }
1506 
1507  const size_t N = dst.extent(0);
1508 
1509  for (size_t i = 0; i < N; ++i) {
1510  dst(i) = src(i);
1511  }
1512 }
1513 //----------------------------------------------------------------------------
1514 template <class DT, class... DP, class ST, class... SP>
1515 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1516  const View<DT, DP...>& dst, const View<ST, SP...>& src,
1517  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 2 &&
1518  unsigned(ViewTraits<ST, SP...>::rank) == 2)>* = nullptr) {
1519  if (dst.data() == nullptr) {
1520  return;
1521  }
1522 
1523  if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1524  local_deep_copy_contiguous(dst, src);
1525  } else {
1526  for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1527  for (size_t i1 = 0; i1 < dst.extent(1); ++i1) dst(i0, i1) = src(i0, i1);
1528  }
1529 }
1530 //----------------------------------------------------------------------------
1531 template <class DT, class... DP, class ST, class... SP>
1532 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1533  const View<DT, DP...>& dst, const View<ST, SP...>& src,
1534  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 3 &&
1535  unsigned(ViewTraits<ST, SP...>::rank) == 3)>* = nullptr) {
1536  if (dst.data() == nullptr) {
1537  return;
1538  }
1539 
1540  if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1541  local_deep_copy_contiguous(dst, src);
1542  } else {
1543  for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1544  for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1545  for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1546  dst(i0, i1, i2) = src(i0, i1, i2);
1547  }
1548 }
1549 //----------------------------------------------------------------------------
1550 template <class DT, class... DP, class ST, class... SP>
1551 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1552  const View<DT, DP...>& dst, const View<ST, SP...>& src,
1553  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 4 &&
1554  unsigned(ViewTraits<ST, SP...>::rank) == 4)>* = nullptr) {
1555  if (dst.data() == nullptr) {
1556  return;
1557  }
1558 
1559  if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1560  local_deep_copy_contiguous(dst, src);
1561  } else {
1562  for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1563  for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1564  for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1565  for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
1566  dst(i0, i1, i2, i3) = src(i0, i1, i2, i3);
1567  }
1568 }
1569 //----------------------------------------------------------------------------
1570 template <class DT, class... DP, class ST, class... SP>
1571 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1572  const View<DT, DP...>& dst, const View<ST, SP...>& src,
1573  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 5 &&
1574  unsigned(ViewTraits<ST, SP...>::rank) == 5)>* = nullptr) {
1575  if (dst.data() == nullptr) {
1576  return;
1577  }
1578 
1579  if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1580  local_deep_copy_contiguous(dst, src);
1581  } else {
1582  for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1583  for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1584  for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1585  for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
1586  for (size_t i4 = 0; i4 < dst.extent(4); ++i4)
1587  dst(i0, i1, i2, i3, i4) = src(i0, i1, i2, i3, i4);
1588  }
1589 }
1590 //----------------------------------------------------------------------------
1591 template <class DT, class... DP, class ST, class... SP>
1592 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1593  const View<DT, DP...>& dst, const View<ST, SP...>& src,
1594  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 6 &&
1595  unsigned(ViewTraits<ST, SP...>::rank) == 6)>* = nullptr) {
1596  if (dst.data() == nullptr) {
1597  return;
1598  }
1599 
1600  if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1601  local_deep_copy_contiguous(dst, src);
1602  } else {
1603  for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1604  for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1605  for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1606  for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
1607  for (size_t i4 = 0; i4 < dst.extent(4); ++i4)
1608  for (size_t i5 = 0; i5 < dst.extent(5); ++i5)
1609  dst(i0, i1, i2, i3, i4, i5) = src(i0, i1, i2, i3, i4, i5);
1610  }
1611 }
1612 //----------------------------------------------------------------------------
1613 template <class DT, class... DP, class ST, class... SP>
1614 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1615  const View<DT, DP...>& dst, const View<ST, SP...>& src,
1616  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 7 &&
1617  unsigned(ViewTraits<ST, SP...>::rank) == 7)>* = nullptr) {
1618  if (dst.data() == nullptr) {
1619  return;
1620  }
1621 
1622  if (dst.span_is_contiguous() && src.span_is_contiguous()) {
1623  local_deep_copy_contiguous(dst, src);
1624  } else {
1625  for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1626  for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1627  for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1628  for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
1629  for (size_t i4 = 0; i4 < dst.extent(4); ++i4)
1630  for (size_t i5 = 0; i5 < dst.extent(5); ++i5)
1631  for (size_t i6 = 0; i6 < dst.extent(6); ++i6)
1632  dst(i0, i1, i2, i3, i4, i5, i6) =
1633  src(i0, i1, i2, i3, i4, i5, i6);
1634  }
1635 }
1636 //----------------------------------------------------------------------------
1637 //----------------------------------------------------------------------------
1639 template <class TeamType, class DT, class... DP>
1640 void KOKKOS_INLINE_FUNCTION local_deep_copy_contiguous(
1641  const TeamType& team, const View<DT, DP...>& dst,
1642  typename ViewTraits<DT, DP...>::const_value_type& value,
1643  std::enable_if_t<std::is_same_v<typename ViewTraits<DT, DP...>::specialize,
1644  void>>* = nullptr) {
1645  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, dst.span()),
1646  [&](const int& i) { dst.data()[i] = value; });
1647 }
1648 //----------------------------------------------------------------------------
1649 template <class DT, class... DP>
1650 void KOKKOS_INLINE_FUNCTION local_deep_copy_contiguous(
1651  const View<DT, DP...>& dst,
1652  typename ViewTraits<DT, DP...>::const_value_type& value,
1653  std::enable_if_t<std::is_same_v<typename ViewTraits<DT, DP...>::specialize,
1654  void>>* = nullptr) {
1655  for (size_t i = 0; i < dst.span(); ++i) {
1656  dst.data()[i] = value;
1657  }
1658 }
1659 //----------------------------------------------------------------------------
1660 template <class TeamType, class DT, class... DP>
1661 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1662  const TeamType& team, const View<DT, DP...>& dst,
1663  typename ViewTraits<DT, DP...>::const_value_type& value,
1664  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 1)>* = nullptr) {
1665  if (dst.data() == nullptr) {
1666  return;
1667  }
1668 
1669  const size_t N = dst.extent(0);
1670 
1671  team.team_barrier();
1672  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N),
1673  [&](const int& i) { dst(i) = value; });
1674  team.team_barrier();
1675 }
1676 //----------------------------------------------------------------------------
1677 template <class TeamType, class DT, class... DP>
1678 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1679  const TeamType& team, const View<DT, DP...>& dst,
1680  typename ViewTraits<DT, DP...>::const_value_type& value,
1681  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 2)>* = nullptr) {
1682  if (dst.data() == nullptr) {
1683  return;
1684  }
1685 
1686  const size_t N = dst.extent(0) * dst.extent(1);
1687 
1688  if (dst.span_is_contiguous()) {
1689  team.team_barrier();
1690  local_deep_copy_contiguous(team, dst, value);
1691  team.team_barrier();
1692  } else {
1693  team.team_barrier();
1694  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1695  int i0 = i % dst.extent(0);
1696  int i1 = i / dst.extent(0);
1697  dst(i0, i1) = value;
1698  });
1699  team.team_barrier();
1700  }
1701 }
1702 //----------------------------------------------------------------------------
1703 template <class TeamType, class DT, class... DP>
1704 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1705  const TeamType& team, const View<DT, DP...>& dst,
1706  typename ViewTraits<DT, DP...>::const_value_type& value,
1707  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 3)>* = nullptr) {
1708  if (dst.data() == nullptr) {
1709  return;
1710  }
1711 
1712  const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2);
1713 
1714  if (dst.span_is_contiguous()) {
1715  team.team_barrier();
1716  local_deep_copy_contiguous(team, dst, value);
1717  team.team_barrier();
1718  } else {
1719  team.team_barrier();
1720  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1721  int i0 = i % dst.extent(0);
1722  int itmp = i / dst.extent(0);
1723  int i1 = itmp % dst.extent(1);
1724  int i2 = itmp / dst.extent(1);
1725  dst(i0, i1, i2) = value;
1726  });
1727  team.team_barrier();
1728  }
1729 }
1730 //----------------------------------------------------------------------------
1731 template <class TeamType, class DT, class... DP>
1732 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1733  const TeamType& team, const View<DT, DP...>& dst,
1734  typename ViewTraits<DT, DP...>::const_value_type& value,
1735  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 4)>* = nullptr) {
1736  if (dst.data() == nullptr) {
1737  return;
1738  }
1739 
1740  const size_t N =
1741  dst.extent(0) * dst.extent(1) * dst.extent(2) * dst.extent(3);
1742 
1743  if (dst.span_is_contiguous()) {
1744  team.team_barrier();
1745  local_deep_copy_contiguous(team, dst, value);
1746  team.team_barrier();
1747  } else {
1748  team.team_barrier();
1749  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1750  int i0 = i % dst.extent(0);
1751  int itmp = i / dst.extent(0);
1752  int i1 = itmp % dst.extent(1);
1753  itmp = itmp / dst.extent(1);
1754  int i2 = itmp % dst.extent(2);
1755  int i3 = itmp / dst.extent(2);
1756  dst(i0, i1, i2, i3) = value;
1757  });
1758  team.team_barrier();
1759  }
1760 }
1761 //----------------------------------------------------------------------------
1762 template <class TeamType, class DT, class... DP>
1763 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1764  const TeamType& team, const View<DT, DP...>& dst,
1765  typename ViewTraits<DT, DP...>::const_value_type& value,
1766  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 5)>* = nullptr) {
1767  if (dst.data() == nullptr) {
1768  return;
1769  }
1770 
1771  const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1772  dst.extent(3) * dst.extent(4);
1773 
1774  if (dst.span_is_contiguous()) {
1775  team.team_barrier();
1776  local_deep_copy_contiguous(team, dst, value);
1777  team.team_barrier();
1778  } else {
1779  team.team_barrier();
1780  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1781  int i0 = i % dst.extent(0);
1782  int itmp = i / dst.extent(0);
1783  int i1 = itmp % dst.extent(1);
1784  itmp = itmp / dst.extent(1);
1785  int i2 = itmp % dst.extent(2);
1786  itmp = itmp / dst.extent(2);
1787  int i3 = itmp % dst.extent(3);
1788  int i4 = itmp / dst.extent(3);
1789  dst(i0, i1, i2, i3, i4) = value;
1790  });
1791  team.team_barrier();
1792  }
1793 }
1794 //----------------------------------------------------------------------------
1795 template <class TeamType, class DT, class... DP>
1796 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1797  const TeamType& team, const View<DT, DP...>& dst,
1798  typename ViewTraits<DT, DP...>::const_value_type& value,
1799  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 6)>* = nullptr) {
1800  if (dst.data() == nullptr) {
1801  return;
1802  }
1803 
1804  const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1805  dst.extent(3) * dst.extent(4) * dst.extent(5);
1806 
1807  if (dst.span_is_contiguous()) {
1808  team.team_barrier();
1809  local_deep_copy_contiguous(team, dst, value);
1810  team.team_barrier();
1811  } else {
1812  team.team_barrier();
1813  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1814  int i0 = i % dst.extent(0);
1815  int itmp = i / dst.extent(0);
1816  int i1 = itmp % dst.extent(1);
1817  itmp = itmp / dst.extent(1);
1818  int i2 = itmp % dst.extent(2);
1819  itmp = itmp / dst.extent(2);
1820  int i3 = itmp % dst.extent(3);
1821  itmp = itmp / dst.extent(3);
1822  int i4 = itmp % dst.extent(4);
1823  int i5 = itmp / dst.extent(4);
1824  dst(i0, i1, i2, i3, i4, i5) = value;
1825  });
1826  team.team_barrier();
1827  }
1828 }
1829 //----------------------------------------------------------------------------
1830 template <class TeamType, class DT, class... DP>
1831 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1832  const TeamType& team, const View<DT, DP...>& dst,
1833  typename ViewTraits<DT, DP...>::const_value_type& value,
1834  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 7)>* = nullptr) {
1835  if (dst.data() == nullptr) {
1836  return;
1837  }
1838 
1839  const size_t N = dst.extent(0) * dst.extent(1) * dst.extent(2) *
1840  dst.extent(3) * dst.extent(4) * dst.extent(5) *
1841  dst.extent(6);
1842 
1843  if (dst.span_is_contiguous()) {
1844  team.team_barrier();
1845  local_deep_copy_contiguous(team, dst, value);
1846  team.team_barrier();
1847  } else {
1848  team.team_barrier();
1849  Kokkos::parallel_for(Kokkos::TeamVectorRange(team, N), [&](const int& i) {
1850  int i0 = i % dst.extent(0);
1851  int itmp = i / dst.extent(0);
1852  int i1 = itmp % dst.extent(1);
1853  itmp = itmp / dst.extent(1);
1854  int i2 = itmp % dst.extent(2);
1855  itmp = itmp / dst.extent(2);
1856  int i3 = itmp % dst.extent(3);
1857  itmp = itmp / dst.extent(3);
1858  int i4 = itmp % dst.extent(4);
1859  itmp = itmp / dst.extent(4);
1860  int i5 = itmp % dst.extent(5);
1861  int i6 = itmp / dst.extent(5);
1862  dst(i0, i1, i2, i3, i4, i5, i6) = value;
1863  });
1864  team.team_barrier();
1865  }
1866 }
1867 //----------------------------------------------------------------------------
1868 template <class DT, class... DP>
1869 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1870  const View<DT, DP...>& dst,
1871  typename ViewTraits<DT, DP...>::const_value_type& value,
1872  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 1)>* = nullptr) {
1873  if (dst.data() == nullptr) {
1874  return;
1875  }
1876 
1877  const size_t N = dst.extent(0);
1878 
1879  for (size_t i = 0; i < N; ++i) {
1880  dst(i) = value;
1881  }
1882 }
1883 //----------------------------------------------------------------------------
1884 template <class DT, class... DP>
1885 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1886  const View<DT, DP...>& dst,
1887  typename ViewTraits<DT, DP...>::const_value_type& value,
1888  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 2)>* = nullptr) {
1889  if (dst.data() == nullptr) {
1890  return;
1891  }
1892 
1893  if (dst.span_is_contiguous()) {
1894  local_deep_copy_contiguous(dst, value);
1895  } else {
1896  for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1897  for (size_t i1 = 0; i1 < dst.extent(1); ++i1) dst(i0, i1) = value;
1898  }
1899 }
1900 //----------------------------------------------------------------------------
1901 template <class DT, class... DP>
1902 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1903  const View<DT, DP...>& dst,
1904  typename ViewTraits<DT, DP...>::const_value_type& value,
1905  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 3)>* = nullptr) {
1906  if (dst.data() == nullptr) {
1907  return;
1908  }
1909 
1910  if (dst.span_is_contiguous()) {
1911  local_deep_copy_contiguous(dst, value);
1912  } else {
1913  for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1914  for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1915  for (size_t i2 = 0; i2 < dst.extent(2); ++i2) dst(i0, i1, i2) = value;
1916  }
1917 }
1918 //----------------------------------------------------------------------------
1919 template <class DT, class... DP>
1920 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1921  const View<DT, DP...>& dst,
1922  typename ViewTraits<DT, DP...>::const_value_type& value,
1923  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 4)>* = nullptr) {
1924  if (dst.data() == nullptr) {
1925  return;
1926  }
1927 
1928  if (dst.span_is_contiguous()) {
1929  local_deep_copy_contiguous(dst, value);
1930  } else {
1931  for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1932  for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1933  for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1934  for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
1935  dst(i0, i1, i2, i3) = value;
1936  }
1937 }
1938 //----------------------------------------------------------------------------
1939 template <class DT, class... DP>
1940 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1941  const View<DT, DP...>& dst,
1942  typename ViewTraits<DT, DP...>::const_value_type& value,
1943  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 5)>* = nullptr) {
1944  if (dst.data() == nullptr) {
1945  return;
1946  }
1947 
1948  if (dst.span_is_contiguous()) {
1949  local_deep_copy_contiguous(dst, value);
1950  } else {
1951  for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1952  for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1953  for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1954  for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
1955  for (size_t i4 = 0; i4 < dst.extent(4); ++i4)
1956  dst(i0, i1, i2, i3, i4) = value;
1957  }
1958 }
1959 //----------------------------------------------------------------------------
1960 template <class DT, class... DP>
1961 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1962  const View<DT, DP...>& dst,
1963  typename ViewTraits<DT, DP...>::const_value_type& value,
1964  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 6)>* = nullptr) {
1965  if (dst.data() == nullptr) {
1966  return;
1967  }
1968 
1969  if (dst.span_is_contiguous()) {
1970  local_deep_copy_contiguous(dst, value);
1971  } else {
1972  for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1973  for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1974  for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1975  for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
1976  for (size_t i4 = 0; i4 < dst.extent(4); ++i4)
1977  for (size_t i5 = 0; i5 < dst.extent(5); ++i5)
1978  dst(i0, i1, i2, i3, i4, i5) = value;
1979  }
1980 }
1981 //----------------------------------------------------------------------------
1982 template <class DT, class... DP>
1983 void KOKKOS_INLINE_FUNCTION local_deep_copy(
1984  const View<DT, DP...>& dst,
1985  typename ViewTraits<DT, DP...>::const_value_type& value,
1986  std::enable_if_t<(unsigned(ViewTraits<DT, DP...>::rank) == 7)>* = nullptr) {
1987  if (dst.data() == nullptr) {
1988  return;
1989  }
1990 
1991  if (dst.span_is_contiguous()) {
1992  local_deep_copy_contiguous(dst, value);
1993  } else {
1994  for (size_t i0 = 0; i0 < dst.extent(0); ++i0)
1995  for (size_t i1 = 0; i1 < dst.extent(1); ++i1)
1996  for (size_t i2 = 0; i2 < dst.extent(2); ++i2)
1997  for (size_t i3 = 0; i3 < dst.extent(3); ++i3)
1998  for (size_t i4 = 0; i4 < dst.extent(4); ++i4)
1999  for (size_t i5 = 0; i5 < dst.extent(5); ++i5)
2000  for (size_t i6 = 0; i6 < dst.extent(6); ++i6)
2001  dst(i0, i1, i2, i3, i4, i5, i6) = value;
2002  }
2003 }
2004 } /* namespace Experimental */
2005 } /* namespace Kokkos */
2006 
2007 //----------------------------------------------------------------------------
2008 //----------------------------------------------------------------------------
2009 
2010 namespace Kokkos {
2011 
2014 template <class ExecSpace, class DT, class... DP>
2015 inline void deep_copy(
2016  const ExecSpace& space, const View<DT, DP...>& dst,
2017  typename ViewTraits<DT, DP...>::const_value_type& value,
2018  std::enable_if_t<
2019  Kokkos::is_execution_space<ExecSpace>::value &&
2020  std::is_void_v<typename ViewTraits<DT, DP...>::specialize> &&
2021  Kokkos::SpaceAccessibility<ExecSpace, typename ViewTraits<DT, DP...>::
2022  memory_space>::accessible>* =
2023  nullptr) {
2024  using dst_traits = ViewTraits<DT, DP...>;
2025  static_assert(std::is_same_v<typename dst_traits::non_const_value_type,
2026  typename dst_traits::value_type>,
2027  "deep_copy requires non-const type");
2028  using dst_memory_space = typename dst_traits::memory_space;
2029  if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
2030  Kokkos::Profiling::beginDeepCopy(
2031  Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
2032  dst.label(), dst.data(),
2033  Kokkos::Profiling::make_space_handle(Kokkos::HostSpace::name()),
2034  "(none)", &value, dst.span() * sizeof(typename dst_traits::value_type));
2035  }
2036  if (dst.data() == nullptr) {
2037  space.fence("Kokkos::deep_copy: scalar copy on space, dst data is null");
2038  } else if (dst.span_is_contiguous()) {
2039  Impl::contiguous_fill_or_memset(space, dst, value);
2040  } else {
2041  using ViewType = View<DT, DP...>;
2042  // Figure out iteration order to do the ViewFill
2043  int64_t strides[ViewType::rank + 1];
2044  dst.stride(strides);
2045  Kokkos::Iterate iterate;
2046  if (std::is_same_v<typename ViewType::array_layout, Kokkos::LayoutRight>) {
2047  iterate = Kokkos::Iterate::Right;
2048  } else if (std::is_same_v<typename ViewType::array_layout,
2049  Kokkos::LayoutLeft>) {
2050  iterate = Kokkos::Iterate::Left;
2051  } else if (std::is_same_v<typename ViewType::array_layout,
2052  Kokkos::LayoutStride>) {
2053  if (strides[0] > strides[ViewType::rank > 0 ? ViewType::rank - 1 : 0])
2054  iterate = Kokkos::Iterate::Right;
2055  else
2056  iterate = Kokkos::Iterate::Left;
2057  } else {
2058  if (std::is_same_v<typename ViewType::execution_space::array_layout,
2059  Kokkos::LayoutRight>)
2060  iterate = Kokkos::Iterate::Right;
2061  else
2062  iterate = Kokkos::Iterate::Left;
2063  }
2064 
2065  // Lets call the right ViewFill functor based on integer space needed and
2066  // iteration type
2067  using ViewTypeUniform =
2068  std::conditional_t<ViewType::rank == 0,
2069  typename ViewType::uniform_runtime_type,
2070  typename ViewType::uniform_runtime_nomemspace_type>;
2071  if (dst.span() > static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
2072  if (iterate == Kokkos::Iterate::Right)
2073  Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutRight, ExecSpace,
2074  ViewType::rank, int64_t>(dst, value, space);
2075  else
2076  Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutLeft, ExecSpace,
2077  ViewType::rank, int64_t>(dst, value, space);
2078  } else {
2079  if (iterate == Kokkos::Iterate::Right)
2080  Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutRight, ExecSpace,
2081  ViewType::rank, int32_t>(dst, value, space);
2082  else
2083  Kokkos::Impl::ViewFill<ViewTypeUniform, Kokkos::LayoutLeft, ExecSpace,
2084  ViewType::rank, int32_t>(dst, value, space);
2085  }
2086  }
2087  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2088  Kokkos::Profiling::endDeepCopy();
2089  }
2090 }
2091 
2094 template <class ExecSpace, class DT, class... DP>
2095 inline void deep_copy(
2096  const ExecSpace& space, const View<DT, DP...>& dst,
2097  typename ViewTraits<DT, DP...>::const_value_type& value,
2098  std::enable_if_t<
2099  Kokkos::is_execution_space<ExecSpace>::value &&
2100  std::is_void_v<typename ViewTraits<DT, DP...>::specialize> &&
2101  !Kokkos::SpaceAccessibility<ExecSpace, typename ViewTraits<DT, DP...>::
2102  memory_space>::accessible>* =
2103  nullptr) {
2104  using dst_traits = ViewTraits<DT, DP...>;
2105  static_assert(std::is_same_v<typename dst_traits::non_const_value_type,
2106  typename dst_traits::value_type>,
2107  "deep_copy requires non-const type");
2108  using dst_memory_space = typename dst_traits::memory_space;
2109  if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
2110  Kokkos::Profiling::beginDeepCopy(
2111  Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
2112  dst.label(), dst.data(),
2113  Kokkos::Profiling::make_space_handle(Kokkos::HostSpace::name()),
2114  "(none)", &value, dst.span() * sizeof(typename dst_traits::value_type));
2115  }
2116  if (dst.data() == nullptr) {
2117  space.fence(
2118  "Kokkos::deep_copy: scalar-to-view copy on space, dst data is null");
2119  } else {
2120  space.fence("Kokkos::deep_copy: scalar-to-view copy on space, pre copy");
2121  using fill_exec_space = typename dst_traits::memory_space::execution_space;
2122  if (dst.span_is_contiguous()) {
2123  Impl::contiguous_fill_or_memset(fill_exec_space(), dst, value);
2124  } else {
2125  using ViewTypeUniform = std::conditional_t<
2126  View<DT, DP...>::rank == 0,
2127  typename View<DT, DP...>::uniform_runtime_type,
2128  typename View<DT, DP...>::uniform_runtime_nomemspace_type>;
2129  Kokkos::Impl::ViewFill<ViewTypeUniform, typename dst_traits::array_layout,
2130  fill_exec_space>(dst, value, fill_exec_space());
2131  }
2132  fill_exec_space().fence(
2133  "Kokkos::deep_copy: scalar-to-view copy on space, fence after fill");
2134  }
2135  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2136  Kokkos::Profiling::endDeepCopy();
2137  }
2138 }
2139 
2141 template <class ExecSpace, class ST, class... SP>
2142 inline void deep_copy(
2143  const ExecSpace& exec_space,
2144  typename ViewTraits<ST, SP...>::non_const_value_type& dst,
2145  const View<ST, SP...>& src,
2146  std::enable_if_t<Kokkos::is_execution_space<ExecSpace>::value &&
2147  std::is_same_v<typename ViewTraits<ST, SP...>::specialize,
2148  void>>* = nullptr) {
2149  using src_traits = ViewTraits<ST, SP...>;
2150  using src_memory_space = typename src_traits::memory_space;
2151  static_assert(src_traits::rank == 0,
2152  "ERROR: Non-rank-zero view in deep_copy( value , View )");
2153  if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
2154  Kokkos::Profiling::beginDeepCopy(
2155  Kokkos::Profiling::make_space_handle(Kokkos::HostSpace::name()),
2156  "(none)", &dst,
2157  Kokkos::Profiling::make_space_handle(src_memory_space::name()),
2158  src.label(), src.data(), sizeof(ST));
2159  }
2160 
2161  if (src.data() == nullptr) {
2162  exec_space.fence(
2163  "Kokkos::deep_copy: view-to-scalar copy on space, src data is null");
2164  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2165  Kokkos::Profiling::endDeepCopy();
2166  }
2167  return;
2168  }
2169 
2170  Kokkos::Impl::DeepCopy<HostSpace, src_memory_space, ExecSpace>(
2171  exec_space, &dst, src.data(), sizeof(ST));
2172  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2173  Kokkos::Profiling::endDeepCopy();
2174  }
2175 }
2176 
2177 //----------------------------------------------------------------------------
2179 template <class ExecSpace, class DT, class... DP, class ST, class... SP>
2180 inline void deep_copy(
2181  const ExecSpace& exec_space, const View<DT, DP...>& dst,
2182  const View<ST, SP...>& src,
2183  std::enable_if_t<
2184  (Kokkos::is_execution_space<ExecSpace>::value &&
2185  std::is_void_v<typename ViewTraits<DT, DP...>::specialize> &&
2186  std::is_void_v<typename ViewTraits<ST, SP...>::specialize> &&
2187  (unsigned(ViewTraits<DT, DP...>::rank) == unsigned(0) &&
2188  unsigned(ViewTraits<ST, SP...>::rank) == unsigned(0)))>* = nullptr) {
2189  using src_traits = ViewTraits<ST, SP...>;
2190  using dst_traits = ViewTraits<DT, DP...>;
2191 
2192  using src_memory_space = typename src_traits::memory_space;
2193  using dst_memory_space = typename dst_traits::memory_space;
2194  static_assert(std::is_same_v<typename dst_traits::value_type,
2195  typename src_traits::non_const_value_type>,
2196  "deep_copy requires matching non-const destination type");
2197 
2198  if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
2199  Kokkos::Profiling::beginDeepCopy(
2200  Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
2201  dst.label(), dst.data(),
2202  Kokkos::Profiling::make_space_handle(src_memory_space::name()),
2203  src.label(), src.data(), sizeof(DT));
2204  }
2205 
2206  if (dst.data() == nullptr && src.data() == nullptr) {
2207  exec_space.fence(
2208  "Kokkos::deep_copy: view-to-view copy on space, data is null");
2209  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2210  Kokkos::Profiling::endDeepCopy();
2211  }
2212  return;
2213  }
2214 
2215  if (dst.data() != src.data()) {
2216  Kokkos::Impl::DeepCopy<dst_memory_space, src_memory_space, ExecSpace>(
2217  exec_space, dst.data(), src.data(),
2218  sizeof(typename dst_traits::value_type));
2219  }
2220  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2221  Kokkos::Profiling::endDeepCopy();
2222  }
2223 }
2224 
2225 //----------------------------------------------------------------------------
2229 template <class ExecSpace, class DT, class... DP, class ST, class... SP>
2230 inline void deep_copy(
2231  const ExecSpace& exec_space, const View<DT, DP...>& dst,
2232  const View<ST, SP...>& src,
2233  std::enable_if_t<
2234  (Kokkos::is_execution_space<ExecSpace>::value &&
2235  std::is_void_v<typename ViewTraits<DT, DP...>::specialize> &&
2236  std::is_void_v<typename ViewTraits<ST, SP...>::specialize> &&
2237  (unsigned(ViewTraits<DT, DP...>::rank) != 0 ||
2238  unsigned(ViewTraits<ST, SP...>::rank) != 0))>* = nullptr) {
2239  using dst_type = View<DT, DP...>;
2240  using src_type = View<ST, SP...>;
2241 
2242  static_assert(std::is_same_v<typename dst_type::value_type,
2243  typename dst_type::non_const_value_type>,
2244  "deep_copy requires non-const destination type");
2245 
2246  static_assert((unsigned(dst_type::rank) == unsigned(src_type::rank)),
2247  "deep_copy requires Views of equal rank");
2248 
2249  using dst_execution_space = typename dst_type::execution_space;
2250  using src_execution_space = typename src_type::execution_space;
2251  using dst_memory_space = typename dst_type::memory_space;
2252  using src_memory_space = typename src_type::memory_space;
2253  using dst_value_type = typename dst_type::value_type;
2254  using src_value_type = typename src_type::value_type;
2255 
2256  if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) {
2257  Kokkos::Profiling::beginDeepCopy(
2258  Kokkos::Profiling::make_space_handle(dst_memory_space::name()),
2259  dst.label(), dst.data(),
2260  Kokkos::Profiling::make_space_handle(src_memory_space::name()),
2261  src.label(), src.data(), dst.span() * sizeof(dst_value_type));
2262  }
2263 
2264  dst_value_type* dst_start = dst.data();
2265  dst_value_type* dst_end = dst.data() + dst.span();
2266  src_value_type* src_start = src.data();
2267  src_value_type* src_end = src.data() + src.span();
2268 
2269  // Early dropout if identical range
2270  if ((dst_start == nullptr || src_start == nullptr) ||
2271  ((std::ptrdiff_t(dst_start) == std::ptrdiff_t(src_start)) &&
2272  (std::ptrdiff_t(dst_end) == std::ptrdiff_t(src_end)))) {
2273  // throw if dimension mismatch
2274  if ((src.extent(0) != dst.extent(0)) || (src.extent(1) != dst.extent(1)) ||
2275  (src.extent(2) != dst.extent(2)) || (src.extent(3) != dst.extent(3)) ||
2276  (src.extent(4) != dst.extent(4)) || (src.extent(5) != dst.extent(5)) ||
2277  (src.extent(6) != dst.extent(6)) || (src.extent(7) != dst.extent(7))) {
2278  std::string message(
2279  "Deprecation Error: Kokkos::deep_copy extents of views don't "
2280  "match: ");
2281  message += dst.label();
2282  message += "(";
2283  message += std::to_string(dst.extent(0));
2284  for (size_t r = 1; r < dst_type::rank; r++) {
2285  message += ",";
2286  message += std::to_string(dst.extent(r));
2287  }
2288  message += ") ";
2289  message += src.label();
2290  message += "(";
2291  message += std::to_string(src.extent(0));
2292  for (size_t r = 1; r < src_type::rank; r++) {
2293  message += ",";
2294  message += std::to_string(src.extent(r));
2295  }
2296  message += ") ";
2297 
2298  Kokkos::Impl::throw_runtime_exception(message);
2299  }
2300  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2301  Kokkos::Profiling::endDeepCopy();
2302  }
2303  return;
2304  }
2305 
2306  // Error out for non-identical overlapping views.
2307  if ((((std::ptrdiff_t)dst_start < (std::ptrdiff_t)src_end) &&
2308  ((std::ptrdiff_t)dst_end > (std::ptrdiff_t)src_start)) &&
2309  ((dst.span_is_contiguous() && src.span_is_contiguous()))) {
2310  std::string message("Error: Kokkos::deep_copy of overlapping views: ");
2311  message += dst.label();
2312  message += "(";
2313  message += std::to_string((std::ptrdiff_t)dst_start);
2314  message += ",";
2315  message += std::to_string((std::ptrdiff_t)dst_end);
2316  message += ") ";
2317  message += src.label();
2318  message += "(";
2319  message += std::to_string((std::ptrdiff_t)src_start);
2320  message += ",";
2321  message += std::to_string((std::ptrdiff_t)src_end);
2322  message += ") ";
2323  Kokkos::Impl::throw_runtime_exception(message);
2324  }
2325 
2326  // Check for same extents
2327  if ((src.extent(0) != dst.extent(0)) || (src.extent(1) != dst.extent(1)) ||
2328  (src.extent(2) != dst.extent(2)) || (src.extent(3) != dst.extent(3)) ||
2329  (src.extent(4) != dst.extent(4)) || (src.extent(5) != dst.extent(5)) ||
2330  (src.extent(6) != dst.extent(6)) || (src.extent(7) != dst.extent(7))) {
2331  std::string message(
2332  "Deprecation Error: Kokkos::deep_copy extents of views don't match: ");
2333  message += dst.label();
2334  message += "(";
2335  message += std::to_string(dst.extent(0));
2336  for (size_t r = 1; r < dst_type::rank; r++) {
2337  message += ",";
2338  message += std::to_string(dst.extent(r));
2339  }
2340  message += ") ";
2341  message += src.label();
2342  message += "(";
2343  message += std::to_string(src.extent(0));
2344  for (size_t r = 1; r < src_type::rank; r++) {
2345  message += ",";
2346  message += std::to_string(src.extent(r));
2347  }
2348  message += ") ";
2349 
2350  Kokkos::Impl::throw_runtime_exception(message);
2351  }
2352 
2353  // If same type, equal layout, equal dimensions, equal span, and contiguous
2354  // memory then can byte-wise copy
2355 
2356  if (std::is_same_v<typename dst_type::value_type,
2357  typename src_type::non_const_value_type> &&
2358  (std::is_same_v<typename dst_type::array_layout,
2359  typename src_type::array_layout> ||
2360  (dst_type::rank == 1 && src_type::rank == 1)) &&
2361  dst.span_is_contiguous() && src.span_is_contiguous() &&
2362  ((dst_type::rank < 1) || (dst.stride_0() == src.stride_0())) &&
2363  ((dst_type::rank < 2) || (dst.stride_1() == src.stride_1())) &&
2364  ((dst_type::rank < 3) || (dst.stride_2() == src.stride_2())) &&
2365  ((dst_type::rank < 4) || (dst.stride_3() == src.stride_3())) &&
2366  ((dst_type::rank < 5) || (dst.stride_4() == src.stride_4())) &&
2367  ((dst_type::rank < 6) || (dst.stride_5() == src.stride_5())) &&
2368  ((dst_type::rank < 7) || (dst.stride_6() == src.stride_6())) &&
2369  ((dst_type::rank < 8) || (dst.stride_7() == src.stride_7()))) {
2370  const size_t nbytes = sizeof(typename dst_type::value_type) * dst.span();
2371  if ((void*)dst.data() != (void*)src.data() && 0 < nbytes) {
2372  Kokkos::Impl::DeepCopy<dst_memory_space, src_memory_space, ExecSpace>(
2373  exec_space, dst.data(), src.data(), nbytes);
2374  }
2375  } else {
2376  // Copying data between views in accessible memory spaces and either
2377  // non-contiguous or incompatible shape.
2378 
2379  constexpr bool ExecCanAccessSrcDst =
2380  Kokkos::SpaceAccessibility<ExecSpace, dst_memory_space>::accessible &&
2381  Kokkos::SpaceAccessibility<ExecSpace, src_memory_space>::accessible;
2382  constexpr bool DstExecCanAccessSrc =
2383  Kokkos::SpaceAccessibility<dst_execution_space,
2384  src_memory_space>::accessible;
2385  constexpr bool SrcExecCanAccessDst =
2386  Kokkos::SpaceAccessibility<src_execution_space,
2387  dst_memory_space>::accessible;
2388 
2389  if constexpr (ExecCanAccessSrcDst) {
2390  Impl::view_copy(exec_space, dst, src);
2391  } else if constexpr (DstExecCanAccessSrc || SrcExecCanAccessDst) {
2392  using cpy_exec_space =
2393  std::conditional_t<DstExecCanAccessSrc, dst_execution_space,
2394  src_execution_space>;
2395  exec_space.fence(
2396  "Kokkos::deep_copy: view-to-view noncontiguous copy on space, pre "
2397  "copy");
2398  Impl::view_copy(cpy_exec_space(), dst, src);
2399  cpy_exec_space().fence(
2400  "Kokkos::deep_copy: view-to-view noncontiguous copy on space, post "
2401  "copy");
2402  } else {
2403  Kokkos::Impl::throw_runtime_exception(
2404  "deep_copy given views that would require a temporary allocation");
2405  }
2406  }
2407  if (Kokkos::Tools::Experimental::get_callbacks().end_deep_copy != nullptr) {
2408  Kokkos::Profiling::endDeepCopy();
2409  }
2410 }
2411 
2412 } /* namespace Kokkos */
2413 
2414 //----------------------------------------------------------------------------
2415 //----------------------------------------------------------------------------
2416 
2417 namespace Kokkos {
2418 
2419 namespace Impl {
2420 template <typename ViewType>
2421 bool size_mismatch(const ViewType& view, unsigned int max_extent,
2422  const size_t new_extents[8]) {
2423  for (unsigned int dim = 0; dim < max_extent; ++dim)
2424  if (new_extents[dim] != view.extent(dim)) {
2425  return true;
2426  }
2427  for (unsigned int dim = max_extent; dim < 8; ++dim)
2428  if (new_extents[dim] != KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2429  return true;
2430  }
2431  return false;
2432 }
2433 
2434 } // namespace Impl
2435 
2438 template <class T, class... P, class... ViewCtorArgs>
2439 inline std::enable_if_t<
2440  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2441  Kokkos::LayoutLeft> ||
2442  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2443  Kokkos::LayoutRight>>
2444 impl_resize(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2445  Kokkos::View<T, P...>& v, const size_t n0, const size_t n1,
2446  const size_t n2, const size_t n3, const size_t n4, const size_t n5,
2447  const size_t n6, const size_t n7) {
2448  using view_type = Kokkos::View<T, P...>;
2449  using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2450 
2451  static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2452  "Can only resize managed views");
2453  static_assert(!alloc_prop_input::has_label,
2454  "The view constructor arguments passed to Kokkos::resize "
2455  "must not include a label!");
2456  static_assert(!alloc_prop_input::has_pointer,
2457  "The view constructor arguments passed to Kokkos::resize must "
2458  "not include a pointer!");
2459  static_assert(!alloc_prop_input::has_memory_space,
2460  "The view constructor arguments passed to Kokkos::resize must "
2461  "not include a memory space instance!");
2462 
2463  // TODO (mfh 27 Jun 2017) If the old View has enough space but just
2464  // different dimensions (e.g., if the product of the dimensions,
2465  // including extra space for alignment, will not change), then
2466  // consider just reusing storage. For now, Kokkos always
2467  // reallocates if any of the dimensions change, even if the old View
2468  // has enough space.
2469 
2470  const size_t new_extents[8] = {n0, n1, n2, n3, n4, n5, n6, n7};
2471  const bool sizeMismatch = Impl::size_mismatch(v, v.rank_dynamic, new_extents);
2472 
2473  if (sizeMismatch) {
2474  auto prop_copy = Impl::with_properties_if_unset(
2475  arg_prop, typename view_type::execution_space{}, v.label());
2476 
2477  view_type v_resized(prop_copy, n0, n1, n2, n3, n4, n5, n6, n7);
2478 
2479  if constexpr (alloc_prop_input::has_execution_space)
2480  Kokkos::Impl::ViewRemap<view_type, view_type>(
2481  v_resized, v, Impl::get_property<Impl::ExecutionSpaceTag>(prop_copy));
2482  else {
2483  Kokkos::Impl::ViewRemap<view_type, view_type>(v_resized, v);
2484  Kokkos::fence("Kokkos::resize(View)");
2485  }
2486 
2487  v = v_resized;
2488  }
2489 }
2490 
2491 template <class T, class... P, class... ViewCtorArgs>
2492 inline std::enable_if_t<
2493  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2494  Kokkos::LayoutLeft> ||
2495  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2496  Kokkos::LayoutRight>>
2497 resize(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2498  Kokkos::View<T, P...>& v, const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2499  const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2500  const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2501  const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2502  const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2503  const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2504  const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2505  const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2506  impl_resize(arg_prop, v, n0, n1, n2, n3, n4, n5, n6, n7);
2507 }
2508 
2509 template <class T, class... P>
2510 inline std::enable_if_t<
2511  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2512  Kokkos::LayoutLeft> ||
2513  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2514  Kokkos::LayoutRight>>
2515 resize(Kokkos::View<T, P...>& v, const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2516  const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2517  const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2518  const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2519  const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2520  const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2521  const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2522  const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2523  impl_resize(Impl::ViewCtorProp<>{}, v, n0, n1, n2, n3, n4, n5, n6, n7);
2524 }
2525 
2526 template <class I, class T, class... P>
2527 inline std::enable_if_t<
2528  (Impl::is_view_ctor_property<I>::value ||
2529  Kokkos::is_execution_space<I>::value) &&
2530  (std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2531  Kokkos::LayoutLeft> ||
2532  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2533  Kokkos::LayoutRight>)>
2534 resize(const I& arg_prop, Kokkos::View<T, P...>& v,
2535  const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2536  const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2537  const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2538  const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2539  const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2540  const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2541  const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2542  const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2543  impl_resize(Kokkos::view_alloc(arg_prop), v, n0, n1, n2, n3, n4, n5, n6, n7);
2544 }
2545 
2546 template <class T, class... P, class... ViewCtorArgs>
2547 inline std::enable_if_t<
2548  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2549  Kokkos::LayoutLeft> ||
2550  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2551  Kokkos::LayoutRight> ||
2552  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2553  Kokkos::LayoutStride>>
2554 impl_resize(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2555  Kokkos::View<T, P...>& v,
2556  const typename Kokkos::View<T, P...>::array_layout& layout) {
2557  using view_type = Kokkos::View<T, P...>;
2558  using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2559 
2560  static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2561  "Can only resize managed views");
2562  static_assert(!alloc_prop_input::has_label,
2563  "The view constructor arguments passed to Kokkos::resize "
2564  "must not include a label!");
2565  static_assert(!alloc_prop_input::has_pointer,
2566  "The view constructor arguments passed to Kokkos::resize must "
2567  "not include a pointer!");
2568  static_assert(!alloc_prop_input::has_memory_space,
2569  "The view constructor arguments passed to Kokkos::resize must "
2570  "not include a memory space instance!");
2571 
2572  if (v.layout() != layout) {
2573  auto prop_copy = Impl::with_properties_if_unset(arg_prop, v.label());
2574 
2575  view_type v_resized(prop_copy, layout);
2576 
2577  if constexpr (alloc_prop_input::has_execution_space)
2578  Kokkos::Impl::ViewRemap<view_type, view_type>(
2579  v_resized, v, Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop));
2580  else {
2581  Kokkos::Impl::ViewRemap<view_type, view_type>(v_resized, v);
2582  Kokkos::fence("Kokkos::resize(View)");
2583  }
2584 
2585  v = v_resized;
2586  }
2587 }
2588 
2589 // FIXME User-provided (custom) layouts are not required to have a comparison
2590 // operator. Hence, there is no way to check if the requested layout is actually
2591 // the same as the existing one.
2592 template <class T, class... P, class... ViewCtorArgs>
2593 inline std::enable_if_t<
2594  !(std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2595  Kokkos::LayoutLeft> ||
2596  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2597  Kokkos::LayoutRight> ||
2598  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2599  Kokkos::LayoutStride>)>
2600 impl_resize(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2601  Kokkos::View<T, P...>& v,
2602  const typename Kokkos::View<T, P...>::array_layout& layout) {
2603  using view_type = Kokkos::View<T, P...>;
2604  using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2605 
2606  static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2607  "Can only resize managed views");
2608  static_assert(!alloc_prop_input::has_label,
2609  "The view constructor arguments passed to Kokkos::resize "
2610  "must not include a label!");
2611  static_assert(!alloc_prop_input::has_pointer,
2612  "The view constructor arguments passed to Kokkos::resize must "
2613  "not include a pointer!");
2614  static_assert(!alloc_prop_input::has_memory_space,
2615  "The view constructor arguments passed to Kokkos::resize must "
2616  "not include a memory space instance!");
2617 
2618  auto prop_copy = Impl::with_properties_if_unset(arg_prop, v.label());
2619 
2620  view_type v_resized(prop_copy, layout);
2621 
2622  if constexpr (alloc_prop_input::has_execution_space)
2623  Kokkos::Impl::ViewRemap<view_type, view_type>(
2624  v_resized, v, Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop));
2625  else {
2626  Kokkos::Impl::ViewRemap<view_type, view_type>(v_resized, v);
2627  Kokkos::fence("Kokkos::resize(View)");
2628  }
2629 
2630  v = v_resized;
2631 }
2632 
2633 template <class T, class... P, class... ViewCtorArgs>
2634 inline void resize(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2635  Kokkos::View<T, P...>& v,
2636  const typename Kokkos::View<T, P...>::array_layout& layout) {
2637  impl_resize(arg_prop, v, layout);
2638 }
2639 
2640 template <class I, class T, class... P>
2641 inline std::enable_if_t<Impl::is_view_ctor_property<I>::value ||
2642  Kokkos::is_execution_space<I>::value>
2643 resize(const I& arg_prop, Kokkos::View<T, P...>& v,
2644  const typename Kokkos::View<T, P...>::array_layout& layout) {
2645  impl_resize(arg_prop, v, layout);
2646 }
2647 
2648 template <class ExecutionSpace, class T, class... P>
2649 inline void resize(const ExecutionSpace& exec_space, Kokkos::View<T, P...>& v,
2650  const typename Kokkos::View<T, P...>::array_layout& layout) {
2651  impl_resize(Impl::ViewCtorProp<>(), exec_space, v, layout);
2652 }
2653 
2654 template <class T, class... P>
2655 inline void resize(Kokkos::View<T, P...>& v,
2656  const typename Kokkos::View<T, P...>::array_layout& layout) {
2657  impl_resize(Impl::ViewCtorProp<>{}, v, layout);
2658 }
2659 
2661 template <class T, class... P, class... ViewCtorArgs>
2662 inline std::enable_if_t<
2663  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2664  Kokkos::LayoutLeft> ||
2665  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2666  Kokkos::LayoutRight>>
2667 impl_realloc(Kokkos::View<T, P...>& v, const size_t n0, const size_t n1,
2668  const size_t n2, const size_t n3, const size_t n4, const size_t n5,
2669  const size_t n6, const size_t n7,
2670  const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
2671  using view_type = Kokkos::View<T, P...>;
2672  using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2673 
2674  static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2675  "Can only realloc managed views");
2676  static_assert(!alloc_prop_input::has_label,
2677  "The view constructor arguments passed to Kokkos::realloc must "
2678  "not include a label!");
2679  static_assert(!alloc_prop_input::has_pointer,
2680  "The view constructor arguments passed to Kokkos::realloc must "
2681  "not include a pointer!");
2682  static_assert(!alloc_prop_input::has_memory_space,
2683  "The view constructor arguments passed to Kokkos::realloc must "
2684  "not include a memory space instance!");
2685 
2686  const size_t new_extents[8] = {n0, n1, n2, n3, n4, n5, n6, n7};
2687  const bool sizeMismatch = Impl::size_mismatch(v, v.rank_dynamic, new_extents);
2688 
2689  if (sizeMismatch) {
2690  auto arg_prop_copy = Impl::with_properties_if_unset(arg_prop, v.label());
2691  v = view_type(); // Best effort to deallocate in case no other view refers
2692  // to the shared allocation
2693  v = view_type(arg_prop_copy, n0, n1, n2, n3, n4, n5, n6, n7);
2694  return;
2695  }
2696 
2697  if constexpr (alloc_prop_input::initialize) {
2698  if constexpr (alloc_prop_input::has_execution_space) {
2699  const auto& exec_space =
2700  Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop);
2701  Kokkos::deep_copy(exec_space, v, typename view_type::value_type{});
2702  } else
2703  Kokkos::deep_copy(v, typename view_type::value_type{});
2704  }
2705 }
2706 
2707 template <class T, class... P, class... ViewCtorArgs>
2708 inline std::enable_if_t<
2709  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2710  Kokkos::LayoutLeft> ||
2711  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2712  Kokkos::LayoutRight>>
2713 realloc(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2714  Kokkos::View<T, P...>& v,
2715  const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2716  const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2717  const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2718  const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2719  const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2720  const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2721  const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2722  const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2723  impl_realloc(v, n0, n1, n2, n3, n4, n5, n6, n7, arg_prop);
2724 }
2725 
2726 template <class T, class... P>
2727 inline std::enable_if_t<
2728  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2729  Kokkos::LayoutLeft> ||
2730  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2731  Kokkos::LayoutRight>>
2732 realloc(Kokkos::View<T, P...>& v,
2733  const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2734  const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2735  const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2736  const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2737  const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2738  const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2739  const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2740  const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2741  impl_realloc(v, n0, n1, n2, n3, n4, n5, n6, n7, Impl::ViewCtorProp<>{});
2742 }
2743 
2744 template <class I, class T, class... P>
2745 inline std::enable_if_t<
2746  Impl::is_view_ctor_property<I>::value &&
2747  (std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2748  Kokkos::LayoutLeft> ||
2749  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2750  Kokkos::LayoutRight>)>
2751 realloc(const I& arg_prop, Kokkos::View<T, P...>& v,
2752  const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2753  const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2754  const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2755  const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2756  const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2757  const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2758  const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
2759  const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
2760  impl_realloc(v, n0, n1, n2, n3, n4, n5, n6, n7, Kokkos::view_alloc(arg_prop));
2761 }
2762 
2763 template <class T, class... P, class... ViewCtorArgs>
2764 inline std::enable_if_t<
2765  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2766  Kokkos::LayoutLeft> ||
2767  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2768  Kokkos::LayoutRight> ||
2769  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2770  Kokkos::LayoutStride>>
2771 impl_realloc(Kokkos::View<T, P...>& v,
2772  const typename Kokkos::View<T, P...>::array_layout& layout,
2773  const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
2774  using view_type = Kokkos::View<T, P...>;
2775  using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2776 
2777  static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2778  "Can only realloc managed views");
2779  static_assert(!alloc_prop_input::has_label,
2780  "The view constructor arguments passed to Kokkos::realloc must "
2781  "not include a label!");
2782  static_assert(!alloc_prop_input::has_pointer,
2783  "The view constructor arguments passed to Kokkos::realloc must "
2784  "not include a pointer!");
2785  static_assert(!alloc_prop_input::has_memory_space,
2786  "The view constructor arguments passed to Kokkos::realloc must "
2787  "not include a memory space instance!");
2788 
2789  if (v.layout() != layout) {
2790  v = view_type(); // Deallocate first, if the only view to allocation
2791  v = view_type(arg_prop, layout);
2792  return;
2793  }
2794 
2795  if constexpr (alloc_prop_input::initialize) {
2796  if constexpr (alloc_prop_input::has_execution_space) {
2797  const auto& exec_space =
2798  Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop);
2799  Kokkos::deep_copy(exec_space, v, typename view_type::value_type{});
2800  } else
2801  Kokkos::deep_copy(v, typename view_type::value_type{});
2802  }
2803 }
2804 
2805 // FIXME User-provided (custom) layouts are not required to have a comparison
2806 // operator. Hence, there is no way to check if the requested layout is actually
2807 // the same as the existing one.
2808 template <class T, class... P, class... ViewCtorArgs>
2809 inline std::enable_if_t<
2810  !(std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2811  Kokkos::LayoutLeft> ||
2812  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2813  Kokkos::LayoutRight> ||
2814  std::is_same_v<typename Kokkos::View<T, P...>::array_layout,
2815  Kokkos::LayoutStride>)>
2816 impl_realloc(Kokkos::View<T, P...>& v,
2817  const typename Kokkos::View<T, P...>::array_layout& layout,
2818  const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
2819  using view_type = Kokkos::View<T, P...>;
2820  using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2821 
2822  static_assert(Kokkos::ViewTraits<T, P...>::is_managed,
2823  "Can only realloc managed views");
2824  static_assert(!alloc_prop_input::has_label,
2825  "The view constructor arguments passed to Kokkos::realloc must "
2826  "not include a label!");
2827  static_assert(!alloc_prop_input::has_pointer,
2828  "The view constructor arguments passed to Kokkos::realloc must "
2829  "not include a pointer!");
2830  static_assert(!alloc_prop_input::has_memory_space,
2831  "The view constructor arguments passed to Kokkos::realloc must "
2832  "not include a memory space instance!");
2833 
2834  auto arg_prop_copy = Impl::with_properties_if_unset(arg_prop, v.label());
2835 
2836  v = view_type(); // Deallocate first, if the only view to allocation
2837  v = view_type(arg_prop_copy, layout);
2838 }
2839 
2840 template <class T, class... P, class... ViewCtorArgs>
2841 inline void realloc(
2842  const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
2843  Kokkos::View<T, P...>& v,
2844  const typename Kokkos::View<T, P...>::array_layout& layout) {
2845  impl_realloc(v, layout, arg_prop);
2846 }
2847 
2848 template <class I, class T, class... P>
2849 inline std::enable_if_t<Impl::is_view_ctor_property<I>::value> realloc(
2850  const I& arg_prop, Kokkos::View<T, P...>& v,
2851  const typename Kokkos::View<T, P...>::array_layout& layout) {
2852  impl_realloc(v, layout, Kokkos::view_alloc(arg_prop));
2853 }
2854 
2855 template <class T, class... P>
2856 inline void realloc(
2857  Kokkos::View<T, P...>& v,
2858  const typename Kokkos::View<T, P...>::array_layout& layout) {
2859  impl_realloc(v, layout, Impl::ViewCtorProp<>{});
2860 }
2861 
2862 } /* namespace Kokkos */
2863 
2864 //----------------------------------------------------------------------------
2865 //----------------------------------------------------------------------------
2866 
2867 namespace Kokkos {
2868 namespace Impl {
2869 
2870 // Deduce Mirror Types
2871 template <class Space, class T, class... P>
2872 struct MirrorViewType {
2873  // The incoming view_type
2874  using src_view_type = typename Kokkos::View<T, P...>;
2875  // The memory space for the mirror view
2876  using memory_space = typename Space::memory_space;
2877  // Check whether it is the same memory space
2878  static constexpr bool is_same_memspace =
2879  std::is_same_v<memory_space, typename src_view_type::memory_space>;
2880  // The array_layout
2881  using array_layout = typename src_view_type::array_layout;
2882  // The data type (we probably want it non-const since otherwise we can't even
2883  // deep_copy to it.
2884  using data_type = typename src_view_type::non_const_data_type;
2885  // The destination view type if it is not the same memory space
2886  using dest_view_type = Kokkos::View<data_type, array_layout, Space>;
2887  // If it is the same memory_space return the existsing view_type
2888  // This will also keep the unmanaged trait if necessary
2889  using view_type =
2890  std::conditional_t<is_same_memspace, src_view_type, dest_view_type>;
2891 };
2892 
2893 // collection of static asserts for create_mirror and create_mirror_view
2894 template <class... ViewCtorArgs>
2895 void check_view_ctor_args_create_mirror() {
2896  using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
2897 
2898  static_assert(
2899  !alloc_prop_input::has_label,
2900  "The view constructor arguments passed to Kokkos::create_mirror[_view] "
2901  "must not include a label!");
2902  static_assert(!alloc_prop_input::has_pointer,
2903  "The view constructor arguments passed to "
2904  "Kokkos::create_mirror[_view] must "
2905  "not include a pointer!");
2906  static_assert(!alloc_prop_input::allow_padding,
2907  "The view constructor arguments passed to "
2908  "Kokkos::create_mirror[_view] must "
2909  "not explicitly allow padding!");
2910 }
2911 
2912 // create a mirror
2913 // private interface that accepts arbitrary view constructor args passed by a
2914 // view_alloc
2915 template <class T, class... P, class... ViewCtorArgs>
2916 inline auto create_mirror(const Kokkos::View<T, P...>& src,
2917  const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
2918  check_view_ctor_args_create_mirror<ViewCtorArgs...>();
2919 
2920  auto prop_copy = Impl::with_properties_if_unset(
2921  arg_prop, std::string(src.label()).append("_mirror"));
2922 
2923  if constexpr (Impl::ViewCtorProp<ViewCtorArgs...>::has_memory_space) {
2924  using memory_space = typename decltype(prop_copy)::memory_space;
2925  using dst_type =
2926  typename Impl::MirrorViewType<memory_space, T, P...>::dest_view_type;
2927  return dst_type(prop_copy, src.layout());
2928  } else {
2929  using dst_type = typename View<T, P...>::HostMirror;
2930  return dst_type(prop_copy, src.layout());
2931  }
2932 #if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
2933  !defined(KOKKOS_COMPILER_MSVC)
2934  __builtin_unreachable();
2935 #endif
2936 }
2937 } // namespace Impl
2938 
2939 // public interface
2940 template <class T, class... P,
2941  typename = std::enable_if_t<
2942  std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
2943 auto create_mirror(Kokkos::View<T, P...> const& src) {
2944  return Impl::create_mirror(src, Impl::ViewCtorProp<>{});
2945 }
2946 
2947 // public interface that accepts a without initializing flag
2948 template <class T, class... P,
2949  typename = std::enable_if_t<
2950  std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
2951 auto create_mirror(Kokkos::Impl::WithoutInitializing_t wi,
2952  Kokkos::View<T, P...> const& src) {
2953  return Impl::create_mirror(src, view_alloc(wi));
2954 }
2955 
2956 // public interface that accepts a space
2957 template <class Space, class T, class... P,
2958  typename Enable = std::enable_if_t<
2959  Kokkos::is_space<Space>::value &&
2960  std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
2961 auto create_mirror(Space const&, Kokkos::View<T, P...> const& src) {
2962  return Impl::create_mirror(src, view_alloc(typename Space::memory_space{}));
2963 }
2964 
2965 // public interface that accepts arbitrary view constructor args passed by a
2966 // view_alloc
2967 template <class T, class... P, class... ViewCtorArgs,
2968  typename = std::enable_if_t<
2969  std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
2970 auto create_mirror(Impl::ViewCtorProp<ViewCtorArgs...> const& arg_prop,
2971  Kokkos::View<T, P...> const& src) {
2972  return Impl::create_mirror(src, arg_prop);
2973 }
2974 
2975 // public interface that accepts a space and a without initializing flag
2976 template <class Space, class T, class... P,
2977  typename Enable = std::enable_if_t<
2978  Kokkos::is_space<Space>::value &&
2979  std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
2980 auto create_mirror(Kokkos::Impl::WithoutInitializing_t wi, Space const&,
2981  Kokkos::View<T, P...> const& src) {
2982  return Impl::create_mirror(src,
2983  view_alloc(typename Space::memory_space{}, wi));
2984 }
2985 
2986 namespace Impl {
2987 
2988 // choose a `Kokkos::create_mirror` adapted for the provided view and the
2989 // provided arguments
2990 template <class View, class... ViewCtorArgs>
2991 inline auto choose_create_mirror(
2992  const View& src, const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
2993  // Due to the fact that users can overload `Kokkos::create_mirror`, but also
2994  // that they may not have implemented all of its different possible
2995  // variations, this function chooses the correct private or public version of
2996  // it to call.
2997  // This helper should be used by any overload of
2998  // `Kokkos::Impl::create_mirror_view`.
2999 
3000  if constexpr (std::is_void_v<typename View::traits::specialize>) {
3001  // if the view is not specialized, just call the Impl function
3002 
3003  // using ADL to find the later defined overload of the function
3004  using namespace Kokkos::Impl;
3005 
3006  return create_mirror(src, arg_prop);
3007  } else {
3008  // otherwise, recreate the public call
3009  using ViewProp = Impl::ViewCtorProp<ViewCtorArgs...>;
3010 
3011  // using ADL to find the later defined overload of the function
3012  using namespace Kokkos;
3013 
3014  if constexpr (sizeof...(ViewCtorArgs) == 0) {
3015  // if there are no view constructor args, call the specific public
3016  // function
3017  return create_mirror(src);
3018  } else if constexpr (sizeof...(ViewCtorArgs) == 1 &&
3019  ViewProp::has_memory_space) {
3020  // if there is one view constructor arg and it has a memory space, call
3021  // the specific public function
3022  return create_mirror(typename ViewProp::memory_space{}, src);
3023  } else if constexpr (sizeof...(ViewCtorArgs) == 1 &&
3024  !ViewProp::initialize) {
3025  // if there is one view constructor arg and it has a without initializing
3026  // mark, call the specific public function
3027  return create_mirror(typename Kokkos::Impl::WithoutInitializing_t{}, src);
3028  } else if constexpr (sizeof...(ViewCtorArgs) == 2 &&
3029  ViewProp::has_memory_space && !ViewProp::initialize) {
3030  // if there is two view constructor args and they have a memory space and
3031  // a without initializing mark, call the specific public function
3032  return create_mirror(typename Kokkos::Impl::WithoutInitializing_t{},
3033  typename ViewProp::memory_space{}, src);
3034  } else {
3035  // if there are other constructor args, call the generic public function
3036 
3037  // Beware, there are some libraries using Kokkos that don't implement
3038  // this overload (hence the reason for this present function to exist).
3039  return create_mirror(arg_prop, src);
3040  }
3041  }
3042 
3043 #if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
3044  !defined(KOKKOS_COMPILER_MSVC)
3045  __builtin_unreachable();
3046 #endif
3047 }
3048 
3049 // create a mirror view
3050 // private interface that accepts arbitrary view constructor args passed by a
3051 // view_alloc
3052 template <class T, class... P, class... ViewCtorArgs>
3053 inline auto create_mirror_view(
3054  const Kokkos::View<T, P...>& src,
3055  [[maybe_unused]] const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
3056  if constexpr (!Impl::ViewCtorProp<ViewCtorArgs...>::has_memory_space) {
3057  if constexpr (std::is_same_v<typename Kokkos::View<T, P...>::memory_space,
3058  typename Kokkos::View<
3059  T, P...>::HostMirror::memory_space> &&
3060  std::is_same_v<
3061  typename Kokkos::View<T, P...>::data_type,
3062  typename Kokkos::View<T, P...>::HostMirror::data_type>) {
3063  check_view_ctor_args_create_mirror<ViewCtorArgs...>();
3064  return typename Kokkos::View<T, P...>::HostMirror(src);
3065  } else {
3066  return Kokkos::Impl::choose_create_mirror(src, arg_prop);
3067  }
3068  } else {
3069  if constexpr (Impl::MirrorViewType<typename Impl::ViewCtorProp<
3070  ViewCtorArgs...>::memory_space,
3071  T, P...>::is_same_memspace) {
3072  check_view_ctor_args_create_mirror<ViewCtorArgs...>();
3073  return typename Impl::MirrorViewType<
3074  typename Impl::ViewCtorProp<ViewCtorArgs...>::memory_space, T,
3075  P...>::view_type(src);
3076  } else {
3077  return Kokkos::Impl::choose_create_mirror(src, arg_prop);
3078  }
3079  }
3080 #if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
3081  !defined(KOKKOS_COMPILER_MSVC)
3082  __builtin_unreachable();
3083 #endif
3084 }
3085 } // namespace Impl
3086 
3087 // public interface
3088 template <class T, class... P>
3089 auto create_mirror_view(const Kokkos::View<T, P...>& src) {
3090  return Impl::create_mirror_view(src, view_alloc());
3091 }
3092 
3093 // public interface that accepts a without initializing flag
3094 template <class T, class... P>
3095 auto create_mirror_view(Kokkos::Impl::WithoutInitializing_t wi,
3096  Kokkos::View<T, P...> const& src) {
3097  return Impl::create_mirror_view(src, view_alloc(wi));
3098 }
3099 
3100 // public interface that accepts a space
3101 template <class Space, class T, class... P,
3102  class Enable = std::enable_if_t<Kokkos::is_space<Space>::value>>
3103 auto create_mirror_view(const Space&, const Kokkos::View<T, P...>& src) {
3104  return Impl::create_mirror_view(src,
3105  view_alloc(typename Space::memory_space()));
3106 }
3107 
3108 // public interface that accepts a space and a without initializing flag
3109 template <class Space, class T, class... P,
3110  typename Enable = std::enable_if_t<Kokkos::is_space<Space>::value>>
3111 auto create_mirror_view(Kokkos::Impl::WithoutInitializing_t wi, Space const&,
3112  Kokkos::View<T, P...> const& src) {
3113  return Impl::create_mirror_view(
3114  src, view_alloc(typename Space::memory_space{}, wi));
3115 }
3116 
3117 // public interface that accepts arbitrary view constructor args passed by a
3118 // view_alloc
3119 template <class T, class... P, class... ViewCtorArgs,
3120  typename = std::enable_if_t<
3121  std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
3122 auto create_mirror_view(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
3123  const Kokkos::View<T, P...>& src) {
3124  return Impl::create_mirror_view(src, arg_prop);
3125 }
3126 
3127 namespace Impl {
3128 
3129 // collection of static asserts for create_mirror_view_and_copy
3130 template <class... ViewCtorArgs>
3131 void check_view_ctor_args_create_mirror_view_and_copy() {
3132  using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
3133 
3134  static_assert(
3135  alloc_prop_input::has_memory_space,
3136  "The view constructor arguments passed to "
3137  "Kokkos::create_mirror_view_and_copy must include a memory space!");
3138  static_assert(!alloc_prop_input::has_pointer,
3139  "The view constructor arguments passed to "
3140  "Kokkos::create_mirror_view_and_copy must "
3141  "not include a pointer!");
3142  static_assert(!alloc_prop_input::allow_padding,
3143  "The view constructor arguments passed to "
3144  "Kokkos::create_mirror_view_and_copy must "
3145  "not explicitly allow padding!");
3146 }
3147 
3148 } // namespace Impl
3149 
3150 // create a mirror view and deep copy it
3151 // public interface that accepts arbitrary view constructor args passed by a
3152 // view_alloc
3153 template <class... ViewCtorArgs, class T, class... P,
3154  class Enable = std::enable_if_t<
3155  std::is_void_v<typename ViewTraits<T, P...>::specialize>>>
3156 auto create_mirror_view_and_copy(
3157  [[maybe_unused]] const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
3158  const Kokkos::View<T, P...>& src) {
3159  using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
3160 
3161  Impl::check_view_ctor_args_create_mirror_view_and_copy<ViewCtorArgs...>();
3162 
3163  if constexpr (Impl::MirrorViewType<typename alloc_prop_input::memory_space, T,
3164  P...>::is_same_memspace) {
3165  // same behavior as deep_copy(src, src)
3166  if constexpr (!alloc_prop_input::has_execution_space)
3167  fence(
3168  "Kokkos::create_mirror_view_and_copy: fence before returning src "
3169  "view");
3170  return src;
3171  } else {
3172  using Space = typename alloc_prop_input::memory_space;
3173  using Mirror = typename Impl::MirrorViewType<Space, T, P...>::view_type;
3174 
3175  auto arg_prop_copy = Impl::with_properties_if_unset(
3176  arg_prop, std::string{}, WithoutInitializing,
3177  typename Space::execution_space{});
3178 
3179  std::string& label = Impl::get_property<Impl::LabelTag>(arg_prop_copy);
3180  if (label.empty()) label = src.label();
3181  auto mirror = typename Mirror::non_const_type{arg_prop_copy, src.layout()};
3182  if constexpr (alloc_prop_input::has_execution_space) {
3183  deep_copy(Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop_copy),
3184  mirror, src);
3185  } else
3186  deep_copy(mirror, src);
3187  return mirror;
3188  }
3189 #if defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \
3190  !defined(KOKKOS_COMPILER_MSVC)
3191  __builtin_unreachable();
3192 #endif
3193 }
3194 
3195 // Previously when using auto here, the intel compiler 19.3 would
3196 // sometimes not create a symbol, guessing that it somehow is a combination
3197 // of auto and just forwarding arguments (see issue #5196)
3198 template <class Space, class T, class... P,
3199  typename Enable = std::enable_if_t<Kokkos::is_space<Space>::value>>
3200 typename Impl::MirrorViewType<Space, T, P...>::view_type
3201 create_mirror_view_and_copy(
3202  const Space&, const Kokkos::View<T, P...>& src,
3203  std::string const& name = "",
3204  std::enable_if_t<
3205  std::is_void_v<typename ViewTraits<T, P...>::specialize>>* = nullptr) {
3206  return create_mirror_view_and_copy(
3207  Kokkos::view_alloc(typename Space::memory_space{}, name), src);
3208 }
3209 
3210 } /* namespace Kokkos */
3211 
3212 //----------------------------------------------------------------------------
3213 //----------------------------------------------------------------------------
3214 
3215 #endif
Kokkos::LayoutLeft
Memory layout tag indicating left-to-right (Fortran scheme) striding of multi-indices.
Definition: Kokkos_Layout.hpp:50
Kokkos::SpaceAccessibility
Can AccessSpace access MemorySpace ?
Definition: Kokkos_Concepts.hpp:359
Kokkos::pair
Replacement for std::pair that works on CUDA devices.
Definition: Kokkos_Pair.hpp:44
Kokkos::LayoutStride
Memory layout tag indicated arbitrarily strided multi-index mapping into contiguous memory...
Definition: Kokkos_Layout.hpp:148
Kokkos::HostSpace
Memory management for host memory.
Definition: Kokkos_HostSpace.hpp:49
Kokkos::LayoutRight
Memory layout tag indicating right-to-left (C or lexigraphical scheme) striding of multi-indices...
Definition: Kokkos_Layout.hpp:104
Kokkos::HostSpace::name
static constexpr const char * name()
Kokkos::RangePolicy
Execution policy for work over a range of an integral type.
Definition: Kokkos_ExecPolicy.hpp:76
Generated on Tue Apr 22 2025 09:19:24 for Kokkos Core Kernels Package by   doxygen 1.8.5