51 #ifndef KOKKOS_SCATTER_VIEW_HPP
52 #define KOKKOS_SCATTER_VIEW_HPP
54 #include <Kokkos_Core.hpp>
58 namespace Experimental {
73 ScatterNonDuplicated = 0,
86 namespace Experimental {
88 template <
typename ExecSpace>
89 struct DefaultDuplication;
91 template <
typename ExecSpace,
int duplication>
92 struct DefaultContribution;
94 #ifdef KOKKOS_ENABLE_SERIAL
96 struct DefaultDuplication<Kokkos::Serial> {
97 enum :
int { value = Kokkos::Experimental::ScatterNonDuplicated };
100 struct DefaultContribution<Kokkos::Serial, Kokkos::Experimental::ScatterNonDuplicated> {
101 enum :
int { value = Kokkos::Experimental::ScatterNonAtomic };
104 struct DefaultContribution<Kokkos::Serial, Kokkos::Experimental::ScatterDuplicated> {
105 enum :
int { value = Kokkos::Experimental::ScatterNonAtomic };
109 #ifdef KOKKOS_ENABLE_OPENMP
111 struct DefaultDuplication<Kokkos::OpenMP> {
112 enum :
int { value = Kokkos::Experimental::ScatterDuplicated };
115 struct DefaultContribution<Kokkos::OpenMP, Kokkos::Experimental::ScatterNonDuplicated> {
116 enum :
int { value = Kokkos::Experimental::ScatterAtomic };
119 struct DefaultContribution<Kokkos::OpenMP, Kokkos::Experimental::ScatterDuplicated> {
120 enum :
int { value = Kokkos::Experimental::ScatterNonAtomic };
124 #ifdef KOKKOS_ENABLE_HPX
126 struct DefaultDuplication<Kokkos::Experimental::HPX> {
127 enum :
int { value = Kokkos::Experimental::ScatterDuplicated };
130 struct DefaultContribution<Kokkos::Experimental::HPX, Kokkos::Experimental::ScatterNonDuplicated> {
131 enum :
int { value = Kokkos::Experimental::ScatterAtomic };
134 struct DefaultContribution<Kokkos::Experimental::HPX, Kokkos::Experimental::ScatterDuplicated> {
135 enum :
int { value = Kokkos::Experimental::ScatterNonAtomic };
139 #ifdef KOKKOS_ENABLE_THREADS
141 struct DefaultDuplication<Kokkos::Threads> {
142 enum :
int { value = Kokkos::Experimental::ScatterDuplicated };
145 struct DefaultContribution<Kokkos::Threads, Kokkos::Experimental::ScatterNonDuplicated> {
146 enum :
int { value = Kokkos::Experimental::ScatterAtomic };
149 struct DefaultContribution<Kokkos::Threads, Kokkos::Experimental::ScatterDuplicated> {
150 enum :
int { value = Kokkos::Experimental::ScatterNonAtomic };
154 #ifdef KOKKOS_ENABLE_CUDA
156 struct DefaultDuplication<Kokkos::Cuda> {
157 enum :
int { value = Kokkos::Experimental::ScatterNonDuplicated };
160 struct DefaultContribution<Kokkos::Cuda, Kokkos::Experimental::ScatterNonDuplicated> {
161 enum :
int { value = Kokkos::Experimental::ScatterAtomic };
164 struct DefaultContribution<Kokkos::Cuda, Kokkos::Experimental::ScatterDuplicated> {
165 enum :
int { value = Kokkos::Experimental::ScatterAtomic };
173 template <
typename ValueType,
int Op,
int contribution>
176 template <
typename ValueType>
177 struct ScatterValue<ValueType, Kokkos::Experimental::ScatterSum, Kokkos::Experimental::ScatterNonAtomic> :
178 Sum<ValueType,Kokkos::DefaultExecutionSpace> {
180 KOKKOS_FORCEINLINE_FUNCTION ScatterValue(ValueType& value_in) :
181 Sum<ValueType,Kokkos::DefaultExecutionSpace>(value_in)
183 KOKKOS_FORCEINLINE_FUNCTION ScatterValue(ScatterValue&& other) :
184 Sum<ValueType,Kokkos::DefaultExecutionSpace>(other.reference())
186 KOKKOS_FORCEINLINE_FUNCTION
void operator+=(ValueType
const& rhs) {
187 this->join( this->reference(), rhs );
189 KOKKOS_FORCEINLINE_FUNCTION
void operator-=(ValueType
const& rhs) {
190 this->join( this->reference(), -rhs );
192 KOKKOS_FORCEINLINE_FUNCTION
void update(ValueType
const& rhs) {
193 this->join( this->reference(), rhs );
195 KOKKOS_FORCEINLINE_FUNCTION
void reset() {
196 this->init( this->reference() );
203 template <
typename ValueType>
204 struct ScatterValue<ValueType, Kokkos::Experimental::ScatterSum, Kokkos::Experimental::ScatterAtomic> :
205 Sum<ValueType,Kokkos::DefaultExecutionSpace> {
207 KOKKOS_FORCEINLINE_FUNCTION ScatterValue(ValueType& value_in) :
208 Sum<ValueType,Kokkos::DefaultExecutionSpace>(value_in)
211 KOKKOS_FORCEINLINE_FUNCTION
void operator+=(ValueType
const& rhs) {
212 this->join(this->reference(), rhs);
214 KOKKOS_FORCEINLINE_FUNCTION
void operator-=(ValueType
const& rhs) {
215 this->join(this->reference(), -rhs);
218 KOKKOS_INLINE_FUNCTION
219 void join(ValueType& dest,
const ValueType& src)
const {
220 Kokkos::atomic_add(&dest, src);
223 KOKKOS_INLINE_FUNCTION
224 void join(
volatile ValueType& dest,
const volatile ValueType& src)
const {
225 Kokkos::atomic_add(&dest, src);
228 KOKKOS_FORCEINLINE_FUNCTION
void update(ValueType
const& rhs) {
229 this->join( this->reference(), rhs );
232 KOKKOS_FORCEINLINE_FUNCTION
void reset() {
233 this->init( this->reference() );
241 template <
typename ValueType>
242 struct ScatterValue<ValueType, Kokkos::Experimental::ScatterProd, Kokkos::Experimental::ScatterNonAtomic> :
243 Prod<ValueType,Kokkos::DefaultExecutionSpace> {
245 KOKKOS_FORCEINLINE_FUNCTION ScatterValue(ValueType& value_in) :
246 Prod<ValueType,Kokkos::DefaultExecutionSpace>(value_in)
248 KOKKOS_FORCEINLINE_FUNCTION ScatterValue(ScatterValue&& other) :
249 Prod<ValueType,Kokkos::DefaultExecutionSpace>(other.reference())
251 KOKKOS_FORCEINLINE_FUNCTION
void operator*=(ValueType
const& rhs) {
252 this->join( this->reference(), rhs );
254 KOKKOS_FORCEINLINE_FUNCTION
void operator/=(ValueType
const& rhs) {
255 this->join( this->reference(), static_cast<ValueType>(1)/rhs );
257 KOKKOS_FORCEINLINE_FUNCTION
void update(ValueType
const& rhs) {
258 this->join( this->reference(), rhs );
260 KOKKOS_FORCEINLINE_FUNCTION
void reset() {
261 this->init( this->reference() );
268 template <
typename ValueType>
269 struct ScatterValue<ValueType, Kokkos::Experimental::ScatterProd, Kokkos::Experimental::ScatterAtomic> :
270 Prod<ValueType,Kokkos::DefaultExecutionSpace> {
272 KOKKOS_FORCEINLINE_FUNCTION ScatterValue(ValueType& value_in) :
273 Prod<ValueType,Kokkos::DefaultExecutionSpace>(value_in)
276 KOKKOS_FORCEINLINE_FUNCTION
void operator*=(ValueType
const& rhs) {
277 this->join(this->reference(), rhs);
279 KOKKOS_FORCEINLINE_FUNCTION
void operator/=(ValueType
const& rhs) {
280 this->join(this->reference(), static_cast<ValueType>(1)/rhs);
283 KOKKOS_FORCEINLINE_FUNCTION
284 void atomic_prod(ValueType & dest,
const ValueType& src)
const {
286 bool success =
false;
288 ValueType dest_old = dest;
289 ValueType dest_new = dest_old * src;
290 dest_new = Kokkos::atomic_compare_exchange<ValueType>(&dest,dest_old,dest_new);
291 success = ( (dest_new - dest_old)/dest_old <= 1e-15 );
295 KOKKOS_INLINE_FUNCTION
296 void join(ValueType& dest,
const ValueType& src)
const {
297 atomic_prod(dest, src);
300 KOKKOS_INLINE_FUNCTION
301 void join(
volatile ValueType& dest,
const volatile ValueType& src)
const {
302 atomic_prod(dest, src);
305 KOKKOS_FORCEINLINE_FUNCTION
void update(ValueType
const& rhs) {
306 this->join( this->reference(), rhs );
308 KOKKOS_FORCEINLINE_FUNCTION
void reset() {
309 this->init( this->reference() );
318 template <
typename ValueType>
319 struct ScatterValue<ValueType, Kokkos::Experimental::ScatterMin, Kokkos::Experimental::ScatterNonAtomic> :
320 Min<ValueType,Kokkos::DefaultExecutionSpace> {
322 KOKKOS_FORCEINLINE_FUNCTION ScatterValue(ValueType& value_in) :
323 Min<ValueType,Kokkos::DefaultExecutionSpace>(value_in)
325 KOKKOS_FORCEINLINE_FUNCTION ScatterValue(ScatterValue&& other) :
326 Min<ValueType,Kokkos::DefaultExecutionSpace>(other.reference())
328 KOKKOS_FORCEINLINE_FUNCTION
void update(ValueType
const& rhs) {
329 this->join( this->reference(), rhs );
331 KOKKOS_FORCEINLINE_FUNCTION
void reset() {
332 this->init( this->reference() );
339 template <
typename ValueType>
340 struct ScatterValue<ValueType, Kokkos::Experimental::ScatterMin, Kokkos::Experimental::ScatterAtomic> :
341 Min<ValueType,Kokkos::DefaultExecutionSpace> {
343 KOKKOS_FORCEINLINE_FUNCTION ScatterValue(ValueType& value_in) :
344 Min<ValueType,Kokkos::DefaultExecutionSpace>(value_in)
347 KOKKOS_FORCEINLINE_FUNCTION
348 void atomic_min(ValueType & dest,
const ValueType& src)
const {
350 bool success =
false;
352 ValueType dest_old = dest;
353 ValueType dest_new = ( dest_old > src ) ? src : dest_old;
354 dest_new = Kokkos::atomic_compare_exchange<ValueType>(&dest,dest_old,dest_new);
355 success = ( (dest_new - dest_old)/dest_old <= 1e-15 );
359 KOKKOS_INLINE_FUNCTION
360 void join(ValueType& dest,
const ValueType& src)
const {
361 atomic_min(dest, src);
364 KOKKOS_INLINE_FUNCTION
365 void join(
volatile ValueType& dest,
const volatile ValueType& src)
const {
366 atomic_min(dest, src);
369 KOKKOS_FORCEINLINE_FUNCTION
void update(ValueType
const& rhs) {
370 this->join( this->reference(), rhs );
372 KOKKOS_FORCEINLINE_FUNCTION
void reset() {
373 this->init( this->reference() );
382 template <
typename ValueType>
383 struct ScatterValue<ValueType, Kokkos::Experimental::ScatterMax, Kokkos::Experimental::ScatterNonAtomic> :
384 Max<ValueType,Kokkos::DefaultExecutionSpace> {
386 KOKKOS_FORCEINLINE_FUNCTION ScatterValue(ValueType& value_in) :
387 Max<ValueType,Kokkos::DefaultExecutionSpace>(value_in)
389 KOKKOS_FORCEINLINE_FUNCTION ScatterValue(ScatterValue&& other) :
390 Max<ValueType,Kokkos::DefaultExecutionSpace>(other.reference())
392 KOKKOS_FORCEINLINE_FUNCTION
void update(ValueType
const& rhs) {
393 this->join( this->reference(), rhs );
395 KOKKOS_FORCEINLINE_FUNCTION
void reset() {
396 this->init( this->reference() );
403 template <
typename ValueType>
404 struct ScatterValue<ValueType, Kokkos::Experimental::ScatterMax, Kokkos::Experimental::ScatterAtomic> :
405 Max<ValueType,Kokkos::DefaultExecutionSpace> {
407 KOKKOS_FORCEINLINE_FUNCTION ScatterValue(ValueType& value_in) :
408 Max<ValueType,Kokkos::DefaultExecutionSpace>(value_in)
411 KOKKOS_FORCEINLINE_FUNCTION
412 void atomic_max(ValueType & dest,
const ValueType& src)
const {
414 bool success =
false;
416 ValueType dest_old = dest;
417 ValueType dest_new = ( dest_old < src ) ? src : dest_old;
418 dest_new = Kokkos::atomic_compare_exchange<ValueType>(&dest,dest_old,dest_new);
419 success = ( (dest_new - dest_old)/dest_old <= 1e-15 );
423 KOKKOS_INLINE_FUNCTION
424 void join(ValueType& dest,
const ValueType& src)
const {
425 atomic_max(dest, src);
428 KOKKOS_INLINE_FUNCTION
429 void join(
volatile ValueType& dest,
const volatile ValueType& src)
const {
430 atomic_max(dest, src);
433 KOKKOS_FORCEINLINE_FUNCTION
void update(ValueType
const& rhs) {
434 this->join( this->reference(), rhs );
436 KOKKOS_FORCEINLINE_FUNCTION
void reset() {
437 this->init( this->reference() );
446 template <
typename T,
typename Layout>
447 struct DuplicatedDataType;
449 template <
typename T>
450 struct DuplicatedDataType<T, Kokkos::LayoutRight> {
451 typedef T* value_type;
454 template <
typename T,
size_t N>
455 struct DuplicatedDataType<T[N], Kokkos::LayoutRight> {
456 typedef typename DuplicatedDataType<T, Kokkos::LayoutRight>::value_type value_type[N];
459 template <
typename T>
460 struct DuplicatedDataType<T[], Kokkos::LayoutRight> {
461 typedef typename DuplicatedDataType<T, Kokkos::LayoutRight>::value_type value_type[];
464 template <
typename T>
466 typedef typename DuplicatedDataType<T, Kokkos::LayoutRight>::value_type* value_type;
469 template <
typename T>
470 struct DuplicatedDataType<T, Kokkos::LayoutLeft> {
471 typedef T* value_type;
474 template <
typename T,
size_t N>
475 struct DuplicatedDataType<T[N], Kokkos::LayoutLeft> {
476 typedef typename DuplicatedDataType<T, Kokkos::LayoutLeft>::value_type* value_type;
479 template <
typename T>
480 struct DuplicatedDataType<T[], Kokkos::LayoutLeft> {
481 typedef typename DuplicatedDataType<T, Kokkos::LayoutLeft>::value_type* value_type;
484 template <
typename T>
486 typedef typename DuplicatedDataType<T, Kokkos::LayoutLeft>::value_type* value_type;
492 void args_to_array(
size_t* array,
int pos, T dim0) {
495 template<
class T,
class ... Dims>
496 void args_to_array(
size_t* array,
int pos, T dim0, Dims ... dims) {
498 args_to_array(array,pos+1,dims...);
504 template <
typename Layout,
int rank,
typename V,
typename ... Args>
506 typedef Slice<Layout,
rank - 1, V, Kokkos::Impl::ALL_t, Args...> next;
507 typedef typename next::value_type value_type;
510 value_type
get(V
const& src,
const size_t i, Args ... args) {
511 return next::get(src, i, Kokkos::ALL, args...);
515 template <
typename V,
typename ... Args>
516 struct Slice<Kokkos::LayoutRight, 1, V, Args...> {
517 typedef typename Kokkos::Impl::ViewMapping
524 value_type
get(V
const& src,
const size_t i, Args ... args) {
525 return Kokkos::subview(src, i, args...);
529 template <
typename V,
typename ... Args>
530 struct Slice<Kokkos::LayoutLeft, 1, V, Args...> {
531 typedef typename Kokkos::Impl::ViewMapping
538 value_type
get(V
const& src,
const size_t i, Args ... args) {
539 return Kokkos::subview(src, args..., i);
543 template <
typename ExecSpace,
typename ValueType,
int Op>
544 struct ReduceDuplicates;
546 template <
typename ExecSpace,
typename ValueType,
int Op>
547 struct ReduceDuplicatesBase {
548 typedef ReduceDuplicates<ExecSpace, ValueType, Op> Derived;
549 ValueType
const* src;
554 ReduceDuplicatesBase(ValueType
const* src_in, ValueType* dest_in,
size_t stride_in,
size_t start_in,
size_t n_in, std::string
const& name)
561 #if defined(KOKKOS_ENABLE_PROFILING)
563 if(Kokkos::Profiling::profileLibraryLoaded()) {
564 Kokkos::Profiling::beginParallelFor(std::string(
"reduce_") + name, 0, &kpID);
567 typedef RangePolicy<ExecSpace, size_t> policy_type;
569 const closure_type closure(*(static_cast<Derived*>(
this)), policy_type(0, stride));
571 #if defined(KOKKOS_ENABLE_PROFILING)
572 if(Kokkos::Profiling::profileLibraryLoaded()) {
573 Kokkos::Profiling::endParallelFor(kpID);
582 template <
typename ExecSpace,
typename ValueType,
int Op>
583 struct ReduceDuplicates :
584 public ReduceDuplicatesBase<ExecSpace, ValueType, Op>
586 typedef ReduceDuplicatesBase<ExecSpace, ValueType, Op> Base;
587 ReduceDuplicates(ValueType
const* src_in, ValueType* dst_in,
size_t stride_in,
size_t start_in,
size_t n_in, std::string
const& name):
588 Base(src_in, dst_in, stride_in, start_in, n_in, name)
590 KOKKOS_FORCEINLINE_FUNCTION
void operator()(
size_t i)
const {
591 for (
size_t j = Base::start; j < Base::n; ++j) {
592 ScatterValue<ValueType, Op, Kokkos::Experimental::ScatterNonAtomic> sv(Base::dst[i]);
593 sv.update( Base::src[i + Base::stride * j] );
599 template <
typename ExecSpace,
typename ValueType,
int Op>
600 struct ResetDuplicates;
602 template <
typename ExecSpace,
typename ValueType,
int Op>
603 struct ResetDuplicatesBase {
604 typedef ResetDuplicates<ExecSpace, ValueType, Op> Derived;
606 ResetDuplicatesBase(ValueType* data_in,
size_t size_in, std::string
const& name)
609 #if defined(KOKKOS_ENABLE_PROFILING)
611 if(Kokkos::Profiling::profileLibraryLoaded()) {
612 Kokkos::Profiling::beginParallelFor(std::string(
"reduce_") + name, 0, &kpID);
615 typedef RangePolicy<ExecSpace, size_t> policy_type;
617 const closure_type closure(*(static_cast<Derived*>(
this)), policy_type(0, size_in));
619 #if defined(KOKKOS_ENABLE_PROFILING)
620 if(Kokkos::Profiling::profileLibraryLoaded()) {
621 Kokkos::Profiling::endParallelFor(kpID);
630 template <
typename ExecSpace,
typename ValueType,
int Op>
631 struct ResetDuplicates :
632 public ResetDuplicatesBase<ExecSpace, ValueType, Op>
634 typedef ResetDuplicatesBase<ExecSpace, ValueType, Op> Base;
635 ResetDuplicates(ValueType* data_in,
size_t size_in, std::string
const& name):
636 Base(data_in, size_in, name)
638 KOKKOS_FORCEINLINE_FUNCTION
void operator()(
size_t i)
const {
639 ScatterValue<ValueType, Op, Kokkos::Experimental::ScatterNonAtomic> sv(Base::data[i]);
648 namespace Experimental {
650 template <
typename DataType
651 ,
typename Layout = Kokkos::DefaultExecutionSpace::array_layout
652 ,
typename ExecSpace = Kokkos::DefaultExecutionSpace
654 ,
int duplication = Kokkos::Impl::Experimental::DefaultDuplication<ExecSpace>::value
655 ,
int contribution = Kokkos::Impl::Experimental::DefaultContribution<ExecSpace, duplication>::value
659 template <
typename DataType
665 ,
int override_contribution
670 template <
typename DataType
676 class ScatterView<DataType
680 ,ScatterNonDuplicated
685 typedef typename original_view_type::value_type original_value_type;
686 typedef typename original_view_type::reference_type original_reference_type;
687 friend class ScatterAccess<DataType, Op, ExecSpace, Layout, ScatterNonDuplicated, contribution, ScatterNonAtomic>;
688 friend class ScatterAccess<DataType, Op, ExecSpace, Layout, ScatterNonDuplicated, contribution, ScatterAtomic>;
694 template <
typename RT,
typename ... RP>
695 ScatterView(View<RT, RP...>
const& original_view)
696 : internal_view(original_view)
700 template <
typename ... Dims>
701 ScatterView(std::string
const& name, Dims ... dims)
702 : internal_view(name, dims ...)
706 template <
int overr
ide_contrib = contribution>
707 KOKKOS_FORCEINLINE_FUNCTION
708 ScatterAccess<DataType, Op, ExecSpace, Layout, ScatterNonDuplicated, contribution, override_contrib>
710 return ScatterAccess<DataType, Op, ExecSpace, Layout, ScatterNonDuplicated, contribution, override_contrib>{*
this};
713 original_view_type subview()
const {
714 return internal_view;
717 template <
typename DT,
typename ... RP>
718 void contribute_into(View<DT, RP...>
const& dest)
const
720 typedef View<DT, RP...> dest_type;
721 static_assert(std::is_same<
722 typename dest_type::array_layout,
724 "ScatterView contribute destination has different layout");
725 static_assert(Kokkos::Impl::VerifyExecutionCanAccessMemorySpace<
726 typename ExecSpace::memory_space,
727 typename dest_type::memory_space>::value,
728 "ScatterView contribute destination memory space not accessible");
729 if (dest.data() == internal_view.data())
return;
730 Kokkos::Impl::Experimental::ReduceDuplicates<ExecSpace, original_value_type, Op>(
731 internal_view.data(),
736 internal_view.label());
740 Kokkos::Impl::Experimental::ResetDuplicates<ExecSpace, original_value_type, Op>(
741 internal_view.data(),
742 internal_view.size(),
743 internal_view.label());
745 template <
typename DT,
typename ... RP>
746 void reset_except(View<DT, RP...>
const& view) {
747 if (view.data() != internal_view.data()) reset();
750 void resize(
const size_t n0 = 0,
757 const size_t n7 = 0) {
761 void realloc(
const size_t n0 = 0,
768 const size_t n7 = 0) {
773 template <
typename ... Args>
774 KOKKOS_FORCEINLINE_FUNCTION
775 original_reference_type at(Args ... args)
const {
776 return internal_view(args...);
779 typedef original_view_type internal_view_type;
780 internal_view_type internal_view;
783 template <
typename DataType
788 ,
int override_contribution
790 class ScatterAccess<DataType
794 ,ScatterNonDuplicated
796 ,override_contribution>
799 typedef ScatterView<DataType, Layout, ExecSpace, Op, ScatterNonDuplicated, contribution> view_type;
800 typedef typename view_type::original_value_type original_value_type;
801 typedef Kokkos::Impl::Experimental::ScatterValue<
802 original_value_type, Op, override_contribution> value_type;
804 KOKKOS_INLINE_FUNCTION
809 KOKKOS_INLINE_FUNCTION
810 ScatterAccess(view_type
const& view_in)
815 KOKKOS_INLINE_FUNCTION
820 template <
typename ... Args>
821 KOKKOS_FORCEINLINE_FUNCTION
822 value_type operator()(Args ... args)
const {
823 return view.at(args...);
826 template <
typename Arg>
827 KOKKOS_FORCEINLINE_FUNCTION
828 typename std::enable_if<view_type::original_view_type::rank == 1 &&
829 std::is_integral<Arg>::value, value_type>::type
830 operator[](Arg arg)
const {
835 view_type
const& view;
841 template <
typename DataType
846 class ScatterView<DataType
855 typedef typename original_view_type::value_type original_value_type;
856 typedef typename original_view_type::reference_type original_reference_type;
857 friend class ScatterAccess<DataType, Op, ExecSpace, Kokkos::LayoutRight, ScatterDuplicated, contribution, ScatterNonAtomic>;
858 friend class ScatterAccess<DataType, Op, ExecSpace, Kokkos::LayoutRight, ScatterDuplicated, contribution, ScatterAtomic>;
859 typedef typename Kokkos::Impl::Experimental::DuplicatedDataType<DataType, Kokkos::LayoutRight> data_type_info;
860 typedef typename data_type_info::value_type internal_data_type;
861 typedef
Kokkos::View<internal_data_type, Kokkos::LayoutRight, ExecSpace> internal_view_type;
867 template <
typename RT,
typename ... RP >
868 ScatterView(View<RT, RP...>
const& original_view)
870 , internal_view(Kokkos::ViewAllocateWithoutInitializing(
871 std::string(
"duplicated_") + original_view.label()),
873 #ifdef KOKKOS_ENABLE_DEPRECATED_CODE
874 original_view.extent(0),
875 original_view.extent(1),
876 original_view.extent(2),
877 original_view.extent(3),
878 original_view.extent(4),
879 original_view.extent(5),
880 original_view.extent(6) )
882 original_view.rank_dynamic > 0 ? original_view.extent(0): KOKKOS_IMPL_CTOR_DEFAULT_ARG,
883 original_view.rank_dynamic > 1 ? original_view.extent(1): KOKKOS_IMPL_CTOR_DEFAULT_ARG,
884 original_view.rank_dynamic > 2 ? original_view.extent(2): KOKKOS_IMPL_CTOR_DEFAULT_ARG,
885 original_view.rank_dynamic > 3 ? original_view.extent(3): KOKKOS_IMPL_CTOR_DEFAULT_ARG,
886 original_view.rank_dynamic > 4 ? original_view.extent(4): KOKKOS_IMPL_CTOR_DEFAULT_ARG,
887 original_view.rank_dynamic > 5 ? original_view.extent(5): KOKKOS_IMPL_CTOR_DEFAULT_ARG,
888 original_view.rank_dynamic > 6 ? original_view.extent(6): KOKKOS_IMPL_CTOR_DEFAULT_ARG)
895 template <
typename ... Dims>
896 ScatterView(std::string
const& name, Dims ... dims)
897 : internal_view(Kokkos::ViewAllocateWithoutInitializing(name), unique_token.size(), dims ...)
902 template <
int overr
ide_contribution = contribution>
903 KOKKOS_FORCEINLINE_FUNCTION
904 ScatterAccess<DataType, Op, ExecSpace, Kokkos::LayoutRight, ScatterDuplicated, contribution, override_contribution>
906 return ScatterAccess<DataType, Op, ExecSpace, Kokkos::LayoutRight, ScatterDuplicated, contribution, override_contribution>{*
this};
909 typename Kokkos::Impl::Experimental::Slice<
913 return Kokkos::Impl::Experimental::Slice<
917 template <
typename DT,
typename ... RP>
918 void contribute_into(View<DT, RP...>
const& dest)
const
920 typedef View<DT, RP...> dest_type;
921 static_assert(std::is_same<
922 typename dest_type::array_layout,
924 "ScatterView deep_copy destination has different layout");
925 static_assert(Kokkos::Impl::VerifyExecutionCanAccessMemorySpace<
926 typename ExecSpace::memory_space,
927 typename dest_type::memory_space>::value,
928 "ScatterView deep_copy destination memory space not accessible");
929 bool is_equal = (dest.data() == internal_view.data());
930 size_t start = is_equal ? 1 : 0;
931 Kokkos::Impl::Experimental::ReduceDuplicates<ExecSpace, original_value_type, Op>(
932 internal_view.data(),
934 internal_view.stride(0),
936 internal_view.extent(0),
937 internal_view.label());
941 Kokkos::Impl::Experimental::ResetDuplicates<ExecSpace, original_value_type, Op>(
942 internal_view.data(),
943 internal_view.size(),
944 internal_view.label());
946 template <
typename DT,
typename ... RP>
947 void reset_except(View<DT, RP...>
const& view) {
948 if (view.data() != internal_view.data()) {
952 Kokkos::Impl::Experimental::ResetDuplicates<ExecSpace, original_value_type, Op>(
953 internal_view.data() + view.size(),
954 internal_view.size() - view.size(),
955 internal_view.label());
958 void resize(
const size_t n0 = 0,
964 const size_t n6 = 0) {
968 void realloc(
const size_t n0 = 0,
974 const size_t n6 = 0) {
979 template <
typename ... Args>
980 KOKKOS_FORCEINLINE_FUNCTION
981 original_reference_type at(
int rank, Args ... args)
const {
982 return internal_view(rank, args...);
987 ExecSpace, Kokkos::Experimental::UniqueTokenScope::Global> unique_token_type;
989 unique_token_type unique_token;
990 internal_view_type internal_view;
993 template <
typename DataType
998 class ScatterView<DataType
1007 typedef typename original_view_type::value_type original_value_type;
1008 typedef typename original_view_type::reference_type original_reference_type;
1009 friend class ScatterAccess<DataType, Op, ExecSpace, Kokkos::LayoutLeft, ScatterDuplicated, contribution, ScatterNonAtomic>;
1010 friend class ScatterAccess<DataType, Op, ExecSpace, Kokkos::LayoutLeft, ScatterDuplicated, contribution, ScatterAtomic>;
1011 typedef typename Kokkos::Impl::Experimental::DuplicatedDataType<DataType, Kokkos::LayoutLeft> data_type_info;
1012 typedef typename data_type_info::value_type internal_data_type;
1013 typedef
Kokkos::View<internal_data_type, Kokkos::LayoutLeft, ExecSpace> internal_view_type;
1019 template <
typename RT,
typename ... RP >
1020 ScatterView(View<RT, RP...>
const& original_view)
1024 original_view.rank>0?original_view.extent(0):KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1025 original_view.rank>1?original_view.extent(1):KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1026 original_view.rank>2?original_view.extent(2):KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1027 original_view.rank>3?original_view.extent(3):KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1028 original_view.rank>4?original_view.extent(4):KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1029 original_view.rank>5?original_view.extent(5):KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1030 original_view.rank>6?original_view.extent(6):KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1031 KOKKOS_IMPL_CTOR_DEFAULT_ARG
1033 arg_N[internal_view_type::rank - 1] = unique_token.size();
1034 internal_view = internal_view_type(
1035 Kokkos::ViewAllocateWithoutInitializing(
1036 std::string(
"duplicated_") + original_view.label()),
1037 arg_N[0], arg_N[1], arg_N[2], arg_N[3],
1038 arg_N[4], arg_N[5], arg_N[6], arg_N[7]);
1042 template <
typename ... Dims>
1043 ScatterView(std::string
const& name, Dims ... dims) {
1044 original_view_type original_view;
1046 original_view.rank>0?original_view.static_extent(0):KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1047 original_view.rank>1?original_view.static_extent(1):KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1048 original_view.rank>2?original_view.static_extent(2):KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1049 original_view.rank>3?original_view.static_extent(3):KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1050 original_view.rank>4?original_view.static_extent(4):KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1051 original_view.rank>5?original_view.static_extent(5):KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1052 original_view.rank>6?original_view.static_extent(6):KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1053 KOKKOS_IMPL_CTOR_DEFAULT_ARG
1055 Kokkos::Impl::Experimental::args_to_array(arg_N,0,dims ...);
1056 arg_N[internal_view_type::rank - 1] = unique_token.size();
1057 internal_view = internal_view_type(Kokkos::ViewAllocateWithoutInitializing(name),
1058 arg_N[0], arg_N[1], arg_N[2], arg_N[3],
1059 arg_N[4], arg_N[5], arg_N[6], arg_N[7]);
1063 template <
int overr
ide_contribution = contribution>
1064 KOKKOS_FORCEINLINE_FUNCTION
1065 ScatterAccess<DataType, Op, ExecSpace, Kokkos::LayoutLeft, ScatterDuplicated, contribution, override_contribution>
1067 return ScatterAccess<DataType, Op, ExecSpace, Kokkos::LayoutLeft, ScatterDuplicated, contribution, override_contribution>{*
this};
1070 typename Kokkos::Impl::Experimental::Slice<
1074 return Kokkos::Impl::Experimental::Slice<
1075 Kokkos::LayoutLeft, internal_view_type::rank, internal_view_type>::get(internal_view, 0);
1078 template <
typename ... RP>
1079 void contribute_into(View<RP...>
const& dest)
const
1081 typedef View<RP...> dest_type;
1082 static_assert(std::is_same<
1083 typename dest_type::value_type,
1084 typename original_view_type::non_const_value_type>::value,
1085 "ScatterView deep_copy destination has wrong value_type");
1086 static_assert(std::is_same<
1087 typename dest_type::array_layout,
1089 "ScatterView deep_copy destination has different layout");
1090 static_assert(Kokkos::Impl::VerifyExecutionCanAccessMemorySpace<
1091 typename ExecSpace::memory_space,
1092 typename dest_type::memory_space>::value,
1093 "ScatterView deep_copy destination memory space not accessible");
1094 auto extent = internal_view.extent(
1095 internal_view_type::rank - 1);
1096 bool is_equal = (dest.data() == internal_view.data());
1097 size_t start = is_equal ? 1 : 0;
1098 Kokkos::Impl::Experimental::ReduceDuplicates<ExecSpace, original_value_type, Op>(
1099 internal_view.data(),
1101 internal_view.stride(internal_view_type::rank - 1),
1104 internal_view.label());
1108 Kokkos::Impl::Experimental::ResetDuplicates<ExecSpace, original_value_type, Op>(
1109 internal_view.data(),
1110 internal_view.size(),
1111 internal_view.label());
1113 template <
typename DT,
typename ... RP>
1114 void reset_except(View<DT, RP...>
const& view) {
1115 if (view.data() != internal_view.data()) {
1119 Kokkos::Impl::Experimental::ResetDuplicates<ExecSpace, original_value_type, Op>(
1120 internal_view.data() + view.size(),
1121 internal_view.size() - view.size(),
1122 internal_view.label());
1125 void resize(
const size_t n0 = 0,
1126 const size_t n1 = 0,
1127 const size_t n2 = 0,
1128 const size_t n3 = 0,
1129 const size_t n4 = 0,
1130 const size_t n5 = 0,
1131 const size_t n6 = 0) {
1133 size_t arg_N[8] = {n0,n1,n2,n3,n4,n5,n6,0};
1134 const int i = internal_view.rank-1;
1135 arg_N[i] = unique_token.size();
1138 arg_N[0], arg_N[1], arg_N[2], arg_N[3],
1139 arg_N[4], arg_N[5], arg_N[6], arg_N[7]);
1142 void realloc(
const size_t n0 = 0,
1143 const size_t n1 = 0,
1144 const size_t n2 = 0,
1145 const size_t n3 = 0,
1146 const size_t n4 = 0,
1147 const size_t n5 = 0,
1148 const size_t n6 = 0) {
1150 size_t arg_N[8] = {n0,n1,n2,n3,n4,n5,n6,0};
1151 const int i = internal_view.rank-1;
1152 arg_N[i] = unique_token.size();
1155 arg_N[0], arg_N[1], arg_N[2], arg_N[3],
1156 arg_N[4], arg_N[5], arg_N[6], arg_N[7]);
1160 template <
typename ... Args>
1161 inline original_reference_type at(
int thread_id, Args ... args)
const {
1162 return internal_view(args..., thread_id);
1167 ExecSpace, Kokkos::Experimental::UniqueTokenScope::Global> unique_token_type;
1169 unique_token_type unique_token;
1170 internal_view_type internal_view;
1183 template <
typename DataType
1188 ,
int override_contribution
1190 class ScatterAccess<DataType
1196 ,override_contribution>
1199 typedef ScatterView<DataType, Layout, ExecSpace, Op, ScatterDuplicated, contribution> view_type;
1200 typedef typename view_type::original_value_type original_value_type;
1201 typedef Kokkos::Impl::Experimental::ScatterValue<
1202 original_value_type, Op, override_contribution> value_type;
1204 KOKKOS_FORCEINLINE_FUNCTION
1205 ScatterAccess(view_type
const& view_in)
1207 , thread_id(view_in.unique_token.acquire()) {
1210 KOKKOS_FORCEINLINE_FUNCTION
1212 if (thread_id != ~thread_id_type(0)) view.unique_token.release(thread_id);
1215 template <
typename ... Args>
1216 KOKKOS_FORCEINLINE_FUNCTION
1217 value_type operator()(Args ... args)
const {
1218 return view.at(thread_id, args...);
1221 template <
typename Arg>
1222 KOKKOS_FORCEINLINE_FUNCTION
1223 typename std::enable_if<view_type::original_view_type::rank == 1 &&
1224 std::is_integral<Arg>::value, value_type>::type
1225 operator[](Arg arg)
const {
1226 return view.at(thread_id, arg);
1231 view_type
const& view;
1234 ScatterAccess(ScatterAccess
const& other) =
delete;
1235 ScatterAccess& operator=(ScatterAccess
const& other) =
delete;
1236 ScatterAccess& operator=(ScatterAccess&& other) =
delete;
1242 KOKKOS_FORCEINLINE_FUNCTION
1243 ScatterAccess(ScatterAccess&& other)
1245 , thread_id(other.thread_id)
1247 other.thread_id = ~thread_id_type(0);
1252 typedef typename view_type::unique_token_type unique_token_type;
1253 typedef typename unique_token_type::size_type thread_id_type;
1254 thread_id_type thread_id;
1257 template <
int Op = Kokkos::Experimental::ScatterSum,
1258 int duplication = -1,
1259 int contribution = -1,
1260 typename RT,
typename ... RP>
1263 ,
typename ViewTraits<RT, RP...>::array_layout
1264 ,
typename ViewTraits<RT, RP...>::execution_space
1269 , duplication == -1 ? Kokkos::Impl::Experimental::DefaultDuplication<
typename ViewTraits<RT, RP...>::execution_space>::value : duplication
1270 , contribution == -1 ?
1271 Kokkos::Impl::Experimental::DefaultContribution<
1272 typename ViewTraits<RT, RP...>::execution_space,
1273 (duplication == -1 ?
1274 Kokkos::Impl::Experimental::DefaultDuplication<
1275 typename ViewTraits<RT, RP...>::execution_space
1282 create_scatter_view(View<RT, RP...>
const& original_view) {
1283 return original_view;
1289 namespace Experimental {
1291 template <
typename DT1,
typename DT2,
typename LY,
typename ES,
int OP,
int CT,
int DP,
typename ... VP>
1293 contribute(View<DT1, VP...>& dest, Kokkos::Experimental::ScatterView<DT2, LY, ES, OP, CT, DP>
const& src)
1295 src.contribute_into(dest);
1302 template <
typename DT,
typename LY,
typename ES,
int OP,
int CT,
int DP,
typename ... IS>
1304 realloc(Kokkos::Experimental::ScatterView<DT, LY, ES, OP, CT, DP>& scatter_view, IS ... is)
1306 scatter_view.realloc(is ...);
1309 template <
typename DT,
typename LY,
typename ES,
int OP,
int CT,
int DP,
typename ... IS>
1311 resize(Kokkos::Experimental::ScatterView<DT, LY, ES, OP, CT, DP>& scatter_view, IS ... is)
1313 scatter_view.resize(is ...);
Memory layout tag indicating left-to-right (Fortran scheme) striding of multi-indices.
class to generate unique ids base on the required amount of concurrency
std::enable_if< std::is_same< typename Kokkos::View< T, P...>::array_layout, Kokkos::LayoutLeft >::value||std::is_same< typename Kokkos::View< T, P...>::array_layout, Kokkos::LayoutRight >::value >::type resize(Kokkos::View< T, P...> &v, const size_t n0=KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n1=KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n2=KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n3=KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n4=KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n5=KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n6=KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n7=KOKKOS_IMPL_CTOR_DEFAULT_ARG)
Resize a view with copying old data to new data at the corresponding indices.
View to an array of data.
Memory layout tag indicating right-to-left (C or lexigraphical scheme) striding of multi-indices...
void resize(DynRankView< T, P...> &v, const size_t n0=KOKKOS_INVALID_INDEX, const size_t n1=KOKKOS_INVALID_INDEX, const size_t n2=KOKKOS_INVALID_INDEX, const size_t n3=KOKKOS_INVALID_INDEX, const size_t n4=KOKKOS_INVALID_INDEX, const size_t n5=KOKKOS_INVALID_INDEX, const size_t n6=KOKKOS_INVALID_INDEX, const size_t n7=KOKKOS_INVALID_INDEX)
Resize a view with copying old data to new data at the corresponding indices.
Implementation of the ParallelFor operator that has a partial specialization for the device...
std::enable_if< std::is_same< typename Kokkos::View< T, P...>::array_layout, Kokkos::LayoutLeft >::value||std::is_same< typename Kokkos::View< T, P...>::array_layout, Kokkos::LayoutRight >::value >::type realloc(Kokkos::View< T, P...> &v, const size_t n0=KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n1=KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n2=KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n3=KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n4=KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n5=KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n6=KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n7=KOKKOS_IMPL_CTOR_DEFAULT_ARG)
Resize a view with discarding old data.
KOKKOS_INLINE_FUNCTION constexpr unsigned rank(const View< D, P...> &V)
Temporary free function rank() until rank() is implemented in the View.
void realloc(DynRankView< T, P...> &v, const size_t n0=KOKKOS_INVALID_INDEX, const size_t n1=KOKKOS_INVALID_INDEX, const size_t n2=KOKKOS_INVALID_INDEX, const size_t n3=KOKKOS_INVALID_INDEX, const size_t n4=KOKKOS_INVALID_INDEX, const size_t n5=KOKKOS_INVALID_INDEX, const size_t n6=KOKKOS_INVALID_INDEX, const size_t n7=KOKKOS_INVALID_INDEX)
Resize a view with copying old data to new data at the corresponding indices.