Kokkos Core Kernels Package  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Groups Pages
Kokkos_Parallel.hpp
Go to the documentation of this file.
1 /*
2 //@HEADER
3 // ************************************************************************
4 //
5 // Kokkos v. 3.0
6 // Copyright (2020) National Technology & Engineering
7 // Solutions of Sandia, LLC (NTESS).
8 //
9 // Under the terms of Contract DE-NA0003525 with NTESS,
10 // the U.S. Government retains certain rights in this software.
11 //
12 // Redistribution and use in source and binary forms, with or without
13 // modification, are permitted provided that the following conditions are
14 // met:
15 //
16 // 1. Redistributions of source code must retain the above copyright
17 // notice, this list of conditions and the following disclaimer.
18 //
19 // 2. Redistributions in binary form must reproduce the above copyright
20 // notice, this list of conditions and the following disclaimer in the
21 // documentation and/or other materials provided with the distribution.
22 //
23 // 3. Neither the name of the Corporation nor the names of the
24 // contributors may be used to endorse or promote products derived from
25 // this software without specific prior written permission.
26 //
27 // THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
28 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
31 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
32 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
33 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
34 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
35 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
36 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
37 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 //
39 // Questions? Contact Christian R. Trott (crtrott@sandia.gov)
40 //
41 // ************************************************************************
42 //@HEADER
43 */
44 
47 
48 #ifndef KOKKOS_PARALLEL_HPP
49 #define KOKKOS_PARALLEL_HPP
50 
51 #include <cstddef>
52 #include <Kokkos_Core_fwd.hpp>
53 #include <Kokkos_View.hpp>
54 #include <Kokkos_ExecPolicy.hpp>
55 
56 #if defined(KOKKOS_ENABLE_PROFILING)
57 #include <impl/Kokkos_Profiling_Interface.hpp>
58 #include <typeinfo>
59 #endif
60 
61 #include <impl/Kokkos_Tags.hpp>
62 #include <impl/Kokkos_Traits.hpp>
63 #include <impl/Kokkos_FunctorAnalysis.hpp>
64 #include <impl/Kokkos_FunctorAdapter.hpp>
65 
66 #ifdef KOKKOS_DEBUG
67 #include <iostream>
68 #endif
69 
70 //----------------------------------------------------------------------------
71 //----------------------------------------------------------------------------
72 
73 namespace Kokkos {
74 namespace Impl {
75 
76 //----------------------------------------------------------------------------
84 template <class Functor, class Policy, class EnableFunctor, class EnablePolicy>
85 struct FunctorPolicyExecutionSpace {
86  typedef Kokkos::DefaultExecutionSpace execution_space;
87 };
88 
89 template <class Functor, class Policy>
90 struct FunctorPolicyExecutionSpace<
91  Functor, Policy,
92  typename enable_if_type<typename Functor::device_type>::type,
93  typename enable_if_type<typename Policy ::execution_space>::type> {
94  typedef typename Policy ::execution_space execution_space;
95 };
96 
97 template <class Functor, class Policy>
98 struct FunctorPolicyExecutionSpace<
99  Functor, Policy,
100  typename enable_if_type<typename Functor::execution_space>::type,
101  typename enable_if_type<typename Policy ::execution_space>::type> {
102  typedef typename Policy ::execution_space execution_space;
103 };
104 
105 template <class Functor, class Policy, class EnableFunctor>
106 struct FunctorPolicyExecutionSpace<
107  Functor, Policy, EnableFunctor,
108  typename enable_if_type<typename Policy::execution_space>::type> {
109  typedef typename Policy ::execution_space execution_space;
110 };
111 
112 template <class Functor, class Policy, class EnablePolicy>
113 struct FunctorPolicyExecutionSpace<
114  Functor, Policy,
115  typename enable_if_type<typename Functor::device_type>::type,
116  EnablePolicy> {
117  typedef typename Functor::device_type::execution_space execution_space;
118 };
119 
120 template <class Functor, class Policy, class EnablePolicy>
121 struct FunctorPolicyExecutionSpace<
122  Functor, Policy,
123  typename enable_if_type<typename Functor::execution_space>::type,
124  EnablePolicy> {
125  typedef typename Functor::execution_space execution_space;
126 };
127 
128 } // namespace Impl
129 } // namespace Kokkos
130 
131 //----------------------------------------------------------------------------
132 //----------------------------------------------------------------------------
133 
134 namespace Kokkos {
135 
157 template <class ExecPolicy, class FunctorType>
158 inline void parallel_for(
159  const ExecPolicy& policy, const FunctorType& functor,
160  const std::string& str = "",
161  typename std::enable_if<
162  Kokkos::Impl::is_execution_policy<ExecPolicy>::value>::type* =
163  nullptr) {
164 #if defined(KOKKOS_ENABLE_PROFILING)
165  uint64_t kpID = 0;
166  if (Kokkos::Profiling::profileLibraryLoaded()) {
167  Kokkos::Impl::ParallelConstructName<FunctorType,
168  typename ExecPolicy::work_tag>
169  name(str);
170  Kokkos::Profiling::beginParallelFor(
171  name.get(), Kokkos::Profiling::Experimental::device_id(policy.space()),
172  &kpID);
173  }
174 #else
175  (void)str;
176 #endif
177 
178  Kokkos::Impl::shared_allocation_tracking_disable();
179  Impl::ParallelFor<FunctorType, ExecPolicy> closure(functor, policy);
180  Kokkos::Impl::shared_allocation_tracking_enable();
181 
182  closure.execute();
183 
184 #if defined(KOKKOS_ENABLE_PROFILING)
185  if (Kokkos::Profiling::profileLibraryLoaded()) {
186  Kokkos::Profiling::endParallelFor(kpID);
187  }
188 #endif
189 }
190 
191 template <class FunctorType>
192 inline void parallel_for(const size_t work_count, const FunctorType& functor,
193  const std::string& str = "") {
194  typedef typename Impl::FunctorPolicyExecutionSpace<
195  FunctorType, void>::execution_space execution_space;
196  typedef RangePolicy<execution_space> policy;
197 
198 #if defined(KOKKOS_ENABLE_PROFILING)
199  uint64_t kpID = 0;
200  if (Kokkos::Profiling::profileLibraryLoaded()) {
201  Kokkos::Impl::ParallelConstructName<FunctorType, void> name(str);
202  Kokkos::Profiling::beginParallelFor(
203  name.get(),
204  Kokkos::Profiling::Experimental::device_id(policy().space()), &kpID);
205  }
206 #else
207  (void)str;
208 #endif
209 
210  Kokkos::Impl::shared_allocation_tracking_disable();
211  Impl::ParallelFor<FunctorType, policy> closure(functor,
212  policy(0, work_count));
213  Kokkos::Impl::shared_allocation_tracking_enable();
214 
215  closure.execute();
216 
217 #if defined(KOKKOS_ENABLE_PROFILING)
218  if (Kokkos::Profiling::profileLibraryLoaded()) {
219  Kokkos::Profiling::endParallelFor(kpID);
220  }
221 #endif
222 }
223 
224 template <class ExecPolicy, class FunctorType>
225 inline void parallel_for(const std::string& str, const ExecPolicy& policy,
226  const FunctorType& functor) {
227 #if KOKKOS_ENABLE_DEBUG_PRINT_KERNEL_NAMES
228  Kokkos::fence();
229  std::cout << "KOKKOS_DEBUG Start parallel_for kernel: " << str << std::endl;
230 #endif
231 
232  ::Kokkos::parallel_for(policy, functor, str);
233 
234 #if KOKKOS_ENABLE_DEBUG_PRINT_KERNEL_NAMES
235  Kokkos::fence();
236  std::cout << "KOKKOS_DEBUG End parallel_for kernel: " << str << std::endl;
237 #endif
238  (void)str;
239 }
240 
241 } // namespace Kokkos
242 
243 #include <Kokkos_Parallel_Reduce.hpp>
244 //----------------------------------------------------------------------------
245 //----------------------------------------------------------------------------
246 
247 namespace Kokkos {
248 
394 // i/ }
411 template <class ExecutionPolicy, class FunctorType>
412 inline void parallel_scan(
413  const ExecutionPolicy& policy, const FunctorType& functor,
414  const std::string& str = "",
415  typename std::enable_if<
416  Kokkos::Impl::is_execution_policy<ExecutionPolicy>::value>::type* =
417  nullptr) {
418 #if defined(KOKKOS_ENABLE_PROFILING)
419  uint64_t kpID = 0;
420  if (Kokkos::Profiling::profileLibraryLoaded()) {
421  Kokkos::Impl::ParallelConstructName<FunctorType,
422  typename ExecutionPolicy::work_tag>
423  name(str);
424  Kokkos::Profiling::beginParallelScan(
425  name.get(), Kokkos::Profiling::Experimental::device_id(policy.space()),
426  &kpID);
427  }
428 #else
429  (void)str;
430 #endif
431 
432  Kokkos::Impl::shared_allocation_tracking_disable();
433  Impl::ParallelScan<FunctorType, ExecutionPolicy> closure(functor, policy);
434  Kokkos::Impl::shared_allocation_tracking_enable();
435 
436  closure.execute();
437 
438 #if defined(KOKKOS_ENABLE_PROFILING)
439  if (Kokkos::Profiling::profileLibraryLoaded()) {
440  Kokkos::Profiling::endParallelScan(kpID);
441  }
442 #endif
443 }
444 
445 template <class FunctorType>
446 inline void parallel_scan(const size_t work_count, const FunctorType& functor,
447  const std::string& str = "") {
448  typedef typename Kokkos::Impl::FunctorPolicyExecutionSpace<
449  FunctorType, void>::execution_space execution_space;
450 
451  typedef Kokkos::RangePolicy<execution_space> policy;
452 
453 #if defined(KOKKOS_ENABLE_PROFILING)
454  uint64_t kpID = 0;
455  if (Kokkos::Profiling::profileLibraryLoaded()) {
456  Kokkos::Impl::ParallelConstructName<FunctorType, void> name(str);
457  Kokkos::Profiling::beginParallelScan(
458  name.get(),
459  Kokkos::Profiling::Experimental::device_id(policy().space()), &kpID);
460  }
461 #else
462  (void)str;
463 #endif
464 
465  Kokkos::Impl::shared_allocation_tracking_disable();
466  Impl::ParallelScan<FunctorType, policy> closure(functor,
467  policy(0, work_count));
468  Kokkos::Impl::shared_allocation_tracking_enable();
469 
470  closure.execute();
471 
472 #if defined(KOKKOS_ENABLE_PROFILING)
473  if (Kokkos::Profiling::profileLibraryLoaded()) {
474  Kokkos::Profiling::endParallelScan(kpID);
475  }
476 #endif
477 }
478 
479 template <class ExecutionPolicy, class FunctorType>
480 inline void parallel_scan(const std::string& str, const ExecutionPolicy& policy,
481  const FunctorType& functor) {
482 #if KOKKOS_ENABLE_DEBUG_PRINT_KERNEL_NAMES
483  Kokkos::fence();
484  std::cout << "KOKKOS_DEBUG Start parallel_scan kernel: " << str << std::endl;
485 #endif
486 
487  ::Kokkos::parallel_scan(policy, functor, str);
488 
489 #if KOKKOS_ENABLE_DEBUG_PRINT_KERNEL_NAMES
490  Kokkos::fence();
491  std::cout << "KOKKOS_DEBUG End parallel_scan kernel: " << str << std::endl;
492 #endif
493  (void)str;
494 }
495 
496 template <class ExecutionPolicy, class FunctorType, class ReturnType>
497 inline void parallel_scan(
498  const ExecutionPolicy& policy, const FunctorType& functor,
499  ReturnType& return_value, const std::string& str = "",
500  typename std::enable_if<
501  Kokkos::Impl::is_execution_policy<ExecutionPolicy>::value>::type* =
502  nullptr) {
503 #if defined(KOKKOS_ENABLE_PROFILING)
504  uint64_t kpID = 0;
505  if (Kokkos::Profiling::profileLibraryLoaded()) {
506  Kokkos::Impl::ParallelConstructName<FunctorType,
507  typename ExecutionPolicy::work_tag>
508  name(str);
509  Kokkos::Profiling::beginParallelScan(
510  name.get(), Kokkos::Profiling::Experimental::device_id(policy.space()),
511  &kpID);
512  }
513 #else
514  (void)str;
515 #endif
516 
517  Kokkos::Impl::shared_allocation_tracking_disable();
518  Impl::ParallelScanWithTotal<FunctorType, ExecutionPolicy, ReturnType> closure(
519  functor, policy, return_value);
520  Kokkos::Impl::shared_allocation_tracking_enable();
521 
522  closure.execute();
523 
524 #if defined(KOKKOS_ENABLE_PROFILING)
525  if (Kokkos::Profiling::profileLibraryLoaded()) {
526  Kokkos::Profiling::endParallelScan(kpID);
527  }
528 #endif
529  policy.space().fence();
530 }
531 
532 template <class FunctorType, class ReturnType>
533 inline void parallel_scan(const size_t work_count, const FunctorType& functor,
534  ReturnType& return_value,
535  const std::string& str = "") {
536  typedef typename Kokkos::Impl::FunctorPolicyExecutionSpace<
537  FunctorType, void>::execution_space execution_space;
538 
539  typedef Kokkos::RangePolicy<execution_space> policy;
540 
541 #if defined(KOKKOS_ENABLE_PROFILING)
542  uint64_t kpID = 0;
543  if (Kokkos::Profiling::profileLibraryLoaded()) {
544  Kokkos::Impl::ParallelConstructName<FunctorType, void> name(str);
545  Kokkos::Profiling::beginParallelScan(
546  name.get(),
547  Kokkos::Profiling::Experimental::device_id(policy().space()), &kpID);
548  }
549 #else
550  (void)str;
551 #endif
552 
553  Kokkos::Impl::shared_allocation_tracking_disable();
554  Impl::ParallelScanWithTotal<FunctorType, policy, ReturnType> closure(
555  functor, policy(0, work_count), return_value);
556  Kokkos::Impl::shared_allocation_tracking_enable();
557 
558  closure.execute();
559 
560 #if defined(KOKKOS_ENABLE_PROFILING)
561  if (Kokkos::Profiling::profileLibraryLoaded()) {
562  Kokkos::Profiling::endParallelScan(kpID);
563  }
564 #endif
565  execution_space().fence();
566 }
567 
568 template <class ExecutionPolicy, class FunctorType, class ReturnType>
569 inline void parallel_scan(const std::string& str, const ExecutionPolicy& policy,
570  const FunctorType& functor,
571  ReturnType& return_value) {
572 #if KOKKOS_ENABLE_DEBUG_PRINT_KERNEL_NAMES
573  Kokkos::fence();
574  std::cout << "KOKKOS_DEBUG Start parallel_scan kernel: " << str << std::endl;
575 #endif
576 
577  ::Kokkos::parallel_scan(policy, functor, return_value, str);
578 
579 #if KOKKOS_ENABLE_DEBUG_PRINT_KERNEL_NAMES
580  Kokkos::fence();
581  std::cout << "KOKKOS_DEBUG End parallel_scan kernel: " << str << std::endl;
582 #endif
583  (void)str;
584 }
585 
586 } // namespace Kokkos
587 
588 //----------------------------------------------------------------------------
589 //----------------------------------------------------------------------------
590 
591 namespace Kokkos {
592 namespace Impl {
593 
594 template <class FunctorType,
595  bool HasTeamShmemSize =
596  has_member_team_shmem_size<FunctorType>::value,
597  bool HasShmemSize = has_member_shmem_size<FunctorType>::value>
598 struct FunctorTeamShmemSize {
599  KOKKOS_INLINE_FUNCTION static size_t value(const FunctorType&, int) {
600  return 0;
601  }
602 };
603 
604 template <class FunctorType>
605 struct FunctorTeamShmemSize<FunctorType, true, false> {
606  static inline size_t value(const FunctorType& f, int team_size) {
607  return f.team_shmem_size(team_size);
608  }
609 };
610 
611 template <class FunctorType>
612 struct FunctorTeamShmemSize<FunctorType, false, true> {
613  static inline size_t value(const FunctorType& f, int team_size) {
614  return f.shmem_size(team_size);
615  }
616 };
617 template <class FunctorType>
618 struct FunctorTeamShmemSize<FunctorType, true, true> {
619  static inline size_t value(const FunctorType& /*f*/, int /*team_size*/) {
620  Kokkos::abort(
621  "Functor with both team_shmem_size and shmem_size defined is "
622  "not allowed");
623  return 0;
624  }
625 };
626 
627 } // namespace Impl
628 } // namespace Kokkos
629 
630 //----------------------------------------------------------------------------
631 //----------------------------------------------------------------------------
632 
633 #endif /* KOKKOS_PARALLEL_HPP */
Kokkos::Impl::ParallelFor
Implementation of the ParallelFor operator that has a partial specialization for the device...
Definition: Kokkos_Core_fwd.hpp:294
ReturnType
ReturnType
Kokkos::Impl::FunctorPolicyExecutionSpace
Given a Functor and Execution Policy query an execution space.
Definition: Kokkos_Core_fwd.hpp:282
Kokkos::parallel_for
void parallel_for(const ExecPolicy &policy, const FunctorType &functor, const std::string &str="", typename std::enable_if< Kokkos::Impl::is_execution_policy< ExecPolicy >::value >::type *=nullptr)
Execute functor in parallel according to the execution policy.
Definition: Kokkos_Parallel.hpp:158
Kokkos::RangePolicy
Execution policy for work over a range of an integral type.
Definition: Kokkos_ExecPolicy.hpp:94
Generated by   doxygen 1.8.5