Kokkos_ExecPolicy.hpp

/*
//@HEADER
// ************************************************************************
//
//                        Kokkos v. 3.0
//       Copyright (2020) National Technology & Engineering
//               Solutions of Sandia, LLC (NTESS).
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Christian R. Trott (crtrott@sandia.gov)
//
// ************************************************************************
//@HEADER
*/

#ifndef KOKKOS_EXECPOLICY_HPP
#define KOKKOS_EXECPOLICY_HPP

#include <Kokkos_Core_fwd.hpp>
#include <impl/Kokkos_Traits.hpp>
#include <impl/Kokkos_Error.hpp>
#include <impl/Kokkos_Tags.hpp>
#include <impl/Kokkos_AnalyzePolicy.hpp>
#include <Kokkos_Concepts.hpp>
#include <iostream>
#if defined(KOKKOS_ENABLE_PROFILING)
#include <typeinfo>
#endif  // KOKKOS_ENABLE_PROFILING

//----------------------------------------------------------------------------

namespace Kokkos {

struct ParallelForTag {};
struct ParallelScanTag {};
struct ParallelReduceTag {};

struct ChunkSize {
  int value;
  ChunkSize(int value_) : value(value_) {}
};

template <class... Properties>
class RangePolicy : public Impl::PolicyTraits<Properties...> {
 public:
  typedef Impl::PolicyTraits<Properties...> traits;

 private:
  typename traits::execution_space m_space;
  typename traits::index_type m_begin;
  typename traits::index_type m_end;
  typename traits::index_type m_granularity;
  typename traits::index_type m_granularity_mask;

  template <class... OtherProperties>
  friend class RangePolicy;

 public:
  typedef RangePolicy execution_policy;
  typedef typename traits::index_type member_type;
  typedef typename traits::index_type index_type;

  KOKKOS_INLINE_FUNCTION const typename traits::execution_space& space() const {
    return m_space;
  }
  KOKKOS_INLINE_FUNCTION member_type begin() const { return m_begin; }
  KOKKOS_INLINE_FUNCTION member_type end() const { return m_end; }

  // TODO: find a better workaround for Clangs weird instantiation order
  // This thing is here because of an instantiation error, where the RangePolicy
  // is inserted into FunctorValue Traits, which tries decltype on the operator.
  // It tries to do this even though the first argument of parallel for clearly
  // doesn't match.
  void operator()(const int&) const {}

  RangePolicy(const RangePolicy&) = default;
  RangePolicy(RangePolicy&&)      = default;

  template <class... OtherProperties>
  RangePolicy(const RangePolicy<OtherProperties...> p) {
    m_space            = p.m_space;
    m_begin            = p.m_begin;
    m_end              = p.m_end;
    m_granularity      = p.m_granularity;
    m_granularity_mask = p.m_granularity_mask;
  }

  inline RangePolicy() : m_space(), m_begin(0), m_end(0) {}

  inline RangePolicy(const typename traits::execution_space& work_space,
                     const member_type work_begin, const member_type work_end)
      : m_space(work_space),
        m_begin(work_begin < work_end ? work_begin : 0),
        m_end(work_begin < work_end ? work_end : 0),
        m_granularity(0),
        m_granularity_mask(0) {
    set_auto_chunk_size();
  }

  inline RangePolicy(const member_type work_begin, const member_type work_end)
      : RangePolicy(typename traits::execution_space(), work_begin, work_end) {
    set_auto_chunk_size();
  }

  template <class... Args>
  inline RangePolicy(const typename traits::execution_space& work_space,
                     const member_type work_begin, const member_type work_end,
                     Args... args)
      : m_space(work_space),
        m_begin(work_begin < work_end ? work_begin : 0),
        m_end(work_begin < work_end ? work_end : 0),
        m_granularity(0),
        m_granularity_mask(0) {
    set_auto_chunk_size();
    set(args...);
  }

  template <class... Args>
  inline RangePolicy(const member_type work_begin, const member_type work_end,
                     Args... args)
      : RangePolicy(typename traits::execution_space(), work_begin, work_end) {
    set_auto_chunk_size();
    set(args...);
  }

 private:
  inline void set() {}

 public:
  template <class... Args>
  inline void set(Args...) {
    static_assert(
        0 == sizeof...(Args),
        "Kokkos::RangePolicy: unhandled constructor arguments encountered.");
  }

  template <class... Args>
  inline void set(const ChunkSize& chunksize, Args... args) {
    m_granularity      = chunksize.value;
    m_granularity_mask = m_granularity - 1;
    set(args...);
  }

 public:
  inline member_type chunk_size() const { return m_granularity; }

  inline RangePolicy set_chunk_size(int chunk_size_) const {
    RangePolicy p        = *this;
    p.m_granularity      = chunk_size_;
    p.m_granularity_mask = p.m_granularity - 1;
    return p;
  }

 private:
  inline void set_auto_chunk_size() {
    int64_t concurrency =
        static_cast<int64_t>(traits::execution_space::concurrency());
    if (concurrency == 0) concurrency = 1;

    if (m_granularity > 0) {
      if (!Impl::is_integral_power_of_two(m_granularity))
        Kokkos::abort("RangePolicy blocking granularity must be power of two");
    }

    int64_t new_chunk_size = 1;
    while (new_chunk_size * 100 * concurrency <
           static_cast<int64_t>(m_end - m_begin))
      new_chunk_size *= 2;
    if (new_chunk_size < 128) {
      new_chunk_size = 1;
      while ((new_chunk_size * 40 * concurrency <
              static_cast<int64_t>(m_end - m_begin)) &&
             (new_chunk_size < 128))
        new_chunk_size *= 2;
    }
    m_granularity      = new_chunk_size;
    m_granularity_mask = m_granularity - 1;
  }

 public:
  struct WorkRange {
    typedef typename RangePolicy::work_tag work_tag;
    typedef typename RangePolicy::member_type member_type;

    KOKKOS_INLINE_FUNCTION member_type begin() const { return m_begin; }
    KOKKOS_INLINE_FUNCTION member_type end() const { return m_end; }

    KOKKOS_INLINE_FUNCTION
    WorkRange(const RangePolicy& range, const int part_rank,
              const int part_size)
        : m_begin(0), m_end(0) {
      if (part_size) {
        // Split evenly among partitions, then round up to the granularity.
        const member_type work_part =
            ((((range.end() - range.begin()) + (part_size - 1)) / part_size) +
             range.m_granularity_mask) &
            ~member_type(range.m_granularity_mask);

        m_begin = range.begin() + work_part * part_rank;
        m_end   = m_begin + work_part;

        if (range.end() < m_begin) m_begin = range.end();
        if (range.end() < m_end) m_end = range.end();
      }
    }

   private:
    member_type m_begin;
    member_type m_end;
    WorkRange();
    WorkRange& operator=(const WorkRange&);
  };
};

}  // namespace Kokkos

//----------------------------------------------------------------------------
//----------------------------------------------------------------------------

namespace Kokkos {

namespace Impl {

template <class ExecSpace, class... Properties>
class TeamPolicyInternal : public Impl::PolicyTraits<Properties...> {
 private:
  typedef Impl::PolicyTraits<Properties...> traits;

 public:
  typedef typename traits::index_type index_type;

  //----------------------------------------
  template <class FunctorType>
  static int team_size_max(const FunctorType&);

  template <class FunctorType>
  static int team_size_recommended(const FunctorType&);

  template <class FunctorType>
  static int team_size_recommended(const FunctorType&, const int&);

  template <class FunctorType>
  int team_size_recommended(const FunctorType& functor,
                            const int vector_length);

  //----------------------------------------
  TeamPolicyInternal(const typename traits::execution_space&,
                     int league_size_request, int team_size_request,
                     int vector_length_request = 1);

  TeamPolicyInternal(const typename traits::execution_space&,
                     int league_size_request, const Kokkos::AUTO_t&,
                     int vector_length_request = 1);

  TeamPolicyInternal(int league_size_request, int team_size_request,
                     int vector_length_request = 1);

  TeamPolicyInternal(int league_size_request, const Kokkos::AUTO_t&,
                     int vector_length_request = 1);

  /*  TeamPolicyInternal( int league_size_request , int team_size_request );

    TeamPolicyInternal( int league_size_request , const Kokkos::AUTO_t & );*/

  KOKKOS_INLINE_FUNCTION int league_size() const;

  KOKKOS_INLINE_FUNCTION int team_size() const;

  inline typename traits::index_type chunk_size() const;

#ifdef KOKKOS_ENABLE_DEPRECATED_CODE
  inline TeamPolicyInternal set_chunk_size(int chunk_size) const;
#else
  inline TeamPolicyInternal& set_chunk_size(int chunk_size);
#endif

  struct member_type {
    KOKKOS_INLINE_FUNCTION
    typename traits::execution_space::scratch_memory_space team_shmem() const;

    KOKKOS_INLINE_FUNCTION int league_rank() const;

    KOKKOS_INLINE_FUNCTION int league_size() const;

    KOKKOS_INLINE_FUNCTION int team_rank() const;

    KOKKOS_INLINE_FUNCTION int team_size() const;

    KOKKOS_INLINE_FUNCTION void team_barrier() const;

    template <class JoinOp>
    KOKKOS_INLINE_FUNCTION typename JoinOp::value_type team_reduce(
        const typename JoinOp::value_type, const JoinOp&) const;

    template <typename Type>
    KOKKOS_INLINE_FUNCTION Type team_scan(const Type& value) const;

    template <typename Type>
    KOKKOS_INLINE_FUNCTION Type team_scan(const Type& value,
                                          Type* const global_accum) const;
  };
};

struct PerTeamValue {
  int value;
  PerTeamValue(int arg);
};

struct PerThreadValue {
  int value;
  PerThreadValue(int arg);
};

template <class iType, class... Args>
struct ExtractVectorLength {
  static inline iType value(
      typename std::enable_if<std::is_integral<iType>::value, iType>::type val,
      Args...) {
    return val;
  }
  static inline
      typename std::enable_if<!std::is_integral<iType>::value, int>::type
      value(
          typename std::enable_if<!std::is_integral<iType>::value, iType>::type,
          Args...) {
    return 1;
  }
};

template <class iType, class... Args>
inline typename std::enable_if<std::is_integral<iType>::value, iType>::type
extract_vector_length(iType val, Args...) {
  return val;
}

template <class iType, class... Args>
inline typename std::enable_if<!std::is_integral<iType>::value, int>::type
extract_vector_length(iType, Args...) {
  return 1;
}

}  // namespace Impl

Impl::PerTeamValue PerTeam(const int& arg);
Impl::PerThreadValue PerThread(const int& arg);

struct ScratchRequest {
  int level;

  int per_team;
  int per_thread;

  inline ScratchRequest(const int& level_,
                        const Impl::PerTeamValue& team_value) {
    level      = level_;
    per_team   = team_value.value;
    per_thread = 0;
  }

  inline ScratchRequest(const int& level_,
                        const Impl::PerThreadValue& thread_value) {
    level      = level_;
    per_team   = 0;
    per_thread = thread_value.value;
  }

  inline ScratchRequest(const int& level_, const Impl::PerTeamValue& team_value,
                        const Impl::PerThreadValue& thread_value) {
    level      = level_;
    per_team   = team_value.value;
    per_thread = thread_value.value;
  }

  inline ScratchRequest(const int& level_,
                        const Impl::PerThreadValue& thread_value,
                        const Impl::PerTeamValue& team_value) {
    level      = level_;
    per_team   = team_value.value;
    per_thread = thread_value.value;
  }
};

template <class... Properties>
class TeamPolicy
    : public Impl::TeamPolicyInternal<
          typename Impl::PolicyTraits<Properties...>::execution_space,
          Properties...> {
  typedef Impl::TeamPolicyInternal<
      typename Impl::PolicyTraits<Properties...>::execution_space,
      Properties...>
      internal_policy;

  template <class... OtherProperties>
  friend class TeamPolicy;

 public:
  typedef Impl::PolicyTraits<Properties...> traits;

  typedef TeamPolicy execution_policy;

  TeamPolicy& operator=(const TeamPolicy&) = default;

  TeamPolicy(const typename traits::execution_space& space_,
             int league_size_request, int team_size_request,
             int vector_length_request = 1)
      : internal_policy(space_, league_size_request, team_size_request,
                        vector_length_request) {
    first_arg = false;
  }

  TeamPolicy(const typename traits::execution_space& space_,
             int league_size_request, const Kokkos::AUTO_t&,
             int vector_length_request = 1)
      : internal_policy(space_, league_size_request, Kokkos::AUTO(),
                        vector_length_request) {
    first_arg = false;
  }

  TeamPolicy(int league_size_request, int team_size_request,
             int vector_length_request = 1)
      : internal_policy(league_size_request, team_size_request,
                        vector_length_request) {
    first_arg = false;
  }

  TeamPolicy(int league_size_request, const Kokkos::AUTO_t&,
             int vector_length_request = 1)
      : internal_policy(league_size_request, Kokkos::AUTO(),
                        vector_length_request) {
    first_arg = false;
  }

#ifdef KOKKOS_ENABLE_DEPRECATED_CODE

  template <class... Args>
  TeamPolicy(const typename traits::execution_space&, int league_size_request,
             int team_size_request, int vector_length_request, Args... args)
      : internal_policy(typename traits::execution_space(), league_size_request,
                        team_size_request, vector_length_request) {
    first_arg = false;
    set(args...);
  }

  template <class... Args>
  TeamPolicy(const typename traits::execution_space&, int league_size_request,
             const Kokkos::AUTO_t&, int vector_length_request, Args... args)
      : internal_policy(typename traits::execution_space(), league_size_request,
                        Kokkos::AUTO(), vector_length_request) {
    first_arg = false;
    set(args...);
  }

  template <class... Args>
  TeamPolicy(int league_size_request, int team_size_request,
             int vector_length_request, Args... args)
      : internal_policy(league_size_request, team_size_request,
                        vector_length_request) {
    first_arg = false;
    set(args...);
  }

  template <class... Args>
  TeamPolicy(int league_size_request, const Kokkos::AUTO_t&,
             int vector_length_request, Args... args)
      : internal_policy(league_size_request, Kokkos::AUTO(),
                        vector_length_request) {
    first_arg = false;
    set(args...);
  }

  template <class... Args>
  TeamPolicy(const typename traits::execution_space&, int league_size_request,
             int team_size_request, Args... args)
      : internal_policy(typename traits::execution_space(), league_size_request,
                        team_size_request,
                        Kokkos::Impl::extract_vector_length<Args...>(args...)) {
    first_arg = true;
    set(args...);
  }

  template <class... Args>
  TeamPolicy(const typename traits::execution_space&, int league_size_request,
             const Kokkos::AUTO_t&, Args... args)
      : internal_policy(typename traits::execution_space(), league_size_request,
                        Kokkos::AUTO(),
                        Kokkos::Impl::extract_vector_length<Args...>(args...)) {
    first_arg = true;
    set(args...);
  }

  template <class... Args>
  TeamPolicy(int league_size_request, int team_size_request, Args... args)
      : internal_policy(league_size_request, team_size_request,
                        Kokkos::Impl::extract_vector_length<Args...>(args...)) {
    first_arg = true;
    set(args...);
  }

  template <class... Args>
  TeamPolicy(int league_size_request, const Kokkos::AUTO_t&, Args... args)
      : internal_policy(league_size_request, Kokkos::AUTO(),
                        Kokkos::Impl::extract_vector_length<Args...>(args...)) {
    first_arg = true;
    set(args...);
  }
#endif

  template <class... OtherProperties>
  TeamPolicy(const TeamPolicy<OtherProperties...> p) : internal_policy(p) {
    first_arg = p.first_arg;
  }

 private:
  bool first_arg;
  TeamPolicy(const internal_policy& p) : internal_policy(p) {
    first_arg = false;
  }

#ifdef KOKKOS_ENABLE_DEPRECATED_CODE
  inline void set() {}
#endif

 public:
#ifdef KOKKOS_ENABLE_DEPRECATED_CODE
  template <class... Args>
  inline void set(Args...) {
    static_assert(
        0 == sizeof...(Args),
        "Kokkos::TeamPolicy: unhandled constructor arguments encountered.");
  }

  template <class iType, class... Args>
  inline typename std::enable_if<std::is_integral<iType>::value>::type set(
      iType, Args... args) {
    if (first_arg) {
      first_arg = false;
      set(args...);
    } else {
      first_arg = false;
      Kokkos::Impl::throw_runtime_exception(
          "Kokkos::TeamPolicy: integer argument to constructor in illegal "
          "place.");
    }
  }

  template <class... Args>
  inline void set(const ChunkSize& chunksize, Args... args) {
    first_arg = false;
    internal_policy::internal_set_chunk_size(chunksize.value);
    set(args...);
  }

  template <class... Args>
  inline void set(const ScratchRequest& scr_request, Args... args) {
    first_arg = false;
    internal_policy::internal_set_scratch_size(
        scr_request.level, Impl::PerTeamValue(scr_request.per_team),
        Impl::PerThreadValue(scr_request.per_thread));
    set(args...);
  }

  inline TeamPolicy set_chunk_size(int chunk) const {
    return TeamPolicy(internal_policy::set_chunk_size(chunk));
  }

  inline TeamPolicy set_scratch_size(const int& level,
                                     const Impl::PerTeamValue& per_team) const {
    return TeamPolicy(internal_policy::set_scratch_size(level, per_team));
  }
  inline TeamPolicy set_scratch_size(
      const int& level, const Impl::PerThreadValue& per_thread) const {
    return TeamPolicy(internal_policy::set_scratch_size(level, per_thread));
  }
  inline TeamPolicy set_scratch_size(
      const int& level, const Impl::PerTeamValue& per_team,
      const Impl::PerThreadValue& per_thread) const {
    return TeamPolicy(
        internal_policy::set_scratch_size(level, per_team, per_thread));
  }
  inline TeamPolicy set_scratch_size(const int& level,
                                     const Impl::PerThreadValue& per_thread,
                                     const Impl::PerTeamValue& per_team) const {
    return TeamPolicy(
        internal_policy::set_scratch_size(level, per_team, per_thread));
  }

#else
  inline TeamPolicy& set_chunk_size(int chunk) {
    static_assert(std::is_same<decltype(internal_policy::set_chunk_size(chunk)),
                               internal_policy&>::value,
                  "internal set_chunk_size should return a reference");
    return static_cast<TeamPolicy&>(internal_policy::set_chunk_size(chunk));
  }

  inline TeamPolicy& set_scratch_size(const int& level,
                                      const Impl::PerTeamValue& per_team) {
    static_assert(std::is_same<decltype(internal_policy::set_scratch_size(
                                   level, per_team)),
                               internal_policy&>::value,
                  "internal set_chunk_size should return a reference");
    return static_cast<TeamPolicy&>(
        internal_policy::set_scratch_size(level, per_team));
  }
  inline TeamPolicy& set_scratch_size(const int& level,
                                      const Impl::PerThreadValue& per_thread) {
    return static_cast<TeamPolicy&>(
        internal_policy::set_scratch_size(level, per_thread));
  }
  inline TeamPolicy& set_scratch_size(const int& level,
                                      const Impl::PerTeamValue& per_team,
                                      const Impl::PerThreadValue& per_thread) {
    return static_cast<TeamPolicy&>(
        internal_policy::set_scratch_size(level, per_team, per_thread));
  }
  inline TeamPolicy& set_scratch_size(const int& level,
                                      const Impl::PerThreadValue& per_thread,
                                      const Impl::PerTeamValue& per_team) {
    return static_cast<TeamPolicy&>(
        internal_policy::set_scratch_size(level, per_team, per_thread));
  }
#endif
};

namespace Impl {

template <typename iType, class TeamMemberType>
struct TeamThreadRangeBoundariesStruct {
 private:
  KOKKOS_INLINE_FUNCTION static iType ibegin(const iType& arg_begin,
                                             const iType& arg_end,
                                             const iType& arg_rank,
                                             const iType& arg_size) {
    return arg_begin +
           ((arg_end - arg_begin + arg_size - 1) / arg_size) * arg_rank;
  }

  KOKKOS_INLINE_FUNCTION static iType iend(const iType& arg_begin,
                                           const iType& arg_end,
                                           const iType& arg_rank,
                                           const iType& arg_size) {
    const iType end_ =
        arg_begin +
        ((arg_end - arg_begin + arg_size - 1) / arg_size) * (arg_rank + 1);
    return end_ < arg_end ? end_ : arg_end;
  }

 public:
  typedef iType index_type;
  const iType start;
  const iType end;
  enum { increment = 1 };
  const TeamMemberType& thread;

  KOKKOS_INLINE_FUNCTION
  TeamThreadRangeBoundariesStruct(const TeamMemberType& arg_thread,
                                  const iType& arg_end)
      : start(
            ibegin(0, arg_end, arg_thread.team_rank(), arg_thread.team_size())),
        end(iend(0, arg_end, arg_thread.team_rank(), arg_thread.team_size())),
        thread(arg_thread) {}

  KOKKOS_INLINE_FUNCTION
  TeamThreadRangeBoundariesStruct(const TeamMemberType& arg_thread,
                                  const iType& arg_begin, const iType& arg_end)
      : start(ibegin(arg_begin, arg_end, arg_thread.team_rank(),
                     arg_thread.team_size())),
        end(iend(arg_begin, arg_end, arg_thread.team_rank(),
                 arg_thread.team_size())),
        thread(arg_thread) {}
};

template <typename iType, class TeamMemberType>
struct TeamVectorRangeBoundariesStruct {
 private:
  KOKKOS_INLINE_FUNCTION static iType ibegin(const iType& arg_begin,
                                             const iType& arg_end,
                                             const iType& arg_rank,
                                             const iType& arg_size) {
    return arg_begin +
           ((arg_end - arg_begin + arg_size - 1) / arg_size) * arg_rank;
  }

  KOKKOS_INLINE_FUNCTION static iType iend(const iType& arg_begin,
                                           const iType& arg_end,
                                           const iType& arg_rank,
                                           const iType& arg_size) {
    const iType end_ =
        arg_begin +
        ((arg_end - arg_begin + arg_size - 1) / arg_size) * (arg_rank + 1);
    return end_ < arg_end ? end_ : arg_end;
  }

 public:
  typedef iType index_type;
  const iType start;
  const iType end;
  enum { increment = 1 };
  const TeamMemberType& thread;

  KOKKOS_INLINE_FUNCTION
  TeamVectorRangeBoundariesStruct(const TeamMemberType& arg_thread,
                                  const iType& arg_end)
      : start(
            ibegin(0, arg_end, arg_thread.team_rank(), arg_thread.team_size())),
        end(iend(0, arg_end, arg_thread.team_rank(), arg_thread.team_size())),
        thread(arg_thread) {}

  KOKKOS_INLINE_FUNCTION
  TeamVectorRangeBoundariesStruct(const TeamMemberType& arg_thread,
                                  const iType& arg_begin, const iType& arg_end)
      : start(ibegin(arg_begin, arg_end, arg_thread.team_rank(),
                     arg_thread.team_size())),
        end(iend(arg_begin, arg_end, arg_thread.team_rank(),
                 arg_thread.team_size())),
        thread(arg_thread) {}
};

template <typename iType, class TeamMemberType>
struct ThreadVectorRangeBoundariesStruct {
  typedef iType index_type;
  const index_type start;
  const index_type end;
  enum { increment = 1 };

  KOKKOS_INLINE_FUNCTION
  constexpr ThreadVectorRangeBoundariesStruct(const TeamMemberType,
                                              const index_type& count) noexcept
      : start(static_cast<index_type>(0)), end(count) {}

  KOKKOS_INLINE_FUNCTION
  constexpr ThreadVectorRangeBoundariesStruct(const index_type& count) noexcept
      : start(static_cast<index_type>(0)), end(count) {}

  KOKKOS_INLINE_FUNCTION
  constexpr ThreadVectorRangeBoundariesStruct(
      const TeamMemberType, const index_type& arg_begin,
      const index_type& arg_end) noexcept
      : start(static_cast<index_type>(arg_begin)), end(arg_end) {}

  KOKKOS_INLINE_FUNCTION
  constexpr ThreadVectorRangeBoundariesStruct(
      const index_type& arg_begin, const index_type& arg_end) noexcept
      : start(static_cast<index_type>(arg_begin)), end(arg_end) {}
};

template <class TeamMemberType>
struct ThreadSingleStruct {
  const TeamMemberType& team_member;
  KOKKOS_INLINE_FUNCTION
  ThreadSingleStruct(const TeamMemberType& team_member_)
      : team_member(team_member_) {}
};

template <class TeamMemberType>
struct VectorSingleStruct {
  const TeamMemberType& team_member;
  KOKKOS_INLINE_FUNCTION
  VectorSingleStruct(const TeamMemberType& team_member_)
      : team_member(team_member_) {}
};

}  // namespace Impl

template <typename iType, class TeamMemberType, class _never_use_this_overload>
KOKKOS_INLINE_FUNCTION_DELETED
    Impl::TeamThreadRangeBoundariesStruct<iType, TeamMemberType>
    TeamThreadRange(const TeamMemberType&, const iType& count) = delete;

template <typename iType1, typename iType2, class TeamMemberType,
          class _never_use_this_overload>
KOKKOS_INLINE_FUNCTION_DELETED Impl::TeamThreadRangeBoundariesStruct<
    typename std::common_type<iType1, iType2>::type, TeamMemberType>
TeamThreadRange(const TeamMemberType&, const iType1& begin,
                const iType2& end) = delete;

template <typename iType, class TeamMemberType, class _never_use_this_overload>
KOKKOS_INLINE_FUNCTION_DELETED
    Impl::TeamThreadRangeBoundariesStruct<iType, TeamMemberType>
    TeamVectorRange(const TeamMemberType&, const iType& count) = delete;

template <typename iType1, typename iType2, class TeamMemberType,
          class _never_use_this_overload>
KOKKOS_INLINE_FUNCTION_DELETED Impl::TeamThreadRangeBoundariesStruct<
    typename std::common_type<iType1, iType2>::type, TeamMemberType>
TeamVectorRange(const TeamMemberType&, const iType1& begin,
                const iType2& end) = delete;

template <typename iType, class TeamMemberType, class _never_use_this_overload>
KOKKOS_INLINE_FUNCTION_DELETED
    Impl::ThreadVectorRangeBoundariesStruct<iType, TeamMemberType>
    ThreadVectorRange(const TeamMemberType&, const iType& count) = delete;

template <typename iType, class TeamMemberType, class _never_use_this_overload>
KOKKOS_INLINE_FUNCTION_DELETED
    Impl::ThreadVectorRangeBoundariesStruct<iType, TeamMemberType>
    ThreadVectorRange(const TeamMemberType&, const iType& arg_begin,
                      const iType& arg_end) = delete;

#if defined(KOKKOS_ENABLE_PROFILING)
namespace Impl {

template <typename FunctorType, typename TagType,
          bool HasTag = !std::is_same<TagType, void>::value>
struct ParallelConstructName;

template <typename FunctorType, typename TagType>
struct ParallelConstructName<FunctorType, TagType, true> {
  ParallelConstructName(std::string const& label) : label_ref(label) {
    if (label.empty()) {
      default_name = std::string(typeid(FunctorType).name()) + "/" +
                     typeid(TagType).name();
    }
  }
  std::string const& get() {
    return (label_ref.empty()) ? default_name : label_ref;
  }
  std::string const& label_ref;
  std::string default_name;
};

template <typename FunctorType, typename TagType>
struct ParallelConstructName<FunctorType, TagType, false> {
  ParallelConstructName(std::string const& label) : label_ref(label) {
    if (label.empty()) {
      default_name = std::string(typeid(FunctorType).name());
    }
  }
  std::string const& get() {
    return (label_ref.empty()) ? default_name : label_ref;
  }
  std::string const& label_ref;
  std::string default_name;
};

}  // namespace Impl
#endif /* defined KOKKOS_ENABLE_PROFILING */

}  // namespace Kokkos

namespace Kokkos {
namespace Experimental {

namespace Impl {
template <class Property, class Policy>
struct PolicyPropertyAdaptor;

template <unsigned long P, class... Properties>
struct PolicyPropertyAdaptor<WorkItemProperty::ImplWorkItemProperty<P>,
                             RangePolicy<Properties...>> {
  typedef RangePolicy<Properties...> policy_in_t;
  typedef RangePolicy<typename policy_in_t::traits::execution_space,
                      typename policy_in_t::traits::schedule_type,
                      typename policy_in_t::traits::work_tag,
                      typename policy_in_t::traits::index_type,
                      typename policy_in_t::traits::iteration_pattern,
                      typename policy_in_t::traits::launch_bounds,
                      WorkItemProperty::ImplWorkItemProperty<P>>
      policy_out_t;
};

template <unsigned long P, class... Properties>
struct PolicyPropertyAdaptor<WorkItemProperty::ImplWorkItemProperty<P>,
                             TeamPolicy<Properties...>> {
  typedef TeamPolicy<Properties...> policy_in_t;
  typedef TeamPolicy<typename policy_in_t::traits::execution_space,
                     typename policy_in_t::traits::schedule_type,
                     typename policy_in_t::traits::work_tag,
                     typename policy_in_t::traits::index_type,
                     typename policy_in_t::traits::iteration_pattern,
                     typename policy_in_t::traits::launch_bounds,
                     WorkItemProperty::ImplWorkItemProperty<P>>
      policy_out_t;
};
}  // namespace Impl

template <class PolicyType, unsigned long P>
constexpr typename Impl::PolicyPropertyAdaptor<
    WorkItemProperty::ImplWorkItemProperty<P>, PolicyType>::policy_out_t
require(const PolicyType p, WorkItemProperty::ImplWorkItemProperty<P>) {
  return typename Impl::PolicyPropertyAdaptor<
      WorkItemProperty::ImplWorkItemProperty<P>, PolicyType>::policy_out_t(p);
}
}  // namespace Experimental
}  // namespace Kokkos
#endif /* #define KOKKOS_EXECPOLICY_HPP */