Kokkos Core Kernels Package  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Groups Pages
Kokkos_TaskScheduler.hpp
1 //@HEADER
2 // ************************************************************************
3 //
4 // Kokkos v. 4.0
5 // Copyright (2022) National Technology & Engineering
6 // Solutions of Sandia, LLC (NTESS).
7 //
8 // Under the terms of Contract DE-NA0003525 with NTESS,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12 // See https://kokkos.org/LICENSE for license information.
13 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14 //
15 //@HEADER
16 
17 #ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18 #include <Kokkos_Macros.hpp>
19 static_assert(false,
20  "Including non-public Kokkos header files is not allowed.");
21 #endif
22 #ifndef KOKKOS_TASKSCHEDULER_HPP
23 #define KOKKOS_TASKSCHEDULER_HPP
24 
25 //----------------------------------------------------------------------------
26 
27 #include <Kokkos_Macros.hpp>
28 #if defined(KOKKOS_ENABLE_TASKDAG)
29 
30 #include <Kokkos_Core_fwd.hpp>
31 #include <Kokkos_TaskScheduler_fwd.hpp>
32 //----------------------------------------------------------------------------
33 
34 #include <Kokkos_MemoryPool.hpp>
35 
36 #include <Kokkos_Future.hpp>
37 #include <impl/Kokkos_TaskQueue.hpp>
38 #include <impl/Kokkos_SingleTaskQueue.hpp>
39 #include <impl/Kokkos_TaskQueueMultiple.hpp>
40 #include <impl/Kokkos_TaskPolicyData.hpp>
41 #include <impl/Kokkos_TaskTeamMember.hpp>
42 #include <impl/Kokkos_SimpleTaskScheduler.hpp>
43 
44 //----------------------------------------------------------------------------
45 //----------------------------------------------------------------------------
46 
47 namespace Kokkos {
48 
49 namespace Impl {
50 
51 template <class, class>
52 class TaskExec;
53 
54 } // end namespace Impl
55 
56 template <class ExecSpace, class QueueType>
57 class BasicTaskScheduler : public Impl::TaskSchedulerBase {
58  public:
59  using scheduler_type = BasicTaskScheduler;
60  using execution_space = ExecSpace;
61  using queue_type = QueueType;
62  using memory_space = typename queue_type::memory_space;
63  using memory_pool = typename queue_type::memory_pool;
64  using specialization = Impl::TaskQueueSpecialization<BasicTaskScheduler>;
65  using member_type = typename specialization::member_type;
66  using team_scheduler_type = BasicTaskScheduler;
67  template <class Functor>
68  using runnable_task_type =
69  Impl::Task<scheduler_type, typename Functor::value_type, Functor>;
70  template <class ValueType>
71  using future_type = Kokkos::BasicFuture<ValueType, BasicTaskScheduler>;
72  template <class FunctorType>
73  using future_type_for_functor = future_type<typename FunctorType::value_type>;
74 
75  private:
76  using track_type = Kokkos::Impl::SharedAllocationTracker;
77  using task_base = Impl::TaskBase;
78 
79  track_type m_track;
80  queue_type* m_queue;
81 
82  //----------------------------------------
83 
84  template <typename, typename>
85  friend class Impl::TaskQueue;
86  template <typename>
87  friend struct Impl::TaskQueueSpecialization;
88  template <typename, typename>
89  friend class Impl::TaskQueueSpecializationConstrained;
90  template <typename, typename>
91  friend class Impl::TaskTeamMemberAdapter;
92  template <typename, typename>
93  friend class Impl::TaskExec;
94 
95  //----------------------------------------
96 
97  KOKKOS_INLINE_FUNCTION
98  BasicTaskScheduler(track_type arg_track, queue_type* arg_queue)
99  : m_track(std::move(arg_track)), m_queue(std::move(arg_queue)) {}
100 
101  KOKKOS_INLINE_FUNCTION
102  team_scheduler_type get_team_scheduler(int team_rank) const {
103  return {m_track, &m_queue->get_team_queue(team_rank)};
104  }
105 
106  //----------------------------------------
107 
108  KOKKOS_INLINE_FUNCTION
109  static constexpr task_base* _get_task_ptr(std::nullptr_t) { return nullptr; }
110 
111  template <class ValueType>
112  KOKKOS_INLINE_FUNCTION static constexpr task_base* _get_task_ptr(
113  future_type<ValueType>&& f) {
114  return f.m_task;
115  }
116 
117  template <int TaskEnum, typename DepTaskType, typename FunctorType>
118  KOKKOS_FUNCTION
119  Kokkos::BasicFuture<typename FunctorType::value_type, scheduler_type>
120  _spawn_impl(DepTaskType* arg_predecessor_task, TaskPriority arg_priority,
121  typename task_base::function_type arg_function,
122  typename task_base::destroy_type /*arg_destroy*/,
123  FunctorType&& arg_functor) {
124  using functor_future_type =
125  future_type_for_functor<std::decay_t<FunctorType>>;
126  using task_type =
127  Impl::Task<BasicTaskScheduler, typename functor_future_type::value_type,
128  FunctorType>;
129 
130  //----------------------------------------
131  // Give single-thread back-ends an opportunity to clear
132  // queue of ready tasks before allocating a new task
133 
134  // TODO @tasking @optimization DSH re-enable this, maybe?
135  // specialization::iff_single_thread_recursive_execute(scheduler);
136 
137  //----------------------------------------
138 
139  functor_future_type f;
140 
141  // Allocate task from memory pool
142 
143  const size_t alloc_size =
144  m_queue->template spawn_allocation_size<FunctorType>();
145 
146  void* task_storage = m_queue->allocate(alloc_size);
147 
148  if (task_storage) {
149  // Placement new construction
150  // Reference count starts at two:
151  // +1 for the matching decrement when task is complete
152  // +1 for the future
153  f.m_task =
154  new (task_storage) task_type(std::forward<FunctorType>(arg_functor));
155 
156  f.m_task->m_apply = arg_function;
157  // f.m_task->m_destroy = arg_destroy;
158  f.m_task->m_queue = m_queue;
159  f.m_task->m_next = arg_predecessor_task;
160  f.m_task->m_ref_count = 2;
161  f.m_task->m_alloc_size = alloc_size;
162  f.m_task->m_task_type = TaskEnum;
163  f.m_task->m_priority = (int16_t)arg_priority;
164 
165  Kokkos::memory_fence();
166 
167  // The dependence (if any) is processed immediately
168  // within the schedule function, as such the dependence's
169  // reference count does not need to be incremented for
170  // the assignment.
171 
172  m_queue->schedule_runnable(f.m_task);
173  // This task may be updated or executed at any moment,
174  // even during the call to 'schedule'.
175  }
176 
177  return f;
178  }
179 
180  public:
181  KOKKOS_INLINE_FUNCTION
182  BasicTaskScheduler() : m_track(), m_queue(nullptr) {}
183 
184  KOKKOS_INLINE_FUNCTION
185  BasicTaskScheduler(BasicTaskScheduler&& rhs) noexcept
186  : m_track(rhs.m_track), // probably should be a move, but this is
187  // deprecated code anyway
188  m_queue(std::move(rhs.m_queue)) {}
189 
190  KOKKOS_INLINE_FUNCTION
191  BasicTaskScheduler(BasicTaskScheduler const& rhs)
192  : m_track(rhs.m_track), m_queue(rhs.m_queue) {}
193 
194  KOKKOS_INLINE_FUNCTION
195  BasicTaskScheduler& operator=(BasicTaskScheduler&& rhs) noexcept {
196  m_track = rhs.m_track; // probably should be a move, but this is deprecated
197  // code anyway
198  m_queue = std::move(rhs.m_queue);
199  return *this;
200  }
201 
202  KOKKOS_INLINE_FUNCTION
203  BasicTaskScheduler& operator=(BasicTaskScheduler const& rhs) {
204  m_track = rhs.m_track;
205  m_queue = rhs.m_queue;
206  return *this;
207  }
208 
209  explicit BasicTaskScheduler(memory_pool const& arg_memory_pool) noexcept
210  : m_track(), m_queue(nullptr) {
211  using record_type =
212  Kokkos::Impl::SharedAllocationRecord<memory_space,
213  typename queue_type::Destroy>;
214 
215  record_type* record = record_type::allocate(
216  memory_space(), "Kokkos::TaskQueue", sizeof(queue_type));
217 
218  m_queue = new (record->data()) queue_type(arg_memory_pool);
219 
220  record->m_destroy.m_queue = m_queue;
221 
222  m_track.assign_allocated_record_to_uninitialized(record);
223  }
224 
225  BasicTaskScheduler(memory_space const& arg_memory_space,
226  size_t const mempool_capacity,
227  unsigned const mempool_min_block_size // = 1u << 6
228  ,
229  unsigned const mempool_max_block_size // = 1u << 10
230  ,
231  unsigned const mempool_superblock_size // = 1u << 12
232  )
233  : BasicTaskScheduler(memory_pool(
234  arg_memory_space, mempool_capacity, mempool_min_block_size,
235  mempool_max_block_size, mempool_superblock_size)) {}
236 
237  //----------------------------------------
238 
239  KOKKOS_INLINE_FUNCTION
240  queue_type& queue() const noexcept {
241  KOKKOS_EXPECTS(m_queue != nullptr);
242  return *m_queue;
243  }
244 
245  KOKKOS_INLINE_FUNCTION
246  memory_pool* memory() const noexcept {
247  return m_queue ? &(m_queue->m_memory) : (memory_pool*)0;
248  }
249 
250  //----------------------------------------
252  template <typename FunctorType>
253  KOKKOS_FUNCTION size_t spawn_allocation_size() const {
254  return m_queue->template spawn_allocation_size<FunctorType>();
255  }
256 
258  KOKKOS_FUNCTION
259  size_t when_all_allocation_size(int narg) const {
260  return m_queue->when_all_allocation_size(narg);
261  }
262 
263  //----------------------------------------
264 
265  template <int TaskEnum, typename DepFutureType, typename FunctorType>
266  KOKKOS_FUNCTION static Kokkos::BasicFuture<typename FunctorType::value_type,
267  scheduler_type>
268  spawn(Impl::TaskPolicyWithScheduler<TaskEnum, scheduler_type, DepFutureType>&&
269  arg_policy,
270  typename task_base::function_type arg_function,
271  typename task_base::destroy_type arg_destroy,
272  FunctorType&& arg_functor) {
273  return std::move(arg_policy.scheduler())
274  .template _spawn_impl<TaskEnum>(
275  _get_task_ptr(std::move(arg_policy.predecessor())),
276  arg_policy.priority(), arg_function, arg_destroy,
277  std::forward<FunctorType>(arg_functor));
278  }
279 
280  template <int TaskEnum, typename DepFutureType, typename FunctorType>
281  KOKKOS_FUNCTION future_type_for_functor<std::decay_t<FunctorType>> spawn(
282  Impl::TaskPolicyWithPredecessor<TaskEnum, DepFutureType>&& arg_policy,
283  FunctorType&& arg_functor) {
284  using task_type = runnable_task_type<FunctorType>;
285  typename task_type::function_type const ptr = task_type::apply;
286  typename task_type::destroy_type const dtor = task_type::destroy;
287 
288  return _spawn_impl<TaskEnum>(
289  _get_task_ptr(std::move(arg_policy).predecessor()),
290  arg_policy.priority(), ptr, dtor,
291  std::forward<FunctorType>(arg_functor));
292  }
293 
294  template <typename FunctorType, typename ValueType, typename Scheduler>
295  KOKKOS_FUNCTION static void respawn(
296  FunctorType* arg_self,
297  BasicFuture<ValueType, Scheduler> const& arg_dependence,
298  TaskPriority const& arg_priority) {
299  // Precondition: task is in Executing state
300 
301  using value_type = typename FunctorType::value_type;
302  using task_type = Impl::Task<BasicTaskScheduler, value_type, FunctorType>;
303 
304  task_type* const task = static_cast<task_type*>(arg_self);
305 
306  task->m_priority = static_cast<int>(arg_priority);
307 
308  task->add_dependence(arg_dependence.m_task);
309 
310  // Postcondition: task is in Executing-Respawn state
311  }
312 
313  template <typename FunctorType>
314  KOKKOS_FUNCTION static void respawn(FunctorType* arg_self,
315  BasicTaskScheduler const&,
316  TaskPriority const& arg_priority) {
317  // Precondition: task is in Executing state
318 
319  using value_type = typename FunctorType::value_type;
320  using task_type = Impl::Task<BasicTaskScheduler, value_type, FunctorType>;
321 
322  task_type* const task = static_cast<task_type*>(arg_self);
323 
324  task->m_priority = static_cast<int>(arg_priority);
325 
326  task->add_dependence(nullptr);
327 
328  // Postcondition: task is in Executing-Respawn state
329  }
330 
331  //----------------------------------------
335  template <typename ValueType>
336  KOKKOS_FUNCTION BasicFuture<void, scheduler_type> when_all(
337  BasicFuture<ValueType, BasicTaskScheduler> const arg[], int narg) {
338  future_type<void> f;
339 
340  if (narg) {
341  queue_type* q = m_queue;
342 
343  // BasicTaskScheduler const* scheduler_ptr = nullptr;
344 
345  for (int i = 0; i < narg; ++i) {
346  task_base* const t = arg[i].m_task;
347  if (nullptr != t) {
348  // Increment reference count to track subsequent assignment.
349  // This likely has to be SeqCst
350  desul::atomic_inc(&(t->m_ref_count), desul::MemoryOrderSeqCst(),
351  desul::MemoryScopeDevice());
352  if (q != static_cast<queue_type const*>(t->m_queue)) {
353  Kokkos::abort(
354  "Kokkos when_all Futures must be in the same scheduler");
355  }
356  }
357  }
358 
359  if (q != nullptr) { // this should probably handle the queue == 0 case,
360  // but this is deprecated code anyway
361 
362  size_t const alloc_size = q->when_all_allocation_size(narg);
363 
364  f.m_task = reinterpret_cast<task_base*>(q->allocate(alloc_size));
365  // f.m_scheduler = *scheduler_ptr;
366 
367  if (f.m_task) {
368  // Reference count starts at two:
369  // +1 to match decrement when task completes
370  // +1 for the future
371 
372  new (f.m_task) task_base();
373 
374  f.m_task->m_queue = q;
375  f.m_task->m_ref_count = 2;
376  f.m_task->m_alloc_size = static_cast<int32_t>(alloc_size);
377  f.m_task->m_dep_count = narg;
378  f.m_task->m_task_type = task_base::Aggregate;
379 
380  // Assign dependences, reference counts were already incremented
381 
382  task_base* volatile* const dep = f.m_task->aggregate_dependences();
383 
384  for (int i = 0; i < narg; ++i) {
385  dep[i] = arg[i].m_task;
386  }
387 
388  Kokkos::memory_fence();
389 
390  q->schedule_aggregate(f.m_task);
391  // this when_all may be processed at any moment
392  }
393  }
394  }
395 
396  return f;
397  }
398 
399  template <class F>
400  KOKKOS_FUNCTION BasicFuture<void, scheduler_type> when_all(int narg,
401  F const func) {
402  using input_type = decltype(func(0));
403 
404  static_assert(is_future<input_type>::value,
405  "Functor must return a Kokkos::Future");
406 
407  future_type<void> f;
408 
409  if (0 == narg) return f;
410 
411  size_t const alloc_size = m_queue->when_all_allocation_size(narg);
412 
413  f.m_task = reinterpret_cast<task_base*>(m_queue->allocate(alloc_size));
414 
415  if (f.m_task) {
416  // Reference count starts at two:
417  // +1 to match decrement when task completes
418  // +1 for the future
419 
420  new (f.m_task) task_base();
421  // f.m_scheduler = *this;
422 
423  // f.m_task->m_scheduler = &f.m_scheduler;
424  f.m_task->m_queue = m_queue;
425  f.m_task->m_ref_count = 2;
426  f.m_task->m_alloc_size = static_cast<int32_t>(alloc_size);
427  f.m_task->m_dep_count = narg;
428  f.m_task->m_task_type = task_base::Aggregate;
429  // f.m_task->m_apply = nullptr;
430  // f.m_task->m_destroy = nullptr;
431 
432  // Assign dependences, reference counts were already incremented
433 
434  task_base* volatile* const dep = f.m_task->aggregate_dependences();
435 
436  for (int i = 0; i < narg; ++i) {
437  const input_type arg_f = func(i);
438  if (nullptr != arg_f.m_task) {
439  // Not scheduled, so task scheduler is not yet set
440  // if ( m_queue != static_cast< BasicTaskScheduler const * >(
441  // arg_f.m_task->m_scheduler )->m_queue ) {
442  // Kokkos::abort("Kokkos when_all Futures must be in the same
443  // scheduler" );
444  //}
445  // Increment reference count to track subsequent assignment.
446  // This increment likely has to be SeqCst
447  desul::atomic_inc(&(arg_f.m_task->m_ref_count),
448  desul::MemoryOrderSeqCst(),
449  desul::MemoryScopeDevice());
450  dep[i] = arg_f.m_task;
451  }
452  }
453 
454  Kokkos::memory_fence();
455 
456  m_queue->schedule_aggregate(f.m_task);
457  // this when_all may be processed at any moment
458  }
459  return f;
460  }
461 
462  //----------------------------------------
463 
464  KOKKOS_INLINE_FUNCTION
465  int allocation_capacity() const noexcept {
466  return m_queue->m_memory.capacity();
467  }
468 
469  KOKKOS_INLINE_FUNCTION
470  int allocated_task_count() const noexcept { return m_queue->m_count_alloc; }
471 
472  KOKKOS_INLINE_FUNCTION
473  int allocated_task_count_max() const noexcept { return m_queue->m_max_alloc; }
474 
475  KOKKOS_INLINE_FUNCTION
476  long allocated_task_count_accum() const noexcept {
477  return m_queue->m_accum_alloc;
478  }
479 
480  //----------------------------------------
481 
482  template <class S, class Q>
483  friend void wait(Kokkos::BasicTaskScheduler<S, Q> const&);
484 };
485 
486 } // namespace Kokkos
487 
488 //----------------------------------------------------------------------------
489 //----------------------------------------------------------------------------
490 
491 namespace Kokkos {
492 
493 //----------------------------------------------------------------------------
494 // Construct a TaskTeam execution policy
495 
496 template <class T, class Scheduler>
497 Impl::TaskPolicyWithPredecessor<Impl::TaskType::TaskTeam,
498  Kokkos::BasicFuture<T, Scheduler>>
499  KOKKOS_INLINE_FUNCTION
500  TaskTeam(Kokkos::BasicFuture<T, Scheduler> arg_future,
501  TaskPriority arg_priority = TaskPriority::Regular) {
502  return {std::move(arg_future), arg_priority};
503 }
504 
505 template <class Scheduler>
506 Impl::TaskPolicyWithScheduler<Impl::TaskType::TaskTeam, Scheduler>
507  KOKKOS_INLINE_FUNCTION TaskTeam(
508  Scheduler arg_scheduler,
509  std::enable_if_t<Kokkos::is_scheduler<Scheduler>::value, TaskPriority>
510  arg_priority = TaskPriority::Regular) {
511  return {std::move(arg_scheduler), arg_priority};
512 }
513 
514 template <class Scheduler, class PredecessorFuture>
515 Impl::TaskPolicyWithScheduler<Kokkos::Impl::TaskType::TaskTeam, Scheduler,
516  PredecessorFuture>
517  KOKKOS_INLINE_FUNCTION
518  TaskTeam(Scheduler arg_scheduler, PredecessorFuture arg_future,
519  std::enable_if_t<Kokkos::is_scheduler<Scheduler>::value &&
520  Kokkos::is_future<PredecessorFuture>::value,
521  TaskPriority>
522  arg_priority = TaskPriority::Regular) {
523  static_assert(std::is_same<typename PredecessorFuture::scheduler_type,
524  Scheduler>::value,
525  "Can't create a task policy from a scheduler and a future from "
526  "a different scheduler");
527 
528  return {std::move(arg_scheduler), std::move(arg_future), arg_priority};
529 }
530 
531 // Construct a TaskSingle execution policy
532 
533 template <class T, class Scheduler>
534 Impl::TaskPolicyWithPredecessor<Impl::TaskType::TaskSingle,
535  Kokkos::BasicFuture<T, Scheduler>>
536  KOKKOS_INLINE_FUNCTION
537  TaskSingle(Kokkos::BasicFuture<T, Scheduler> arg_future,
538  TaskPriority arg_priority = TaskPriority::Regular) {
539  return {std::move(arg_future), arg_priority};
540 }
541 
542 template <class Scheduler>
543 Impl::TaskPolicyWithScheduler<Impl::TaskType::TaskSingle, Scheduler>
544  KOKKOS_INLINE_FUNCTION TaskSingle(
545  Scheduler arg_scheduler,
546  std::enable_if_t<Kokkos::is_scheduler<Scheduler>::value, TaskPriority>
547  arg_priority = TaskPriority::Regular) {
548  return {std::move(arg_scheduler), arg_priority};
549 }
550 
551 template <class Scheduler, class PredecessorFuture>
552 Impl::TaskPolicyWithScheduler<Kokkos::Impl::TaskType::TaskSingle, Scheduler,
553  PredecessorFuture>
554  KOKKOS_INLINE_FUNCTION
555  TaskSingle(Scheduler arg_scheduler, PredecessorFuture arg_future,
556  std::enable_if_t<Kokkos::is_scheduler<Scheduler>::value &&
557  Kokkos::is_future<PredecessorFuture>::value,
558  TaskPriority>
559  arg_priority = TaskPriority::Regular) {
560  static_assert(std::is_same<typename PredecessorFuture::scheduler_type,
561  Scheduler>::value,
562  "Can't create a task policy from a scheduler and a future from "
563  "a different scheduler");
564 
565  return {std::move(arg_scheduler), std::move(arg_future), arg_priority};
566 }
567 
568 //----------------------------------------------------------------------------
569 
576 template <int TaskEnum, typename Scheduler, typename DepFutureType,
577  typename FunctorType>
578 typename Scheduler::template future_type_for_functor<std::decay_t<FunctorType>>
579 host_spawn(Impl::TaskPolicyWithScheduler<TaskEnum, Scheduler, DepFutureType>
580  arg_policy,
581  FunctorType&& arg_functor) {
582  using scheduler_type = Scheduler;
583  using task_type =
584  typename scheduler_type::template runnable_task_type<FunctorType>;
585 
586  static_assert(TaskEnum == Impl::TaskType::TaskTeam ||
587  TaskEnum == Impl::TaskType::TaskSingle,
588  "Kokkos host_spawn requires TaskTeam or TaskSingle");
589 
590  // May be spawning a Cuda task, must use the specialization
591  // to query on-device function pointer.
592  typename task_type::function_type ptr;
593  typename task_type::destroy_type dtor;
594  Kokkos::Impl::TaskQueueSpecialization<
595  scheduler_type>::template get_function_pointer<task_type>(ptr, dtor);
596 
597  return scheduler_type::spawn(std::move(arg_policy), ptr, dtor,
598  std::forward<FunctorType>(arg_functor));
599 }
600 
607 template <int TaskEnum, typename Scheduler, typename DepFutureType,
608  typename FunctorType>
609 typename Scheduler::template future_type_for_functor<std::decay_t<FunctorType>>
610  KOKKOS_INLINE_FUNCTION
611  task_spawn(Impl::TaskPolicyWithScheduler<TaskEnum, Scheduler, DepFutureType>
612  arg_policy,
613  FunctorType&& arg_functor) {
614  using scheduler_type = Scheduler;
615 
616  using task_type =
617  typename scheduler_type::template runnable_task_type<FunctorType>;
618 
619  static_assert(TaskEnum == Impl::TaskType::TaskTeam ||
620  TaskEnum == Impl::TaskType::TaskSingle,
621  "Kokkos task_spawn requires TaskTeam or TaskSingle");
622 
623  typename task_type::function_type const ptr = task_type::apply;
624  typename task_type::destroy_type const dtor = task_type::destroy;
625 
626  return scheduler_type::spawn(std::move(arg_policy), ptr, dtor,
627  std::forward<FunctorType>(arg_functor));
628 }
629 
635 template <typename FunctorType, typename T>
636 void KOKKOS_INLINE_FUNCTION
637 respawn(FunctorType* arg_self, T const& arg,
638  TaskPriority const& arg_priority = TaskPriority::Regular) {
639  static_assert(Kokkos::is_future<T>::value || Kokkos::is_scheduler<T>::value,
640  "Kokkos respawn argument must be Future or TaskScheduler");
641 
642  T::scheduler_type::respawn(arg_self, arg, arg_priority);
643 }
644 
645 //----------------------------------------------------------------------------
646 
647 // template<typename ValueType, typename Scheduler>
648 // KOKKOS_INLINE_FUNCTION
649 // BasicFuture<void, Scheduler>
650 // when_all(BasicFuture<ValueType, Scheduler> const arg[], int narg)
651 //{
652 // return BasicFuture<void, Scheduler>::scheduler_type::when_all(arg, narg);
653 //}
654 
655 //----------------------------------------------------------------------------
656 // Wait for all runnable tasks to complete
657 
658 template <class ExecSpace, class QueueType>
659 inline void wait(BasicTaskScheduler<ExecSpace, QueueType> const& scheduler) {
660  using scheduler_type = BasicTaskScheduler<ExecSpace, QueueType>;
661  scheduler_type::specialization::execute(scheduler);
662  // scheduler.m_queue->execute();
663 }
664 
665 } // namespace Kokkos
666 
667 //----------------------------------------------------------------------------
668 //----------------------------------------------------------------------------
669 
671 // END OLD CODE
673 
674 #endif /* #if defined( KOKKOS_ENABLE_TASKDAG ) */
675 #endif /* #ifndef KOKKOS_TASKSCHEDULER_HPP */