16 #ifndef __INTREPID2_UTILS_HPP__
17 #define __INTREPID2_UTILS_HPP__
19 #include "Intrepid2_ConfigDefs.hpp"
23 #include "Kokkos_Core.hpp"
24 #include "Kokkos_Macros.hpp"
25 #include "Kokkos_Random.hpp"
27 #ifdef HAVE_INTREPID2_SACADO
28 #include "Kokkos_View_Fad_Fwd.hpp"
29 #include "Kokkos_LayoutNatural.hpp"
34 #if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__) || defined(__SYCL_DEVICE_ONLY__)
35 #define INTREPID2_COMPILE_DEVICE_CODE
38 #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) || defined(KOKKOS_ENABLE_SYCL)
39 #define INTREPID2_ENABLE_DEVICE
42 #if defined(KOKKOS_OPT_RANGE_AGGRESSIVE_VECTORIZATION) \
43 && defined(KOKKOS_ENABLE_PRAGMA_IVDEP) \
44 && !defined(INTREPID2_COMPILE_DEVICE_CODE)
45 #define INTREPID2_USE_IVDEP
52 #define INTREPID2_TEST_FOR_WARNING(test, msg) \
54 Kokkos::printf("[Intrepid2] Warning in file %s, line %d\n",__FILE__,__LINE__); \
55 Kokkos::printf(" Test that evaluated to true: %s\n", #test); \
56 Kokkos::printf(" %s \n", msg); \
59 #define INTREPID2_TEST_FOR_EXCEPTION(test, x, msg) \
61 Kokkos::printf("[Intrepid2] Error in file %s, line %d\n",__FILE__,__LINE__); \
62 Kokkos::printf(" Test that evaluated to true: %s\n", #test); \
63 Kokkos::printf(" %s \n", msg); \
69 #ifndef INTREPID2_ENABLE_DEVICE
70 #define INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(test, x, msg) \
72 std::cout << "[Intrepid2] Error in file " << __FILE__ << ", line " << __LINE__ << "\n"; \
73 std::cout << " Test that evaluated to true: " << #test << "\n"; \
74 std::cout << " " << msg << " \n"; \
78 #define INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(test, x, msg) \
80 Kokkos::printf("[Intrepid2] Error in file %s, line %d\n",__FILE__,__LINE__); \
81 Kokkos::printf(" Test that evaluated to true: %s\n", #test); \
82 Kokkos::printf(" %s \n", msg); \
83 Kokkos::abort( "[Intrepid2] Abort\n"); \
86 #if defined(INTREPID2_ENABLE_DEBUG) || defined(NDEBUG) || 1
87 #define INTREPID2_TEST_FOR_ABORT(test, msg) \
89 Kokkos::printf("[Intrepid2] Error in file %s, line %d\n",__FILE__,__LINE__); \
90 Kokkos::printf(" Test that evaluated to true: %s\n", #test); \
91 Kokkos::printf(" %s \n", msg); \
92 Kokkos::abort( "[Intrepid2] Abort\n"); \
95 #define INTREPID2_TEST_FOR_ABORT(test, msg) ((void)0)
98 #ifdef INTREPID2_TEST_FOR_DEBUG_ABORT_OVERRIDE_TO_CONTINUE
99 #define INTREPID2_TEST_FOR_DEBUG_ABORT(test, info, msg) \
100 if (!(info) && (test)) { \
101 Kokkos::printf("[Intrepid2] Error in file %s, line %d\n",__FILE__,__LINE__); \
102 Kokkos::printf(" Test that evaluated to true: %s\n", #test); \
103 Kokkos::printf(" %s \n", msg); \
107 #define INTREPID2_TEST_FOR_DEBUG_ABORT(test, info, msg) \
108 if (!(info) && (test)) { \
109 Kokkos::printf("[Intrepid2] Error in file %s, line %d\n",__FILE__,__LINE__); \
110 Kokkos::printf(" Test that evaluated to true: %s\n", #test); \
111 Kokkos::printf(" %s \n", msg); \
113 Kokkos::abort( "[Intrepid2] Abort\n"); \
122 typedef typename T::scalar_type scalar_type;
131 typedef float scalar_type;
138 typedef double scalar_type;
145 typedef int scalar_type;
152 typedef long int scalar_type;
159 typedef long long scalar_type;
167 template<
typename ViewSpaceType,
typename UserSpaceType>
169 typedef UserSpaceType ExecSpaceType;
175 template<
typename ViewSpaceType>
177 typedef ViewSpaceType ExecSpaceType;
184 template <
typename ViewType>
186 using input_layout =
typename ViewType::array_layout;
187 using default_layout =
typename ViewType::device_type::execution_space::array_layout;
188 using result_layout =
189 typename std::conditional<
190 std::is_same< input_layout, Kokkos::LayoutStride >::value,
192 input_layout >::type;
201 template<
typename IdxType,
typename DimType,
typename IterType>
202 KOKKOS_FORCEINLINE_FUNCTION
204 unrollIndex(IdxType &i, IdxType &j,
207 const IterType iter) {
217 template<
typename IdxType,
typename DimType,
typename IterType>
218 KOKKOS_FORCEINLINE_FUNCTION
220 unrollIndex(IdxType &i, IdxType &j, IdxType &k,
224 const IterType iter) {
230 unrollIndex( i, tmp, dim0, dim1*dim2, iter);
231 unrollIndex( j, k, dim1, dim2, tmp);
240 KOKKOS_FORCEINLINE_FUNCTION
241 static T min(
const T a,
const T b) {
242 return (a < b ? a : b);
245 KOKKOS_FORCEINLINE_FUNCTION
246 static T max(
const T a,
const T b) {
247 return (a > b ? a : b);
250 KOKKOS_FORCEINLINE_FUNCTION
251 static T abs(
const T a) {
252 return (a > 0 ? a : T(-a));
258 KOKKOS_FORCEINLINE_FUNCTION
259 static T min(
const T &a,
const T &b) {
260 return (a < b ? a : b);
264 KOKKOS_FORCEINLINE_FUNCTION
265 static T max(
const T &a,
const T &b) {
266 return (a > b ? a : b);
270 KOKKOS_FORCEINLINE_FUNCTION
271 static T abs(
const T &a) {
272 return (a > 0 ? a : T(-a));
283 KOKKOS_FORCEINLINE_FUNCTION
285 std::enable_if< !(std::is_standard_layout<T>::value && std::is_trivial<T>::value),
typename ScalarTraits<T>::scalar_type >::type
286 get_scalar_value(
const T& obj) {
return obj.val();}
289 KOKKOS_FORCEINLINE_FUNCTION
291 std::enable_if< std::is_standard_layout<T>::value && std::is_trivial<T>::value,
typename ScalarTraits<T>::scalar_type >::type
292 get_scalar_value(
const T& obj){
return obj;}
301 template<
typename T,
typename ...P>
302 KOKKOS_INLINE_FUNCTION
304 std::enable_if< std::is_standard_layout<T>::value && std::is_trivial<T>::value,
unsigned >::type
305 dimension_scalar(
const Kokkos::DynRankView<T, P...> ) {
return 1;}
307 template<
typename T,
typename ...P>
308 KOKKOS_INLINE_FUNCTION
310 std::enable_if< std::is_standard_layout<
typename Kokkos::View<T, P...>::value_type>::value && std::is_trivial<
typename Kokkos::View<T, P...>::value_type>::value,
unsigned >::type
311 dimension_scalar(
const Kokkos::View<T, P...> ) {
return 1;}
313 template<
typename T,
typename ...P>
314 KOKKOS_FORCEINLINE_FUNCTION
315 static ordinal_type get_dimension_scalar(
const Kokkos::DynRankView<T, P...> &view) {
316 return dimension_scalar(view);
319 template<
typename T,
typename ...P>
320 KOKKOS_FORCEINLINE_FUNCTION
321 static ordinal_type get_dimension_scalar(
const Kokkos::View<T, P...> &view) {
322 return dimension_scalar(view);
333 template<
class ViewType,
class ... DimArgs>
335 Kokkos::DynRankView<typename ViewType::value_type, typename DeduceLayout< ViewType >::result_layout,
typename ViewType::device_type >
336 getMatchingViewWithLabel(
const ViewType &view,
const std::string &label, DimArgs... dims)
338 using ValueType =
typename ViewType::value_type;
339 using ResultLayout =
typename DeduceLayout< ViewType >::result_layout;
340 using DeviceType =
typename ViewType::device_type;
341 using ViewTypeWithLayout = Kokkos::DynRankView<ValueType, ResultLayout, DeviceType >;
343 const bool allocateFadStorage = !(std::is_standard_layout<ValueType>::value && std::is_trivial<ValueType>::value);
344 if (!allocateFadStorage)
346 return ViewTypeWithLayout(label,dims...);
350 const int derivative_dimension = get_dimension_scalar(view);
351 return ViewTypeWithLayout(label,dims...,derivative_dimension);
355 using std::enable_if_t;
360 template <
typename T,
typename =
void>
366 template <
typename T>
369 static_assert(!
has_rank_member<Kokkos::DynRankView<double> >::value,
"DynRankView does not have a member rank, so this assert should pass -- if not, something may be wrong with has_rank_member.");
370 #if KOKKOS_VERSION < 40099
371 static_assert(
has_rank_member<Kokkos::View<double*> >::value,
"View has a member rank -- if this assert fails, something may be wrong with has_rank_member.");
377 template<
class Functor, ordinal_type default_value>
379 enable_if_t<has_rank_member<Functor>::value, ordinal_type>
382 return Functor::rank;
388 template<
class Functor, ordinal_type default_value>
390 enable_if_t<!has_rank_member<Functor>::value, ordinal_type>
393 return default_value;
399 template <
typename T>
403 struct two {
char x[2]; };
405 template <
typename C>
static one test(
typename std::remove_reference<decltype( std::declval<C>().
operator()(0))>::type );
406 template <
typename C>
static two test(...);
409 enum { value =
sizeof(test<T>(0)) ==
sizeof(char) && (getFixedRank<T,1>() == 1) };
415 template <
typename T>
419 struct two {
char x[2]; };
421 template <
typename C>
static one test(
typename std::remove_reference<decltype( std::declval<C>().
operator()(0,0))>::type ) ;
422 template <
typename C>
static two test(...);
425 enum { value =
sizeof(test<T>(0)) ==
sizeof(char) && (getFixedRank<T,2>() == 2) };
431 template <
typename T>
435 struct two {
char x[2]; };
437 template <
typename C>
static one test(
typename std::remove_reference<decltype( std::declval<C>().
operator()(0,0,0))>::type ) ;
438 template <
typename C>
static two test(...);
441 enum { value = (
sizeof(test<T>(0)) ==
sizeof(char)) && (getFixedRank<T,3>() == 3) };
447 template <
typename T>
451 struct two {
char x[2]; };
453 template <
typename C>
static one test(
typename std::remove_reference<decltype( std::declval<C>().
operator()(0,0,0,0))>::type ) ;
454 template <
typename C>
static two test(...);
457 enum { value =
sizeof(test<T>(0)) ==
sizeof(char) && (getFixedRank<T,4>() == 4) };
463 template <
typename T>
467 struct two {
char x[2]; };
469 template <
typename C>
static one test(
typename std::remove_reference<decltype( std::declval<C>().
operator()(0,0,0,0,0))>::type ) ;
470 template <
typename C>
static two test(...);
473 enum { value =
sizeof(test<T>(0)) ==
sizeof(char) && (getFixedRank<T,5>() == 5) };
479 template <
typename T>
483 struct two {
char x[2]; };
485 template <
typename C>
static one test(
typename std::remove_reference<decltype( std::declval<C>().
operator()(0,0,0,0,0,0))>::type ) ;
486 template <
typename C>
static two test(...);
489 enum { value =
sizeof(test<T>(0)) ==
sizeof(char) && (getFixedRank<T,6>() == 6) };
495 template <
typename T>
499 struct two {
char x[2]; };
501 template <
typename C>
static one test(
typename std::remove_reference<decltype( std::declval<C>().
operator()(0,0,0,0,0,0,0))>::type ) ;
502 template <
typename C>
static two test(...);
505 enum { value =
sizeof(test<T>(0)) ==
sizeof(char) && (getFixedRank<T,7>() == 7) };
511 template <
typename T,
int rank>
515 enum { value =
false };
521 template <
typename T>
529 template <
typename T>
537 template <
typename T>
545 template <
typename T>
553 template <
typename T>
561 template <
typename T>
569 template <
typename T>
581 template<
typename Scalar,
int rank>
589 template<
typename Scalar>
592 using value_type = Scalar;
598 template<
typename Scalar>
601 using value_type = Scalar*;
607 template<
typename Scalar>
610 using value_type = Scalar**;
616 template<
typename Scalar>
619 using value_type = Scalar***;
625 template<
typename Scalar>
628 using value_type = Scalar****;
634 template<
typename Scalar>
637 using value_type = Scalar*****;
643 template<
typename Scalar>
646 using value_type = Scalar******;
652 template<
typename Scalar>
655 using value_type = Scalar*******;
659 static_assert(
supports_rank<Kokkos::DynRankView<double>, 1>::value,
"rank 1 check of supports_rank for DynRankView");
660 static_assert(
supports_rank<Kokkos::DynRankView<double>, 2>::value,
"rank 2 check of supports_rank for DynRankView");
661 static_assert(
supports_rank<Kokkos::DynRankView<double>, 3>::value,
"rank 3 check of supports_rank for DynRankView");
662 static_assert(
supports_rank<Kokkos::DynRankView<double>, 4>::value,
"rank 4 check of supports_rank for DynRankView");
663 static_assert(
supports_rank<Kokkos::DynRankView<double>, 5>::value,
"rank 5 check of supports_rank for DynRankView");
664 static_assert(
supports_rank<Kokkos::DynRankView<double>, 6>::value,
"rank 6 check of supports_rank for DynRankView");
665 static_assert(
supports_rank<Kokkos::DynRankView<double>, 7>::value,
"rank 7 check of supports_rank for DynRankView");
668 static_assert(
supports_rank<Kokkos::View<double*>, 1>::value,
"rank 1 check of supports_rank");
669 static_assert(
supports_rank<Kokkos::View<double**>, 2>::value,
"rank 2 check of supports_rank");
670 static_assert(
supports_rank<Kokkos::View<double***>, 3>::value,
"rank 3 check of supports_rank");
671 static_assert(
supports_rank<Kokkos::View<double****>, 4>::value,
"rank 4 check of supports_rank");
672 static_assert(
supports_rank<Kokkos::View<double*****>, 5>::value,
"rank 5 check of supports_rank");
673 static_assert(
supports_rank<Kokkos::View<double******>, 6>::value,
"rank 6 check of supports_rank");
674 static_assert(
supports_rank<Kokkos::View<double*******>, 7>::value,
"rank 7 check of supports_rank");
677 static_assert(!
supports_rank<Kokkos::View<double*>, 2>::value,
"rank 1 check of supports_rank");
678 static_assert(!
supports_rank<Kokkos::View<double*>, 3>::value,
"rank 1 check of supports_rank");
679 static_assert(!
supports_rank<Kokkos::View<double*>, 4>::value,
"rank 1 check of supports_rank");
680 static_assert(!
supports_rank<Kokkos::View<double*>, 5>::value,
"rank 1 check of supports_rank");
681 static_assert(!
supports_rank<Kokkos::View<double*>, 6>::value,
"rank 1 check of supports_rank");
682 static_assert(!
supports_rank<Kokkos::View<double*>, 7>::value,
"rank 1 check of supports_rank");
683 static_assert(!
supports_rank<Kokkos::View<double**>, 1>::value,
"rank 2 check of supports_rank");
684 static_assert(!
supports_rank<Kokkos::View<double**>, 3>::value,
"rank 2 check of supports_rank");
685 static_assert(!
supports_rank<Kokkos::View<double**>, 4>::value,
"rank 2 check of supports_rank");
686 static_assert(!
supports_rank<Kokkos::View<double**>, 5>::value,
"rank 2 check of supports_rank");
687 static_assert(!
supports_rank<Kokkos::View<double**>, 6>::value,
"rank 2 check of supports_rank");
688 static_assert(!
supports_rank<Kokkos::View<double**>, 7>::value,
"rank 2 check of supports_rank");
689 static_assert(!
supports_rank<Kokkos::View<double***>, 1>::value,
"rank 3 check of supports_rank");
690 static_assert(!
supports_rank<Kokkos::View<double***>, 2>::value,
"rank 3 check of supports_rank");
691 static_assert(!
supports_rank<Kokkos::View<double***>, 4>::value,
"rank 3 check of supports_rank");
692 static_assert(!
supports_rank<Kokkos::View<double***>, 5>::value,
"rank 3 check of supports_rank");
693 static_assert(!
supports_rank<Kokkos::View<double***>, 6>::value,
"rank 3 check of supports_rank");
694 static_assert(!
supports_rank<Kokkos::View<double***>, 7>::value,
"rank 3 check of supports_rank");
695 static_assert(!
supports_rank<Kokkos::View<double****>, 1>::value,
"rank 4 check of supports_rank");
696 static_assert(!
supports_rank<Kokkos::View<double****>, 2>::value,
"rank 4 check of supports_rank");
697 static_assert(!
supports_rank<Kokkos::View<double****>, 3>::value,
"rank 4 check of supports_rank");
698 static_assert(!
supports_rank<Kokkos::View<double****>, 5>::value,
"rank 4 check of supports_rank");
699 static_assert(!
supports_rank<Kokkos::View<double****>, 6>::value,
"rank 4 check of supports_rank");
700 static_assert(!
supports_rank<Kokkos::View<double****>, 7>::value,
"rank 4 check of supports_rank");
701 static_assert(!
supports_rank<Kokkos::View<double*****>, 1>::value,
"rank 5 check of supports_rank");
702 static_assert(!
supports_rank<Kokkos::View<double*****>, 2>::value,
"rank 5 check of supports_rank");
703 static_assert(!
supports_rank<Kokkos::View<double*****>, 3>::value,
"rank 5 check of supports_rank");
704 static_assert(!
supports_rank<Kokkos::View<double*****>, 4>::value,
"rank 5 check of supports_rank");
705 static_assert(!
supports_rank<Kokkos::View<double*****>, 6>::value,
"rank 5 check of supports_rank");
706 static_assert(!
supports_rank<Kokkos::View<double*****>, 7>::value,
"rank 5 check of supports_rank");
707 static_assert(!
supports_rank<Kokkos::View<double******>, 1>::value,
"rank 6 check of supports_rank");
708 static_assert(!
supports_rank<Kokkos::View<double******>, 2>::value,
"rank 6 check of supports_rank");
709 static_assert(!
supports_rank<Kokkos::View<double******>, 3>::value,
"rank 6 check of supports_rank");
710 static_assert(!
supports_rank<Kokkos::View<double******>, 4>::value,
"rank 6 check of supports_rank");
711 static_assert(!
supports_rank<Kokkos::View<double******>, 5>::value,
"rank 6 check of supports_rank");
712 static_assert(!
supports_rank<Kokkos::View<double******>, 7>::value,
"rank 6 check of supports_rank");
713 static_assert(!
supports_rank<Kokkos::View<double*******>, 1>::value,
"rank 7 check of supports_rank");
714 static_assert(!
supports_rank<Kokkos::View<double*******>, 2>::value,
"rank 7 check of supports_rank");
715 static_assert(!
supports_rank<Kokkos::View<double*******>, 3>::value,
"rank 7 check of supports_rank");
716 static_assert(!
supports_rank<Kokkos::View<double*******>, 4>::value,
"rank 7 check of supports_rank");
717 static_assert(!
supports_rank<Kokkos::View<double*******>, 5>::value,
"rank 7 check of supports_rank");
718 static_assert(!
supports_rank<Kokkos::View<double*******>, 6>::value,
"rank 7 check of supports_rank");
723 template <
typename T>
727 struct two {
char x[2]; };
729 template <
typename C>
static one test( decltype( std::declval<C>().rank() ) ) ;
730 template <
typename C>
static two test(...);
733 enum { value =
sizeof(test<T>(0)) ==
sizeof(char) };
736 static_assert(
has_rank_method<Kokkos::DynRankView<double> >::value,
"DynRankView implements rank(), so this assert should pass -- if not, something may be wrong with has_rank_method.");
737 #if KOKKOS_VERSION < 40099
738 static_assert(
has_rank_member<Kokkos::View<double*> >::value,
"View has a member rank -- if this assert fails, something may be wrong with has_rank_member.");
744 template<
class Functor>
745 enable_if_t<has_rank_method<Functor>::value,
unsigned>
746 KOKKOS_INLINE_FUNCTION
747 getFunctorRank(
const Functor &functor)
749 return functor.rank();
755 template<
class Functor>
756 enable_if_t<!has_rank_method<Functor>::value,
unsigned>
757 KOKKOS_INLINE_FUNCTION
758 getFunctorRank(
const Functor &functor)
766 #if defined(HAVE_INTREPID2_SACADO) && !defined(SACADO_HAS_NEW_KOKKOS_VIEW_IMPL)
767 template <
typename ValueType>
768 struct NaturalLayoutForType {
770 typename std::conditional<(std::is_standard_layout<ValueType>::value && std::is_trivial<ValueType>::value),
772 Kokkos::LayoutNatural<Kokkos::LayoutLeft> >::type;
775 template <
typename ValueType>
777 using layout = Kokkos::LayoutLeft;
782 const int VECTOR_SIZE = 1;
783 #if defined(SACADO_VIEW_CUDA_HIERARCHICAL_DFAD) && defined(INTREPID2_ENABLE_DEVICE)
784 const int FAD_VECTOR_SIZE = 32;
786 const int FAD_VECTOR_SIZE = 1;
792 template<
typename Scalar>
793 constexpr
int getVectorSizeForHierarchicalParallelism()
795 return (std::is_standard_layout<Scalar>::value && std::is_trivial<Scalar>::value) ? VECTOR_SIZE : FAD_VECTOR_SIZE;
803 template<
typename ViewType>
804 KOKKOS_INLINE_FUNCTION
805 constexpr
unsigned getScalarDimensionForView(
const ViewType &view)
807 return (std::is_standard_layout<typename ViewType::value_type>::value && std::is_trivial<typename ViewType::value_type>::value) ? 0 : get_dimension_scalar(view);
811 template<
typename Device>
814 void operator()(T* ptr) {
815 Kokkos::parallel_for(Kokkos::RangePolicy<typename Device::execution_space>(0,1),
816 KOKKOS_LAMBDA (
const int i) { ptr->~T(); });
817 typename Device::execution_space().fence();
818 Kokkos::kokkos_free<typename Device::memory_space>(ptr);
825 template<
typename Device,
typename Derived>
826 std::unique_ptr<Derived,DeviceDeleter<Device>>
827 copy_virtual_class_to_device(
const Derived& host_source)
829 auto* p =
static_cast<Derived*
>(Kokkos::kokkos_malloc<typename Device::memory_space>(
sizeof(Derived)));
830 Kokkos::parallel_for(Kokkos::RangePolicy<typename Device::execution_space>(0,1),
831 KOKKOS_LAMBDA (
const int i) {
new (p) Derived(host_source); });
832 typename Device::execution_space().fence();
833 return std::unique_ptr<Derived,DeviceDeleter<Device>>(p);
SFINAE helper to detect whether a type supports a 5-integral-argument operator(). ...
SFINAE helper to detect whether a type supports a 1-integral-argument operator(). ...
SFINAE helper to detect whether a type supports a 6-integral-argument operator(). ...
SFINAE helper to detect whether a type supports a 7-integral-argument operator(). ...
SFINAE helper to detect whether a type supports a rank-integral-argument operator().
SFINAE helper to detect whether a type supports a 2-integral-argument operator(). ...
Implementation of an assert that can safely be called from device code.
Helper to get Scalar[*+] where the number of *'s matches the given rank.
SFINAE helper to detect whether a type supports a 3-integral-argument operator(). ...
Tests whether a class implements rank(). Used in getFunctorRank() method below; allows us to do one t...
Contains definitions of custom data types in Intrepid2.
Define layout that will allow us to wrap Sacado Scalar objects in Views without copying.
SFINAE helper to detect whether a type supports a 4-integral-argument operator(). ...
Tests whether a class has a member rank. Used in getFixedRank() method below, which in turn is used i...
Struct for deleting device instantiation.
layout deduction (temporary meta-function)