8 #ifndef Intrepid2_Data_h
9 #define Intrepid2_Data_h
15 #include "Intrepid2_ScalarView.hpp"
33 template<
class DataScalar,
typename DeviceType>
36 static ScalarView<DataScalar,DeviceType> zeroView()
38 static ScalarView<DataScalar,DeviceType> zeroView = ScalarView<DataScalar,DeviceType>(
"zero",1);
39 static bool havePushedFinalizeHook =
false;
40 if (!havePushedFinalizeHook)
42 Kokkos::push_finalize_hook( [=] {
43 zeroView = ScalarView<DataScalar,DeviceType>();
45 havePushedFinalizeHook =
true;
68 template<
class DataScalar,
typename DeviceType>
71 using value_type = DataScalar;
72 using execution_space =
typename DeviceType::execution_space;
74 using reference_type =
typename ScalarView<DataScalar,DeviceType>::reference_type;
75 using const_reference_type =
typename ScalarView<const DataScalar,DeviceType>::reference_type;
77 ordinal_type dataRank_;
78 Kokkos::View<DataScalar*, DeviceType> data1_;
79 Kokkos::View<DataScalar**, DeviceType> data2_;
80 Kokkos::View<DataScalar***, DeviceType> data3_;
81 Kokkos::View<DataScalar****, DeviceType> data4_;
82 Kokkos::View<DataScalar*****, DeviceType> data5_;
83 Kokkos::View<DataScalar******, DeviceType> data6_;
84 Kokkos::View<DataScalar*******, DeviceType> data7_;
85 Kokkos::Array<int,7> extents_;
86 Kokkos::Array<DataVariationType,7> variationType_;
87 Kokkos::Array<int,7> variationModulus_;
88 int blockPlusDiagonalLastNonDiagonal_ = -1;
90 bool hasNontrivialModulusUNUSED_;
91 bool underlyingMatchesLogical_;
92 Kokkos::Array<ordinal_type,7> activeDims_;
98 using return_type = const_reference_type;
100 ScalarView<DataScalar,DeviceType> zeroView_;
103 KOKKOS_INLINE_FUNCTION
106 return (lastNondiagonal + 1) * (lastNondiagonal + 1);
110 KOKKOS_INLINE_FUNCTION
113 return i * (lastNondiagonal + 1) + j;
117 KOKKOS_INLINE_FUNCTION
120 return i - (lastNondiagonal + 1) + numNondiagonalEntries;
124 KOKKOS_INLINE_FUNCTION
129 case 1:
return data1_.extent_int(dim);
130 case 2:
return data2_.extent_int(dim);
131 case 3:
return data3_.extent_int(dim);
132 case 4:
return data4_.extent_int(dim);
133 case 5:
return data5_.extent_int(dim);
134 case 6:
return data6_.extent_int(dim);
135 case 7:
return data7_.extent_int(dim);
145 for (
int d=rank_; d<7; d++)
147 INTREPID2_TEST_FOR_EXCEPTION(extents_[d] > 1, std::invalid_argument,
"Nominal extents may not be > 1 in dimensions beyond the rank of the container");
151 int blockPlusDiagonalCount = 0;
152 underlyingMatchesLogical_ =
true;
153 for (ordinal_type i=0; i<7; i++)
155 if (variationType_[i] == GENERAL)
157 if (extents_[i] != 0)
159 variationModulus_[i] = extents_[i];
163 variationModulus_[i] = 1;
165 activeDims_[numActiveDims_] = i;
168 else if (variationType_[i] == MODULAR)
170 underlyingMatchesLogical_ =
false;
174 const int logicalExtent = extents_[i];
175 const int modulus = dataExtent;
177 INTREPID2_TEST_FOR_EXCEPTION( dataExtent * (logicalExtent / dataExtent) != logicalExtent, std::invalid_argument,
"data extent must evenly divide logical extent");
179 variationModulus_[i] = modulus;
183 variationModulus_[i] = extents_[i];
185 activeDims_[numActiveDims_] = i;
188 else if (variationType_[i] == BLOCK_PLUS_DIAGONAL)
190 underlyingMatchesLogical_ =
false;
191 blockPlusDiagonalCount++;
192 if (blockPlusDiagonalCount == 1)
195 #ifdef HAVE_INTREPID2_DEBUG
198 const int logicalExtent = extents_[i];
199 const int numDiagonalEntries = logicalExtent - (blockPlusDiagonalLastNonDiagonal_ + 1);
200 const int expectedDataExtent = numNondiagonalEntries + numDiagonalEntries;
201 INTREPID2_TEST_FOR_EXCEPTION(dataExtent != expectedDataExtent, std::invalid_argument, (
"BLOCK_PLUS_DIAGONAL data extent of " + std::to_string(dataExtent) +
" does not match expected based on blockPlusDiagonalLastNonDiagonal setting of " + std::to_string(blockPlusDiagonalLastNonDiagonal_)).c_str());
204 activeDims_[numActiveDims_] = i;
208 INTREPID2_TEST_FOR_EXCEPTION(variationType_[i+1] != BLOCK_PLUS_DIAGONAL, std::invalid_argument,
"BLOCK_PLUS_DIAGONAL ranks must be contiguous");
210 variationModulus_[i] = 1;
211 INTREPID2_TEST_FOR_EXCEPTION(blockPlusDiagonalCount > 1, std::invalid_argument,
"BLOCK_PLUS_DIAGONAL can only apply to two ranks");
217 underlyingMatchesLogical_ =
false;
219 variationModulus_[i] = 1;
223 if (rank_ != dataRank_)
225 underlyingMatchesLogical_ =
false;
228 for (
int d=numActiveDims_; d<7; d++)
234 for (
int d=0; d<7; d++)
236 INTREPID2_TEST_FOR_EXCEPTION(variationModulus_[d] == 0, std::logic_error,
"variationModulus should not ever be 0");
242 template<
class UnaryOperator>
245 using ExecutionSpace =
typename DeviceType::execution_space;
251 const int dataRank = 1;
252 auto view = getUnderlyingView<dataRank>();
255 Kokkos::RangePolicy<ExecutionSpace> policy(ExecutionSpace(),0,dataExtent);
256 Kokkos::parallel_for(
"apply operator in-place", policy,
257 KOKKOS_LAMBDA (
const int &i0) {
258 view(i0) = unaryOperator(view(i0));
265 const int dataRank = 2;
266 auto policy = dataExtentRangePolicy<dataRank>();
267 auto view = getUnderlyingView<dataRank>();
269 Kokkos::parallel_for(
"apply operator in-place", policy,
270 KOKKOS_LAMBDA (
const int &i0,
const int &i1) {
271 view(i0,i1) = unaryOperator(view(i0,i1));
277 const int dataRank = 3;
278 auto policy = dataExtentRangePolicy<dataRank>();
279 auto view = getUnderlyingView<dataRank>();
281 Kokkos::parallel_for(
"apply operator in-place", policy,
282 KOKKOS_LAMBDA (
const int &i0,
const int &i1,
const int &i2) {
283 view(i0,i1,i2) = unaryOperator(view(i0,i1,i2));
289 const int dataRank = 4;
290 auto policy = dataExtentRangePolicy<dataRank>();
291 auto view = getUnderlyingView<dataRank>();
293 Kokkos::parallel_for(
"apply operator in-place", policy,
294 KOKKOS_LAMBDA (
const int &i0,
const int &i1,
const int &i2,
const int &i3) {
295 view(i0,i1,i2,i3) = unaryOperator(view(i0,i1,i2,i3));
301 const int dataRank = 5;
302 auto policy = dataExtentRangePolicy<dataRank>();
303 auto view = getUnderlyingView<dataRank>();
305 Kokkos::parallel_for(
"apply operator in-place", policy,
306 KOKKOS_LAMBDA (
const int &i0,
const int &i1,
const int &i2,
const int &i3,
const int &i4) {
307 view(i0,i1,i2,i3,i4) = unaryOperator(view(i0,i1,i2,i3,i4));
313 const int dataRank = 6;
314 auto policy = dataExtentRangePolicy<dataRank>();
315 auto view = getUnderlyingView<dataRank>();
317 Kokkos::parallel_for(
"apply operator in-place", policy,
318 KOKKOS_LAMBDA (
const int &i0,
const int &i1,
const int &i2,
const int &i3,
const int &i4,
const int &i5) {
319 view(i0,i1,i2,i3,i4,i5) = unaryOperator(view(i0,i1,i2,i3,i4,i5));
325 const int dataRank = 7;
326 auto policy6 = dataExtentRangePolicy<6>();
327 auto view = getUnderlyingView<dataRank>();
329 const int dim_i6 = view.extent_int(6);
331 Kokkos::parallel_for(
"apply operator in-place", policy6,
332 KOKKOS_LAMBDA (
const int &i0,
const int &i1,
const int &i2,
const int &i3,
const int &i4,
const int &i5) {
333 for (
int i6=0; i6<dim_i6; i6++)
335 view(i0,i1,i2,i3,i4,i5,i6) = unaryOperator(view(i0,i1,i2,i3,i4,i5,i6));
341 INTREPID2_TEST_FOR_EXCEPTION(
true,std::invalid_argument,
"Unsupported data rank");
346 template<
class ...IntArgs>
347 KOKKOS_INLINE_FUNCTION
350 #ifdef INTREPID2_HAVE_DEBUG
353 constexpr
int numArgs =
sizeof...(intArgs);
354 if (underlyingMatchesLogical_)
357 return getUnderlyingView<numArgs>()(intArgs...);
361 using int_type = std::tuple_element_t<0, std::tuple<IntArgs...>>;
363 const Kokkos::Array<int_type, numArgs+1> args {intArgs...,0};
364 Kokkos::Array<int_type, 7> refEntry;
365 for (
int d=0; d<numArgs; d++)
367 switch (variationType_[d])
369 case CONSTANT: refEntry[d] = 0;
break;
370 case GENERAL: refEntry[d] = args[d];
break;
371 case MODULAR: refEntry[d] = args[d] % variationModulus_[d];
break;
372 case BLOCK_PLUS_DIAGONAL:
374 if (passThroughBlockDiagonalArgs)
376 refEntry[d] = args[d];
377 refEntry[d+1] = args[d+1];
384 const int_type &i = args[d];
391 const int_type &j = args[d+1];
393 if ((i > static_cast<int_type>(blockPlusDiagonalLastNonDiagonal_)) || (j > static_cast<int_type>(blockPlusDiagonalLastNonDiagonal_)))
419 for (
int d=numArgs; d<7; d++)
426 return data1_(refEntry[activeDims_[0]]);
428 else if (dataRank_ == 2)
430 return data2_(refEntry[activeDims_[0]],refEntry[activeDims_[1]]);
432 else if (dataRank_ == 3)
434 return data3_(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]]);
436 else if (dataRank_ == 4)
438 return data4_(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]],refEntry[activeDims_[3]]);
440 else if (dataRank_ == 5)
442 return data5_(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]],refEntry[activeDims_[3]],
443 refEntry[activeDims_[4]]);
445 else if (dataRank_ == 6)
447 return data6_(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]],refEntry[activeDims_[3]],
448 refEntry[activeDims_[4]],refEntry[activeDims_[5]]);
452 return data7_(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]],refEntry[activeDims_[3]],
453 refEntry[activeDims_[4]],refEntry[activeDims_[5]],refEntry[activeDims_[6]]);
459 template<
class ...IntArgs>
460 KOKKOS_INLINE_FUNCTION
467 template<
class ToContainer,
class FromContainer>
471 auto policy = Kokkos::MDRangePolicy<execution_space,Kokkos::Rank<6>>({0,0,0,0,0,0},{from.extent_int(0),from.extent_int(1),from.extent_int(2), from.extent_int(3), from.extent_int(4), from.extent_int(5)});
473 Kokkos::parallel_for(
"copyContainer", policy,
474 KOKKOS_LAMBDA (
const int &i0,
const int &i1,
const int &i2,
const int &i3,
const int &i4,
const int &i5) {
475 for (
int i6=0; i6<from.extent_int(6); i6++)
477 to.access(i0,i1,i2,i3,i4,i5,i6) = from.access(i0,i1,i2,i3,i4,i5,i6);
488 case 1: data1_ = Kokkos::View<DataScalar*, DeviceType>(
"Intrepid2 Data", data.extent_int(0));
break;
489 case 2: data2_ = Kokkos::View<DataScalar**, DeviceType>(
"Intrepid2 Data", data.extent_int(0), data.extent_int(1));
break;
490 case 3: data3_ = Kokkos::View<DataScalar***, DeviceType>(
"Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2));
break;
491 case 4: data4_ = Kokkos::View<DataScalar****, DeviceType>(
"Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2), data.extent_int(3));
break;
492 case 5: data5_ = Kokkos::View<DataScalar*****, DeviceType>(
"Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2), data.extent_int(3), data.extent_int(4));
break;
493 case 6: data6_ = Kokkos::View<DataScalar******, DeviceType>(
"Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2), data.extent_int(3), data.extent_int(4), data.extent_int(5));
break;
494 case 7: data7_ = Kokkos::View<DataScalar*******, DeviceType>(
"Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2), data.extent_int(3), data.extent_int(4), data.extent_int(5), data.extent_int(6));
break;
495 default: INTREPID2_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Invalid data rank");
507 default: INTREPID2_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Invalid data rank");
512 Data(std::vector<DimensionInfo> dimInfoVector)
515 dataRank_(0), extents_({0,0,0,0,0,0,0}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(-1), rank_(dimInfoVector.size())
519 if (dimInfoVector.size() != 0)
521 std::vector<int> dataExtents;
523 bool blockPlusDiagonalEncountered =
false;
524 for (
int d=0; d<rank_; d++)
526 const DimensionInfo & dimInfo = dimInfoVector[d];
527 extents_[d] = dimInfo.logicalExtent;
528 variationType_[d] = dimInfo.variationType;
529 const bool isBlockPlusDiagonal = (variationType_[d] == BLOCK_PLUS_DIAGONAL);
530 const bool isSecondBlockPlusDiagonal = isBlockPlusDiagonal && blockPlusDiagonalEncountered;
531 if (isBlockPlusDiagonal)
533 blockPlusDiagonalEncountered =
true;
534 blockPlusDiagonalLastNonDiagonal_ = dimInfo.blockPlusDiagonalLastNonDiagonal;
536 if ((variationType_[d] != CONSTANT) && (!isSecondBlockPlusDiagonal))
538 dataExtents.push_back(dimInfo.dataExtent);
541 if (dataExtents.size() == 0)
544 dataExtents.push_back(1);
546 dataRank_ = dataExtents.size();
549 case 1: data1_ = Kokkos::View<DataScalar*, DeviceType>(
"Intrepid2 Data", dataExtents[0]);
break;
550 case 2: data2_ = Kokkos::View<DataScalar**, DeviceType>(
"Intrepid2 Data", dataExtents[0], dataExtents[1]);
break;
551 case 3: data3_ = Kokkos::View<DataScalar***, DeviceType>(
"Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2]);
break;
552 case 4: data4_ = Kokkos::View<DataScalar****, DeviceType>(
"Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2], dataExtents[3]);
break;
553 case 5: data5_ = Kokkos::View<DataScalar*****, DeviceType>(
"Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2], dataExtents[3], dataExtents[4]);
break;
554 case 6: data6_ = Kokkos::View<DataScalar******, DeviceType>(
"Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2], dataExtents[3], dataExtents[4], dataExtents[5]);
break;
555 case 7: data7_ = Kokkos::View<DataScalar*******, DeviceType>(
"Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2], dataExtents[3], dataExtents[4], dataExtents[5], dataExtents[6]);
break;
556 default: INTREPID2_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Invalid data rank");
572 template<typename OtherDeviceType, class = typename std::enable_if< std::is_same<typename DeviceType::memory_space, typename OtherDeviceType::memory_space>::value>::type,
573 class = typename std::enable_if<!std::is_same<DeviceType,OtherDeviceType>::value>::type>
574 Data(const Data<DataScalar,OtherDeviceType> &data)
576 dataRank_(data.getUnderlyingViewRank()), extents_(data.getExtents()), variationType_(data.getVariationTypes()), blockPlusDiagonalLastNonDiagonal_(data.blockPlusDiagonalLastNonDiagonal()), rank_(data.rank())
584 case 1: data1_ = data.getUnderlyingView1();
break;
585 case 2: data2_ = data.getUnderlyingView2();
break;
586 case 3: data3_ = data.getUnderlyingView3();
break;
587 case 4: data4_ = data.getUnderlyingView4();
break;
588 case 5: data5_ = data.getUnderlyingView5();
break;
589 case 6: data6_ = data.getUnderlyingView6();
break;
590 case 7: data7_ = data.getUnderlyingView7();
break;
591 default: INTREPID2_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Invalid data rank");
598 template<typename OtherDeviceType, class = typename std::enable_if<!std::is_same<typename DeviceType::memory_space, typename OtherDeviceType::memory_space>::value>::type>
609 case 1: data1_ = Kokkos::View<DataScalar*, DeviceType>(
"Intrepid2 Data", view.extent_int(0));
break;
610 case 2: data2_ = Kokkos::View<DataScalar**, DeviceType>(
"Intrepid2 Data", view.extent_int(0), view.extent_int(1));
break;
611 case 3: data3_ = Kokkos::View<DataScalar***, DeviceType>(
"Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2));
break;
612 case 4: data4_ = Kokkos::View<DataScalar****, DeviceType>(
"Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3));
break;
613 case 5: data5_ = Kokkos::View<DataScalar*****, DeviceType>(
"Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4));
break;
614 case 6: data6_ = Kokkos::View<DataScalar******, DeviceType>(
"Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4), view.extent_int(5));
break;
615 case 7: data7_ = Kokkos::View<DataScalar*******, DeviceType>(
"Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4), view.extent_int(5), view.extent_int(6));
break;
616 default: INTREPID2_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Invalid data rank");
622 using MemorySpace =
typename DeviceType::memory_space;
625 case 1: {
auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.
getUnderlyingView1());
copyContainer(data1_, dataViewMirror);}
break;
626 case 2: {
auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.
getUnderlyingView2());
copyContainer(data2_, dataViewMirror);}
break;
627 case 3: {
auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.
getUnderlyingView3());
copyContainer(data3_, dataViewMirror);}
break;
628 case 4: {
auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.
getUnderlyingView4());
copyContainer(data4_, dataViewMirror);}
break;
629 case 5: {
auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.
getUnderlyingView5());
copyContainer(data5_, dataViewMirror);}
break;
630 case 6: {
auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.
getUnderlyingView6());
copyContainer(data6_, dataViewMirror);}
break;
631 case 7: {
auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.
getUnderlyingView7());
copyContainer(data7_, dataViewMirror);}
break;
632 default: INTREPID2_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Invalid data rank");
680 Data(ScalarView<DataScalar,DeviceType> data)
684 Kokkos::Array<int,7> {data.extent_int(0),data.extent_int(1),data.extent_int(2),data.extent_int(3),data.extent_int(4),data.extent_int(5),data.extent_int(6)},
685 Kokkos::Array<DataVariationType,7> {GENERAL,GENERAL,GENERAL,GENERAL,GENERAL,GENERAL,GENERAL}, -1)
689 template<
size_t rank,
class ...DynRankViewProperties>
690 Data(
const Kokkos::DynRankView<DataScalar,DeviceType, DynRankViewProperties...> &data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType,
const int blockPlusDiagonalLastNonDiagonal = -1)
692 dataRank_(data.
rank()), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(
blockPlusDiagonalLastNonDiagonal), rank_(
rank)
696 for (
unsigned d=0; d<
rank; d++)
698 extents_[d] = extents[d];
699 variationType_[d] = variationType[d];
704 template<
size_t rank,
class ...ViewProperties>
705 Data(Kokkos::View<DataScalar*,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType,
const int blockPlusDiagonalLastNonDiagonal = -1)
707 dataRank_(data.
rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(
blockPlusDiagonalLastNonDiagonal), rank_(
rank)
710 for (
unsigned d=0; d<
rank; d++)
712 extents_[d] = extents[d];
713 variationType_[d] = variationType[d];
718 template<
size_t rank,
class ...ViewProperties>
719 Data(Kokkos::View<DataScalar**,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType,
const int blockPlusDiagonalLastNonDiagonal = -1)
721 dataRank_(data.
rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(
blockPlusDiagonalLastNonDiagonal), rank_(
rank)
724 for (
unsigned d=0; d<
rank; d++)
726 extents_[d] = extents[d];
727 variationType_[d] = variationType[d];
732 template<
size_t rank,
class ...ViewProperties>
733 Data(Kokkos::View<DataScalar***,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType,
const int blockPlusDiagonalLastNonDiagonal = -1)
735 dataRank_(data.
rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(
blockPlusDiagonalLastNonDiagonal), rank_(
rank)
738 for (
unsigned d=0; d<
rank; d++)
740 extents_[d] = extents[d];
741 variationType_[d] = variationType[d];
746 template<
size_t rank,
class ...ViewProperties>
747 Data(Kokkos::View<DataScalar****,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType,
const int blockPlusDiagonalLastNonDiagonal = -1)
749 dataRank_(data.
rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(
blockPlusDiagonalLastNonDiagonal), rank_(
rank)
752 for (
unsigned d=0; d<
rank; d++)
754 extents_[d] = extents[d];
755 variationType_[d] = variationType[d];
760 template<
size_t rank,
class ...ViewProperties>
761 Data(Kokkos::View<DataScalar*****,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType,
const int blockPlusDiagonalLastNonDiagonal = -1)
763 dataRank_(data.
rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(
blockPlusDiagonalLastNonDiagonal), rank_(
rank)
766 for (
unsigned d=0; d<
rank; d++)
768 extents_[d] = extents[d];
769 variationType_[d] = variationType[d];
774 template<
size_t rank,
class ...ViewProperties>
775 Data(Kokkos::View<DataScalar******,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType,
const int blockPlusDiagonalLastNonDiagonal = -1)
777 dataRank_(data.
rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(
blockPlusDiagonalLastNonDiagonal), rank_(
rank)
780 for (
unsigned d=0; d<
rank; d++)
782 extents_[d] = extents[d];
783 variationType_[d] = variationType[d];
788 template<
size_t rank,
class ...ViewProperties>
789 Data(Kokkos::View<DataScalar*******,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType,
const int blockPlusDiagonalLastNonDiagonal = -1)
791 dataRank_(data.
rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(
blockPlusDiagonalLastNonDiagonal), rank_(
rank)
794 for (
unsigned d=0; d<
rank; d++)
796 extents_[d] = extents[d];
797 variationType_[d] = variationType[d];
803 template<
class ViewScalar,
class ...ViewProperties>
804 Data(
const unsigned rank, Kokkos::View<ViewScalar,DeviceType, ViewProperties...> data, Kokkos::Array<int,7> extents, Kokkos::Array<DataVariationType,7> variationType,
const int blockPlusDiagonalLastNonDiagonal = -1)
806 dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(
blockPlusDiagonalLastNonDiagonal), rank_(
rank)
808 setUnderlyingView<data.rank>(data);
809 for (
unsigned d=0; d<
rank; d++)
811 extents_[d] = extents[d];
812 variationType_[d] = variationType[d];
818 template<
size_t rank>
819 Data(DataScalar constantValue, Kokkos::Array<int,rank> extents)
821 dataRank_(1), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(-1), rank_(
rank)
823 data1_ = Kokkos::View<DataScalar*,DeviceType>(
"Constant Data",1);
824 Kokkos::deep_copy(data1_, constantValue);
825 for (
unsigned d=0; d<
rank; d++)
827 extents_[d] = extents[d];
835 dataRank_(0), extents_({0,0,0,0,0,0,0}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(-1), rank_(0)
841 KOKKOS_INLINE_FUNCTION
844 return blockPlusDiagonalLastNonDiagonal_;
848 KOKKOS_INLINE_FUNCTION
855 KOKKOS_INLINE_FUNCTION
861 dimInfo.variationType = variationType_[dim];
863 dimInfo.variationModulus = variationModulus_[dim];
865 if (dimInfo.variationType == BLOCK_PLUS_DIAGONAL)
867 dimInfo.blockPlusDiagonalLastNonDiagonal = blockPlusDiagonalLastNonDiagonal_;
873 KOKKOS_INLINE_FUNCTION
879 return combinedDimensionInfo(myDimInfo, otherDimInfo);
884 KOKKOS_INLINE_FUNCTION
885 enable_if_t<rank==1, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
888 #ifdef HAVE_INTREPID2_DEBUG
896 KOKKOS_INLINE_FUNCTION
897 enable_if_t<rank==2, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
900 #ifdef HAVE_INTREPID2_DEBUG
908 KOKKOS_INLINE_FUNCTION
909 enable_if_t<rank==3, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
912 #ifdef HAVE_INTREPID2_DEBUG
920 KOKKOS_INLINE_FUNCTION
921 enable_if_t<rank==4, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
924 #ifdef HAVE_INTREPID2_DEBUG
932 KOKKOS_INLINE_FUNCTION
933 enable_if_t<rank==5, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
936 #ifdef HAVE_INTREPID2_DEBUG
944 KOKKOS_INLINE_FUNCTION
945 enable_if_t<rank==6, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
948 #ifdef HAVE_INTREPID2_DEBUG
956 KOKKOS_INLINE_FUNCTION
957 enable_if_t<rank==7, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
960 #ifdef HAVE_INTREPID2_DEBUG
967 KOKKOS_INLINE_FUNCTION
970 return getUnderlyingView<1>();
974 KOKKOS_INLINE_FUNCTION
977 return getUnderlyingView<2>();
981 KOKKOS_INLINE_FUNCTION
984 return getUnderlyingView<3>();
988 KOKKOS_INLINE_FUNCTION
991 return getUnderlyingView<4>();
995 KOKKOS_INLINE_FUNCTION
998 return getUnderlyingView<5>();
1002 KOKKOS_INLINE_FUNCTION
1005 return getUnderlyingView<6>();
1009 KOKKOS_INLINE_FUNCTION
1012 return getUnderlyingView<7>();
1016 KOKKOS_INLINE_FUNCTION
1023 KOKKOS_INLINE_FUNCTION
1030 KOKKOS_INLINE_FUNCTION
1037 KOKKOS_INLINE_FUNCTION
1044 KOKKOS_INLINE_FUNCTION
1051 KOKKOS_INLINE_FUNCTION
1058 KOKKOS_INLINE_FUNCTION
1064 template<
int underlyingRank,
class ViewScalar>
1065 KOKKOS_INLINE_FUNCTION
1066 void setUnderlyingView(
const Kokkos::View<ViewScalar, DeviceType> & view)
1068 if constexpr (underlyingRank == 1)
1072 else if constexpr (underlyingRank == 2)
1076 else if constexpr (underlyingRank == 3)
1080 else if constexpr (underlyingRank == 4)
1084 else if constexpr (underlyingRank == 5)
1088 else if constexpr (underlyingRank == 6)
1092 else if constexpr (underlyingRank == 7)
1107 case 1:
return data1_;
1108 case 2:
return data2_;
1109 case 3:
return data3_;
1110 case 4:
return data4_;
1111 case 5:
return data5_;
1112 case 6:
return data6_;
1113 case 7:
return data7_;
1114 default: INTREPID2_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Invalid data rank");
1119 KOKKOS_INLINE_FUNCTION
1126 KOKKOS_INLINE_FUNCTION
1129 ordinal_type size = 1;
1130 for (ordinal_type r=0; r<dataRank_; r++)
1142 case 1:
return getMatchingViewWithLabel(data1_,
"Intrepid2 Data", data1_.extent_int(0));
1143 case 2:
return getMatchingViewWithLabel(data2_,
"Intrepid2 Data", data2_.extent_int(0), data2_.extent_int(1));
1144 case 3:
return getMatchingViewWithLabel(data3_,
"Intrepid2 Data", data3_.extent_int(0), data3_.extent_int(1), data3_.extent_int(2));
1145 case 4:
return getMatchingViewWithLabel(data4_,
"Intrepid2 Data", data4_.extent_int(0), data4_.extent_int(1), data4_.extent_int(2), data4_.extent_int(3));
1146 case 5:
return getMatchingViewWithLabel(data5_,
"Intrepid2 Data", data5_.extent_int(0), data5_.extent_int(1), data5_.extent_int(2), data5_.extent_int(3), data5_.extent_int(4));
1147 case 6:
return getMatchingViewWithLabel(data6_,
"Intrepid2 Data", data6_.extent_int(0), data6_.extent_int(1), data6_.extent_int(2), data6_.extent_int(3), data6_.extent_int(4), data6_.extent_int(5));
1148 case 7:
return getMatchingViewWithLabel(data7_,
"Intrepid2 Data", data7_.extent_int(0), data7_.extent_int(1), data7_.extent_int(2), data7_.extent_int(3), data7_.extent_int(4), data7_.extent_int(5), data7_.extent_int(6));
1149 default: INTREPID2_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Invalid data rank");
1154 template<
class ... DimArgs>
1159 case 1:
return getMatchingViewWithLabel(data1_,
"Intrepid2 Data", dims...);
1160 case 2:
return getMatchingViewWithLabel(data2_,
"Intrepid2 Data", dims...);
1161 case 3:
return getMatchingViewWithLabel(data3_,
"Intrepid2 Data", dims...);
1162 case 4:
return getMatchingViewWithLabel(data4_,
"Intrepid2 Data", dims...);
1163 case 5:
return getMatchingViewWithLabel(data5_,
"Intrepid2 Data", dims...);
1164 case 6:
return getMatchingViewWithLabel(data6_,
"Intrepid2 Data", dims...);
1165 case 7:
return getMatchingViewWithLabel(data7_,
"Intrepid2 Data", dims...);
1166 default: INTREPID2_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Invalid data rank");
1175 case 1: Kokkos::deep_copy(data1_, 0.0);
break;
1176 case 2: Kokkos::deep_copy(data2_, 0.0);
break;
1177 case 3: Kokkos::deep_copy(data3_, 0.0);
break;
1178 case 4: Kokkos::deep_copy(data4_, 0.0);
break;
1179 case 5: Kokkos::deep_copy(data5_, 0.0);
break;
1180 case 6: Kokkos::deep_copy(data6_, 0.0);
break;
1181 case 7: Kokkos::deep_copy(data7_, 0.0);
break;
1182 default: INTREPID2_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Invalid data rank");
1199 default: INTREPID2_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Invalid data rank");
1206 for (
int i=0; i<numActiveDims_; i++)
1208 if (activeDims_[i] == d)
1212 else if (activeDims_[i] > d)
1231 KOKKOS_INLINE_FUNCTION
1234 return variationModulus_[d];
1238 KOKKOS_INLINE_FUNCTION
1241 return variationType_;
1245 template<
class ...IntArgs>
1246 KOKKOS_INLINE_FUNCTION
1253 template<
class ...IntArgs>
1254 KOKKOS_INLINE_FUNCTION
1262 template <
bool... v>
1265 template <
class ...IntArgs>
1266 using valid_args = all_true<std::is_integral<IntArgs>{}...>;
1268 static_assert(valid_args<int,long,unsigned>::value,
"valid args works");
1271 template <
class ...IntArgs>
1272 KOKKOS_INLINE_FUNCTION
1273 #ifndef __INTEL_COMPILER
1277 enable_if_t<valid_args<IntArgs...>::value && (
sizeof...(IntArgs) <= 7),return_type>
1286 KOKKOS_INLINE_FUNCTION
1292 template <
typename iType>
1293 KOKKOS_INLINE_FUNCTION constexpr
1294 typename std::enable_if<std::is_integral<iType>::value,
size_t>::type
1295 extent(
const iType& r)
const {
1302 if (blockPlusDiagonalLastNonDiagonal_ >= 1)
return false;
1303 int numBlockPlusDiagonalTypes = 0;
1304 for (
unsigned r = 0; r<variationType_.size(); r++)
1306 const auto &entryType = variationType_[r];
1307 if (entryType == BLOCK_PLUS_DIAGONAL) numBlockPlusDiagonalTypes++;
1310 if (numBlockPlusDiagonalTypes == 2)
return true;
1311 else if (numBlockPlusDiagonalTypes == 0)
return false;
1333 std::vector<DimensionInfo> dimInfo(rank);
1334 for (
int d=0; d<
rank; d++)
1354 const int D1_DIM = A_MatData.
rank() - 2;
1355 const int D2_DIM = A_MatData.
rank() - 1;
1357 const int A_rows = A_MatData.
extent_int(D1_DIM);
1358 const int A_cols = A_MatData.
extent_int(D2_DIM);
1359 const int B_rows = B_MatData.
extent_int(D1_DIM);
1360 const int B_cols = B_MatData.
extent_int(D2_DIM);
1362 const int leftRows = transposeA ? A_cols : A_rows;
1363 const int leftCols = transposeA ? A_rows : A_cols;
1364 const int rightRows = transposeB ? B_cols : B_rows;
1365 const int rightCols = transposeB ? B_rows : B_cols;
1367 INTREPID2_TEST_FOR_EXCEPTION(leftCols != rightRows, std::invalid_argument,
"incompatible matrix dimensions");
1369 Kokkos::Array<int,7> resultExtents;
1370 Kokkos::Array<DataVariationType,7> resultVariationTypes;
1372 resultExtents[D1_DIM] = leftRows;
1373 resultExtents[D2_DIM] = rightCols;
1374 int resultBlockPlusDiagonalLastNonDiagonal = -1;
1378 resultVariationTypes[D1_DIM] = BLOCK_PLUS_DIAGONAL;
1379 resultVariationTypes[D2_DIM] = BLOCK_PLUS_DIAGONAL;
1383 const int resultRank = A_MatData.
rank();
1388 Kokkos::Array<int,7> resultActiveDims;
1389 Kokkos::Array<int,7> resultDataDims;
1390 int resultNumActiveDims = 0;
1392 for (
int i=0; i<resultRank-2; i++)
1398 const DataVariationType &A_VariationType = A_VariationTypes[i];
1399 const DataVariationType &B_VariationType = B_VariationTypes[i];
1406 DataVariationType resultVariationType;
1407 if ((A_VariationType == GENERAL) || (B_VariationType == GENERAL))
1409 resultVariationType = GENERAL;
1410 dataSize = resultExtents[i];
1412 else if ((B_VariationType == CONSTANT) && (A_VariationType == CONSTANT))
1414 resultVariationType = CONSTANT;
1417 else if ((B_VariationType == MODULAR) && (A_VariationType == CONSTANT))
1419 resultVariationType = MODULAR;
1422 else if ((B_VariationType == CONSTANT) && (A_VariationType == MODULAR))
1424 resultVariationType = MODULAR;
1433 resultVariationType = MODULAR;
1434 dataSize = A_Modulus;
1436 resultVariationTypes[i] = resultVariationType;
1438 if (resultVariationType != CONSTANT)
1440 resultActiveDims[resultNumActiveDims] = i;
1441 resultDataDims[resultNumActiveDims] = dataSize;
1442 resultNumActiveDims++;
1447 resultExtents[D1_DIM] = leftRows;
1448 resultExtents[D2_DIM] = rightCols;
1453 resultVariationTypes[D1_DIM] = BLOCK_PLUS_DIAGONAL;
1454 resultVariationTypes[D2_DIM] = BLOCK_PLUS_DIAGONAL;
1457 resultActiveDims[resultNumActiveDims] = resultRank - 2;
1459 const int numDiagonalEntries = leftRows - (resultBlockPlusDiagonalLastNonDiagonal + 1);
1460 const int numNondiagonalEntries = (resultBlockPlusDiagonalLastNonDiagonal + 1) * (resultBlockPlusDiagonalLastNonDiagonal + 1);
1462 resultDataDims[resultNumActiveDims] = numDiagonalEntries + numNondiagonalEntries;
1463 resultNumActiveDims++;
1468 resultVariationTypes[D1_DIM] = GENERAL;
1469 resultVariationTypes[D2_DIM] = GENERAL;
1471 resultActiveDims[resultNumActiveDims] = resultRank - 2;
1472 resultActiveDims[resultNumActiveDims+1] = resultRank - 1;
1474 resultDataDims[resultNumActiveDims] = leftRows;
1475 resultDataDims[resultNumActiveDims+1] = rightCols;
1476 resultNumActiveDims += 2;
1479 for (
int i=resultRank; i<7; i++)
1481 resultVariationTypes[i] = CONSTANT;
1482 resultExtents[i] = 1;
1485 ScalarView<DataScalar,DeviceType> data;
1486 if (resultNumActiveDims == 1)
1489 data = getMatchingViewWithLabel(viewToMatch,
"Data mat-mat result", resultDataDims[0]);
1491 else if (resultNumActiveDims == 2)
1494 data = getMatchingViewWithLabel(viewToMatch,
"Data mat-mat result", resultDataDims[0], resultDataDims[1]);
1496 else if (resultNumActiveDims == 3)
1499 data = getMatchingViewWithLabel(viewToMatch,
"Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2]);
1501 else if (resultNumActiveDims == 4)
1504 data = getMatchingViewWithLabel(viewToMatch,
"Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2],
1507 else if (resultNumActiveDims == 5)
1510 data = getMatchingViewWithLabel(viewToMatch,
"Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2],
1511 resultDataDims[3], resultDataDims[4]);
1513 else if (resultNumActiveDims == 6)
1516 data = getMatchingViewWithLabel(viewToMatch,
"Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2],
1517 resultDataDims[3], resultDataDims[4], resultDataDims[5]);
1522 data = getMatchingViewWithLabel(viewToMatch,
"Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2],
1523 resultDataDims[3], resultDataDims[4], resultDataDims[5], resultDataDims[6]);
1539 const int resultRank = vecData.
rank();
1543 Kokkos::Array<int,7> resultExtents;
1544 Kokkos::Array<DataVariationType,7> resultVariationTypes;
1548 Kokkos::Array<int,7> resultActiveDims;
1549 Kokkos::Array<int,7> resultDataDims;
1550 int resultNumActiveDims = 0;
1552 for (
int i=0; i<resultRank-1; i++)
1557 const DataVariationType &vecVariationType = vecVariationTypes[i];
1558 const DataVariationType &matVariationType = matVariationTypes[i];
1565 DataVariationType resultVariationType;
1566 if ((vecVariationType == GENERAL) || (matVariationType == GENERAL))
1568 resultVariationType = GENERAL;
1569 dataSize = resultExtents[i];
1571 else if ((matVariationType == CONSTANT) && (vecVariationType == CONSTANT))
1573 resultVariationType = CONSTANT;
1576 else if ((matVariationType == MODULAR) && (vecVariationType == CONSTANT))
1578 resultVariationType = MODULAR;
1581 else if ((matVariationType == CONSTANT) && (vecVariationType == MODULAR))
1583 resultVariationType = MODULAR;
1592 resultVariationType = MODULAR;
1593 dataSize = matModulus;
1595 resultVariationTypes[i] = resultVariationType;
1597 if (resultVariationType != CONSTANT)
1599 resultActiveDims[resultNumActiveDims] = i;
1600 resultDataDims[resultNumActiveDims] = dataSize;
1601 resultNumActiveDims++;
1606 resultVariationTypes[resultNumActiveDims] = GENERAL;
1607 resultActiveDims[resultNumActiveDims] = resultRank - 1;
1608 resultDataDims[resultNumActiveDims] = matRows;
1609 resultExtents[resultRank-1] = matRows;
1610 resultNumActiveDims++;
1612 for (
int i=resultRank; i<7; i++)
1614 resultVariationTypes[i] = CONSTANT;
1615 resultExtents[i] = 1;
1618 ScalarView<DataScalar,DeviceType> data;
1619 if (resultNumActiveDims == 1)
1623 else if (resultNumActiveDims == 2)
1627 else if (resultNumActiveDims == 3)
1631 else if (resultNumActiveDims == 4)
1636 else if (resultNumActiveDims == 5)
1639 resultDataDims[3], resultDataDims[4]);
1641 else if (resultNumActiveDims == 6)
1644 resultDataDims[3], resultDataDims[4], resultDataDims[5]);
1649 resultDataDims[3], resultDataDims[4], resultDataDims[5], resultDataDims[6]);
1657 enable_if_t<(rank!=1) && (rank!=7), Kokkos::MDRangePolicy<typename DeviceType::execution_space,Kokkos::Rank<rank>> >
1660 using ExecutionSpace =
typename DeviceType::execution_space;
1661 Kokkos::Array<int,rank> startingOrdinals;
1662 Kokkos::Array<int,rank> extents;
1664 for (
int d=0; d<
rank; d++)
1666 startingOrdinals[d] = 0;
1669 auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<rank>>(startingOrdinals,extents);
1675 enable_if_t<rank==7, Kokkos::MDRangePolicy<typename DeviceType::execution_space,Kokkos::Rank<6>> >
1678 using ExecutionSpace =
typename DeviceType::execution_space;
1679 Kokkos::Array<int,6> startingOrdinals;
1680 Kokkos::Array<int,6> extents;
1682 for (
int d=0; d<6; d++)
1684 startingOrdinals[d] = 0;
1687 auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<6>>(startingOrdinals,extents);
1693 enable_if_t<rank==1, Kokkos::RangePolicy<typename DeviceType::execution_space> >
1696 using ExecutionSpace =
typename DeviceType::execution_space;
1697 Kokkos::RangePolicy<ExecutionSpace> policy(ExecutionSpace(),0,
getDataExtent(0));
1702 Data shallowCopy(
const int rank,
const Kokkos::Array<int,7> &extents,
const Kokkos::Array<DataVariationType,7> &variationTypes)
const
1706 case 1:
return Data(rank, data1_, extents, variationTypes);
1707 case 2:
return Data(rank, data2_, extents, variationTypes);
1708 case 3:
return Data(rank, data3_, extents, variationTypes);
1709 case 4:
return Data(rank, data4_, extents, variationTypes);
1710 case 5:
return Data(rank, data5_, extents, variationTypes);
1711 case 6:
return Data(rank, data6_, extents, variationTypes);
1712 case 7:
return Data(rank, data7_, extents, variationTypes);
1714 INTREPID2_TEST_FOR_EXCEPTION(
true, std::invalid_argument,
"Unhandled dataRank_");
1719 template<
class BinaryOperator>
1762 using ExecutionSpace =
typename DeviceType::execution_space;
1768 Kokkos::parallel_for(
"compute mat-vec", policy,
1769 KOKKOS_LAMBDA (
const int &cellOrdinal,
const int &pointOrdinal,
const int &i) {
1772 for (
int j=0; j<matCols; j++)
1774 val_i += matData(cellOrdinal,pointOrdinal,i,j) * vecData(cellOrdinal,pointOrdinal,j);
1778 else if (rank_ == 2)
1781 auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<2>>({0,0},{
getDataExtent(0),matRows});
1782 Kokkos::parallel_for(
"compute mat-vec", policy,
1783 KOKKOS_LAMBDA (
const int &vectorOrdinal,
const int &i) {
1786 for (
int j=0; j<matCols; j++)
1788 val_i += matData(vectorOrdinal,i,j) * vecData(vectorOrdinal,j);
1792 else if (rank_ == 1)
1795 Kokkos::RangePolicy<ExecutionSpace> policy(0,matRows);
1796 Kokkos::parallel_for(
"compute mat-vec", policy,
1797 KOKKOS_LAMBDA (
const int &i) {
1800 for (
int j=0; j<matCols; j++)
1802 val_i += matData(i,j) * vecData(j);
1827 const int D1_DIM = A_MatData.
rank() - 2;
1828 const int D2_DIM = A_MatData.
rank() - 1;
1830 const int A_rows = A_MatData.
extent_int(D1_DIM);
1831 const int A_cols = A_MatData.
extent_int(D2_DIM);
1832 const int B_rows = B_MatData.
extent_int(D1_DIM);
1833 const int B_cols = B_MatData.
extent_int(D2_DIM);
1835 const int leftRows = transposeA ? A_cols : A_rows;
1836 const int leftCols = transposeA ? A_rows : A_cols;
1837 const int rightCols = transposeB ? B_rows : B_cols;
1839 #ifdef INTREPID2_HAVE_DEBUG
1840 const int rightRows = transposeB ? B_cols : B_rows;
1847 using ExecutionSpace =
typename DeviceType::execution_space;
1849 const int diagonalStart = (variationType_[D1_DIM] == BLOCK_PLUS_DIAGONAL) ? blockPlusDiagonalLastNonDiagonal_ + 1 : leftRows;
1854 auto policy = Kokkos::RangePolicy<ExecutionSpace>(0,
getDataExtent(0));
1855 Kokkos::parallel_for(
"compute mat-mat", policy,
1856 KOKKOS_LAMBDA (
const int &matrixOrdinal) {
1857 for (
int i=0; i<diagonalStart; i++)
1859 for (
int j=0; j<rightCols; j++)
1863 for (
int k=0; k<leftCols; k++)
1865 const auto & left = transposeA ? A_MatData(matrixOrdinal,k,i) : A_MatData(matrixOrdinal,i,k);
1866 const auto & right = transposeB ? B_MatData(matrixOrdinal,j,k) : B_MatData(matrixOrdinal,k,j);
1867 val_ij += left * right;
1871 for (
int i=diagonalStart; i<leftRows; i++)
1874 const auto & left = A_MatData(matrixOrdinal,i,i);
1875 const auto & right = B_MatData(matrixOrdinal,i,i);
1876 val_ii = left * right;
1880 else if (rank_ == 4)
1884 if (underlyingMatchesLogical_)
1886 Kokkos::parallel_for(
"compute mat-mat", policy,
1887 KOKKOS_LAMBDA (
const int &cellOrdinal,
const int &pointOrdinal) {
1888 for (
int i=0; i<leftCols; i++)
1890 for (
int j=0; j<rightCols; j++)
1894 for (
int k=0; k<leftCols; k++)
1896 const auto & left = transposeA ? A_MatData(cellOrdinal,pointOrdinal,k,i) : A_MatData(cellOrdinal,pointOrdinal,i,k);
1897 const auto & right = transposeB ? B_MatData(cellOrdinal,pointOrdinal,j,k) : B_MatData(cellOrdinal,pointOrdinal,k,j);
1898 val_ij += left * right;
1906 Kokkos::parallel_for(
"compute mat-mat", policy,
1907 KOKKOS_LAMBDA (
const int &cellOrdinal,
const int &pointOrdinal) {
1908 for (
int i=0; i<diagonalStart; i++)
1910 for (
int j=0; j<rightCols; j++)
1914 for (
int k=0; k<leftCols; k++)
1916 const auto & left = transposeA ? A_MatData(cellOrdinal,pointOrdinal,k,i) : A_MatData(cellOrdinal,pointOrdinal,i,k);
1917 const auto & right = transposeB ? B_MatData(cellOrdinal,pointOrdinal,j,k) : B_MatData(cellOrdinal,pointOrdinal,k,j);
1918 val_ij += left * right;
1922 for (
int i=diagonalStart; i<leftRows; i++)
1925 const auto & left = A_MatData(cellOrdinal,pointOrdinal,i,i);
1926 const auto & right = B_MatData(cellOrdinal,pointOrdinal,i,i);
1927 val_ii = left * right;
1940 KOKKOS_INLINE_FUNCTION constexpr
bool isValid()
const
1942 return extents_[0] > 0;
1946 KOKKOS_INLINE_FUNCTION
1958 void setExtent(
const ordinal_type &d,
const ordinal_type &newExtent)
1960 INTREPID2_TEST_FOR_EXCEPTION(variationType_[d] == BLOCK_PLUS_DIAGONAL, std::invalid_argument,
"setExtent is not supported for BLOCK_PLUS_DIAGONAL dimensions");
1962 if (variationType_[d] == MODULAR)
1964 bool dividesEvenly = ((newExtent / variationModulus_[d]) * variationModulus_[d] == newExtent);
1965 INTREPID2_TEST_FOR_EXCEPTION(!dividesEvenly, std::invalid_argument,
"when setExtent is called on dimenisions with MODULAR variation, the modulus must divide the new extent evenly");
1968 if ((newExtent != extents_[d]) && (variationType_[d] == GENERAL))
1971 std::vector<ordinal_type> newExtents(dataRank_,-1);
1972 for (
int r=0; r<dataRank_; r++)
1974 if (activeDims_[r] == d)
1977 newExtents[r] = newExtent;
1988 case 1: Kokkos::resize(data1_,newExtents[0]);
1990 case 2: Kokkos::resize(data2_,newExtents[0],newExtents[1]);
1992 case 3: Kokkos::resize(data3_,newExtents[0],newExtents[1],newExtents[2]);
1994 case 4: Kokkos::resize(data4_,newExtents[0],newExtents[1],newExtents[2],newExtents[3]);
1996 case 5: Kokkos::resize(data5_,newExtents[0],newExtents[1],newExtents[2],newExtents[3],newExtents[4]);
1998 case 6: Kokkos::resize(data6_,newExtents[0],newExtents[1],newExtents[2],newExtents[3],newExtents[4],newExtents[5]);
2000 case 7: Kokkos::resize(data7_,newExtents[0],newExtents[1],newExtents[2],newExtents[3],newExtents[4],newExtents[5],newExtents[6]);
2002 default: INTREPID2_TEST_FOR_EXCEPTION(
true, std::logic_error,
"Unexpected dataRank_ value");
2006 extents_[d] = newExtent;
2010 KOKKOS_INLINE_FUNCTION
2013 return underlyingMatchesLogical_;
2017 template<
class DataScalar,
typename DeviceType>
2018 KOKKOS_INLINE_FUNCTION constexpr
unsigned rank(
const Data<DataScalar, DeviceType>& D) {
enable_if_t< rank==7, Kokkos::MDRangePolicy< typename DeviceType::execution_space, Kokkos::Rank< 6 > > > dataExtentRangePolicy()
returns an MDRangePolicy over the first six underlying data extents (but with the logical shape)...
void clear() const
Copies 0.0 to the underlying View.
void copyDataFromDynRankViewMatchingUnderlying(const ScalarView< DataScalar, DeviceType > &dynRankView) const
Copies from the provided DynRankView into the underlying Kokkos::View container storing the unique da...
KOKKOS_INLINE_FUNCTION reference_type getWritableEntry(const IntArgs...intArgs) const
Returns an l-value reference to the specified logical entry in the underlying view. Note that for variation types other than GENERAL, multiple valid argument sets will refer to the same memory location. Intended for Intrepid2 developers and expert users only. If passThroughBlockDiagonalArgs is TRUE, the corresponding arguments are interpreted as entries in the 1D packed matrix rather than as logical 2D matrix row and column.
KOKKOS_INLINE_FUNCTION ordinal_type getUnderlyingViewRank() const
returns the rank of the View that stores the unique data
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar *******, DeviceType > & getUnderlyingView7() const
returns the View that stores the unique data. For rank-7 underlying containers.
ScalarView< DataScalar, DeviceType > allocateDynRankViewMatchingUnderlying(DimArgs...dims) const
Returns a DynRankView that matches the underlying Kokkos::View object value_type and layout...
void applyOperator(UnaryOperator unaryOperator)
applies the specified unary operator to each entry
static KOKKOS_INLINE_FUNCTION int blockPlusDiagonalDiagonalEntryIndex(const int &lastNondiagonal, const int &numNondiagonalEntries, const int &i)
Returns flattened index of the specified (i,i) matrix entry, assuming that i > lastNondiagonal. Only applicable for BLOCK_PLUS_DIAGONAL DataVariationType.
static KOKKOS_INLINE_FUNCTION int blockPlusDiagonalBlockEntryIndex(const int &lastNondiagonal, const int &numNondiagonalEntries, const int &i, const int &j)
//! Returns flattened index of the specified (i,j) matrix entry, assuming that i,j ≤ lastNondiagonal...
static Data< DataScalar, DeviceType > allocateInPlaceCombinationResult(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
Data(const unsigned rank, Kokkos::View< ViewScalar, DeviceType, ViewProperties...> data, Kokkos::Array< int, 7 > extents, Kokkos::Array< DataVariationType, 7 > variationType, const int blockPlusDiagonalLastNonDiagonal=-1)
constructor with run-time rank (requires full-length extents, variationType inputs; those beyond the ...
KOKKOS_INLINE_FUNCTION enable_if_t< rank==6, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 6...
KOKKOS_INLINE_FUNCTION enable_if_t< rank==5, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 5...
static void copyContainer(ToContainer to, FromContainer from)
Generic data copying method to allow construction of Data object from DynRankViews for which deep_cop...
KOKKOS_INLINE_FUNCTION void setUnderlyingView3(const Kokkos::View< DataScalar ***, DeviceType > &view)
sets the View that stores the unique data. For rank-3 underlying containers.
#define INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(test, x, msg)
KOKKOS_INLINE_FUNCTION int getVariationModulus(const int &d) const
Variation modulus accessor.
KOKKOS_INLINE_FUNCTION enable_if_t< rank==7, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 7...
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar *****, DeviceType > & getUnderlyingView5() const
returns the View that stores the unique data. For rank-5 underlying containers.
KOKKOS_INLINE_FUNCTION DimensionInfo combinedDataDimensionInfo(const Data &otherData, const int &dim) const
Returns (DataVariationType, data extent) in the specified dimension for a Data container that combine...
ScalarView< DataScalar, DeviceType > getUnderlyingView() const
Returns a DynRankView constructed atop the same underlying data as the fixed-rank Kokkos::View used i...
KOKKOS_INLINE_FUNCTION Kokkos::Array< int, 7 > getExtents() const
Returns an array containing the logical extents in each dimension.
void storeMatVec(const Data< DataScalar, DeviceType > &matData, const Data< DataScalar, DeviceType > &vecData)
Places the result of a matrix-vector multiply corresponding to the two provided containers into this ...
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar *, DeviceType > & getUnderlyingView1() const
returns the View that stores the unique data. For rank-1 underlying containers.
KOKKOS_INLINE_FUNCTION const Kokkos::Array< DataVariationType, 7 > & getVariationTypes() const
Returns an array with the variation types in each logical dimension.
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar ******, DeviceType > & getUnderlyingView6() const
returns the View that stores the unique data. For rank-6 underlying containers.
KOKKOS_INLINE_FUNCTION enable_if_t< rank==1, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 1...
Defines implementations for the Data class that are not present in the declaration.
KOKKOS_INLINE_FUNCTION bool underlyingMatchesLogical() const
Returns true if the underlying container has exactly the same rank and extents as the logical contain...
Wrapper around a Kokkos::View that allows data that is constant or repeating in various logical dimen...
void storeInPlaceCombination(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B, BinaryOperator binaryOperator)
Places the result of an in-place combination (e.g., entrywise sum) into this data container...
Header function for Intrepid2::Util class and other utility functions.
KOKKOS_INLINE_FUNCTION bool isDiagonal() const
returns true for containers that have two dimensions marked as BLOCK_PLUS_DIAGONAL for which the non-...
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar ****, DeviceType > & getUnderlyingView4() const
returns the View that stores the unique data. For rank-4 underlying containers.
KOKKOS_INLINE_FUNCTION void setUnderlyingView5(const Kokkos::View< DataScalar *****, DeviceType > &view)
sets the View that stores the unique data. For rank-5 underlying containers.
Data(std::vector< DimensionInfo > dimInfoVector)
Constructor in terms of DimensionInfo for each logical dimension; does not require a View to be speci...
Struct expressing all variation information about a Data object in a single dimension, including its logical extent and storage extent.
KOKKOS_INLINE_FUNCTION return_type getEntryWithPassThroughOption(const bool &passThroughBlockDiagonalArgs, const IntArgs &...intArgs) const
Returns a (read-only) value corresponding to the specified logical data location. If passThroughBlock...
Data shallowCopy(const int rank, const Kokkos::Array< int, 7 > &extents, const Kokkos::Array< DataVariationType, 7 > &variationTypes) const
Creates a new Data object with the same underlying view, but with the specified logical rank...
void storeInPlaceDifference(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
stores the in-place (entrywise) difference, A .- B, into this container.
Defines functors for use with Data objects: so far, we include simple arithmetical functors for sum...
KOKKOS_INLINE_FUNCTION DimensionInfo getDimensionInfo(const int &dim) const
Returns an object fully specifying the indicated dimension. This is used in determining appropriate s...
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar ***, DeviceType > & getUnderlyingView3() const
returns the View that stores the unique data. For rank-3 underlying containers.
Data(DataScalar constantValue, Kokkos::Array< int, rank > extents)
constructor for everywhere-constant data
enable_if_t<(rank!=1)&&(rank!=7), Kokkos::MDRangePolicy< typename DeviceType::execution_space, Kokkos::Rank< rank > > > dataExtentRangePolicy()
returns an MDRangePolicy over the underlying data extents (but with the logical shape).
KOKKOS_INLINE_FUNCTION reference_type getWritableEntryWithPassThroughOption(const bool &passThroughBlockDiagonalArgs, const IntArgs...intArgs) const
Returns an l-value reference to the specified logical entry in the underlying view. Note that for variation types other than GENERAL, multiple valid argument sets will refer to the same memory location. Intended for Intrepid2 developers and expert users only. If passThroughBlockDiagonalArgs is TRUE, the corresponding arguments are interpreted as entries in the 1D packed matrix rather than as logical 2D matrix row and column.
Data< DataScalar, DeviceType > allocateConstantData(const DataScalar &value)
KOKKOS_INLINE_FUNCTION constexpr bool isValid() const
returns true for containers that have data; false for those that don't (namely, those that have been ...
KOKKOS_INLINE_FUNCTION enable_if_t< valid_args< IntArgs...>::value &&(sizeof...(IntArgs)<=7), return_type > operator()(const IntArgs &...intArgs) const
Returns a value corresponding to the specified logical data location.
KOKKOS_INLINE_FUNCTION void setUnderlyingView6(const Kokkos::View< DataScalar ******, DeviceType > &view)
sets the View that stores the unique data. For rank-6 underlying containers.
KOKKOS_INLINE_FUNCTION void setUnderlyingView2(const Kokkos::View< DataScalar **, DeviceType > &view)
sets the View that stores the unique data. For rank-2 underlying containers.
static Data< DataScalar, DeviceType > allocateMatVecResult(const Data< DataScalar, DeviceType > &matData, const Data< DataScalar, DeviceType > &vecData)
ScalarView< DataScalar, DeviceType > allocateDynRankViewMatchingUnderlying() const
Returns a DynRankView that matches the underlying Kokkos::View object in value_type, layout, and dimension.
void storeInPlaceQuotient(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
stores the in-place (entrywise) quotient, A ./ B, into this container.
void storeInPlaceSum(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
stores the in-place (entrywise) sum, A .+ B, into this container.
Data(const ScalarView< DataScalar, DeviceType > &data, int rank, Kokkos::Array< int, 7 > extents, Kokkos::Array< DataVariationType, 7 > variationType, const int blockPlusDiagonalLastNonDiagonal=-1)
DynRankView constructor. Will copy to a View of appropriate rank.
void storeMatMat(const bool transposeA, const Data< DataScalar, DeviceType > &A_MatData, const bool transposeB, const Data< DataScalar, DeviceType > &B_MatData)
KOKKOS_INLINE_FUNCTION ordinal_type getUnderlyingViewSize() const
returns the number of entries in the View that stores the unique data
void allocateAndCopyFromDynRankView(ScalarView< DataScalar, DeviceType > data)
allocate an underlying View that matches the provided DynRankView in dimensions, and copy...
static KOKKOS_INLINE_FUNCTION int blockPlusDiagonalNumNondiagonalEntries(const int &lastNondiagonal)
Returns the number of non-diagonal entries based on the last non-diagonal. Only applicable for BLOCK_...
KOKKOS_INLINE_FUNCTION enable_if_t< rank==3, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 3...
void setActiveDims()
class initialization method. Called by constructors.
Data()
default constructor (empty data)
void setExtent(const ordinal_type &d, const ordinal_type &newExtent)
sets the logical extent in the specified dimension. If needed, the underlying data container is resiz...
A singleton class for a DynRankView containing exactly one zero entry. (Technically, the entry is DataScalar(), the default value for the scalar type.) This allows View-wrapping classes to return a reference to zero, even when that zero is not explicitly stored in the wrapped views.
void storeInPlaceProduct(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
stores the in-place (entrywise) product, A .* B, into this container.
KOKKOS_INLINE_FUNCTION enable_if_t< rank==4, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 4...
KOKKOS_INLINE_FUNCTION void setUnderlyingView1(const Kokkos::View< DataScalar *, DeviceType > &view)
sets the View that stores the unique data. For rank-1 underlying containers.
KOKKOS_INLINE_FUNCTION unsigned rank() const
Returns the logical rank of the Data container.
KOKKOS_INLINE_FUNCTION int extent_int(const int &r) const
Returns the logical extent in the specified dimension.
Defines DimensionInfo struct that allows specification of a dimension within a Data object...
Defines DataVariationType enum that specifies the types of variation possible within a Data object...
KOKKOS_INLINE_FUNCTION enable_if_t< rank==2, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 2...
KOKKOS_INLINE_FUNCTION int getDataExtent(const ordinal_type &d) const
returns the true extent of the data corresponding to the logical dimension provided; if the data does...
KOKKOS_INLINE_FUNCTION int getUnderlyingViewExtent(const int &dim) const
Returns the extent of the underlying view in the specified dimension.
Data(ScalarView< DataScalar, DeviceType > data)
copy constructor modeled after the copy-like constructor above. Not as efficient as the implicit copy...
KOKKOS_INLINE_FUNCTION void setUnderlyingView4(const Kokkos::View< DataScalar ****, DeviceType > &view)
sets the View that stores the unique data. For rank-4 underlying containers.
KOKKOS_INLINE_FUNCTION const int & blockPlusDiagonalLastNonDiagonal() const
For a Data object containing data with variation type BLOCK_PLUS_DIAGONAL, returns the row and column...
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar **, DeviceType > & getUnderlyingView2() const
returns the View that stores the unique data. For rank-2 underlying containers.
KOKKOS_INLINE_FUNCTION void setUnderlyingView7(const Kokkos::View< DataScalar *******, DeviceType > &view)
sets the View that stores the unique data. For rank-7 underlying containers.
KOKKOS_INLINE_FUNCTION return_type getEntry(const IntArgs &...intArgs) const
Returns a (read-only) value corresponding to the specified logical data location. ...
static Data< DataScalar, DeviceType > allocateMatMatResult(const bool transposeA, const Data< DataScalar, DeviceType > &A_MatData, const bool transposeB, const Data< DataScalar, DeviceType > &B_MatData)
Data(const Data< DataScalar, OtherDeviceType > &data)
copy-like constructor for differing execution spaces. This does a deep_copy of the underlying view...