Intrepid2
Intrepid2_Data.hpp
Go to the documentation of this file.
1 // @HEADER
2 // *****************************************************************************
3 // Intrepid2 Package
4 //
5 // Copyright 2007 NTESS and the Intrepid2 contributors.
6 // SPDX-License-Identifier: BSD-3-Clause
7 // *****************************************************************************
8 // @HEADER
9 //
10 // Intrepid2_Data.hpp
11 // QuadraturePerformance
12 //
13 // Created by Roberts, Nathan V on 8/24/20.
14 //
15 
16 #ifndef Intrepid2_Data_h
17 #define Intrepid2_Data_h
18 
23 #include "Intrepid2_ScalarView.hpp"
24 #include "Intrepid2_Utils.hpp"
25 
26 #include <variant>
27 
33 namespace Intrepid2 {
34 
43 template<class DataScalar,typename DeviceType>
44 class ZeroView {
45 public:
46  static ScalarView<DataScalar,DeviceType> zeroView()
47  {
48  static ScalarView<DataScalar,DeviceType> zeroView = ScalarView<DataScalar,DeviceType>("zero",1);
49  static bool havePushedFinalizeHook = false;
50  if (!havePushedFinalizeHook)
51  {
52  Kokkos::push_finalize_hook( [=] {
53  zeroView = ScalarView<DataScalar,DeviceType>();
54  });
55  havePushedFinalizeHook = true;
56  }
57  return zeroView;
58  }
59 };
60 
78  template<class DataScalar,typename DeviceType>
79  class Data {
80  public:
81  using value_type = DataScalar;
82  using execution_space = typename DeviceType::execution_space;
83 
84  using reference_type = typename ScalarView<DataScalar,DeviceType>::reference_type;
85  using const_reference_type = typename ScalarView<const DataScalar,DeviceType>::reference_type;
86  private:
87  ordinal_type dataRank_;
88  using View1 = Kokkos::View<DataScalar*, DeviceType>;
89  using View2 = Kokkos::View<DataScalar**, DeviceType>;
90  using View3 = Kokkos::View<DataScalar***, DeviceType>;
91  using View4 = Kokkos::View<DataScalar****, DeviceType>;
92  using View5 = Kokkos::View<DataScalar*****, DeviceType>;
93  using View6 = Kokkos::View<DataScalar******, DeviceType>;
94  using View7 = Kokkos::View<DataScalar*******, DeviceType>;
95  std::variant<View1,View2,View3,View4,View5,View6,View7> underlyingView_;
96  Kokkos::Array<int,7> extents_; // logical extents in each dimension
97  Kokkos::Array<DataVariationType,7> variationType_; // for each dimension, whether the data varies in that dimension
98  Kokkos::Array<int,7> variationModulus_; // for each dimension, a value by which indices should be modulused (only used when variationType_ is MODULAR)
99  int blockPlusDiagonalLastNonDiagonal_ = -1; // last row/column that is part of the non-diagonal part of the matrix indicated by BLOCK_PLUS_DIAGONAL (if any dimensions are thus marked)
100 
101  bool underlyingMatchesLogical_; // if true, this Data object has the same rank and extent as the underlying view
102  Kokkos::Array<ordinal_type,7> activeDims_;
103  int numActiveDims_; // how many of the 7 entries are actually filled in
104 
105  ordinal_type rank_;
106 
107  // we use (const_)reference_type as the return for operator() for performance reasons, especially significant when using Sacado types
108  using return_type = const_reference_type;
109 
110  ScalarView<DataScalar,DeviceType> zeroView_; // one-entry (zero); used to allow getEntry() to return 0 for off-diagonal entries in BLOCK_PLUS_DIAGONAL
111 
113  KOKKOS_INLINE_FUNCTION
114  static int blockPlusDiagonalNumNondiagonalEntries(const int &lastNondiagonal)
115  {
116  return (lastNondiagonal + 1) * (lastNondiagonal + 1);
117  }
118 
120  KOKKOS_INLINE_FUNCTION
121  static int blockPlusDiagonalBlockEntryIndex(const int &lastNondiagonal, const int &numNondiagonalEntries, const int &i, const int &j)
122  {
123  return i * (lastNondiagonal + 1) + j;
124  }
125 
127  KOKKOS_INLINE_FUNCTION
128  static int blockPlusDiagonalDiagonalEntryIndex(const int &lastNondiagonal, const int &numNondiagonalEntries, const int &i)
129  {
130  return i - (lastNondiagonal + 1) + numNondiagonalEntries;
131  }
132 
133  template <class T>
134  KOKKOS_INLINE_FUNCTION const T& get_fixed_view(const std::variant<View1,View2,View3,View4,View5,View6,View7> & v) const {
135  return *reinterpret_cast<const T*>(&v);
136  }
137 
139  KOKKOS_INLINE_FUNCTION
140  int getUnderlyingViewExtent(const int &dim) const
141  {
142  switch (dataRank_)
143  {
144  case 1: return get_fixed_view<View1>(underlyingView_).extent_int(dim);
145  case 2: return get_fixed_view<View2>(underlyingView_).extent_int(dim);
146  case 3: return get_fixed_view<View3>(underlyingView_).extent_int(dim);
147  case 4: return get_fixed_view<View4>(underlyingView_).extent_int(dim);
148  case 5: return get_fixed_view<View5>(underlyingView_).extent_int(dim);
149  case 6: return get_fixed_view<View6>(underlyingView_).extent_int(dim);
150  case 7: return get_fixed_view<View7>(underlyingView_).extent_int(dim);
151  default: return -1;
152  }
153  }
154 
157  {
158  INTREPID2_TEST_FOR_EXCEPTION(underlyingView_.valueless_by_exception(), std::invalid_argument, "underlyingView_ not correctly set!");
159  zeroView_ = ZeroView<DataScalar,DeviceType>::zeroView(); // one-entry (zero); used to allow getEntry() to return 0 for off-diagonal entries in BLOCK_PLUS_DIAGONAL
160  // check that rank is compatible with the claimed extents:
161  for (int d=rank_; d<7; d++)
162  {
163  INTREPID2_TEST_FOR_EXCEPTION(extents_[d] > 1, std::invalid_argument, "Nominal extents may not be > 1 in dimensions beyond the rank of the container");
164  }
165 
166  numActiveDims_ = 0;
167  int blockPlusDiagonalCount = 0;
168  underlyingMatchesLogical_ = true;
169  for (ordinal_type i=0; i<7; i++)
170  {
171  if (variationType_[i] == GENERAL)
172  {
173  if (extents_[i] != 0)
174  {
175  variationModulus_[i] = extents_[i];
176  }
177  else
178  {
179  variationModulus_[i] = 1;
180  }
181  activeDims_[numActiveDims_] = i;
182  numActiveDims_++;
183  }
184  else if (variationType_[i] == MODULAR)
185  {
186  underlyingMatchesLogical_ = false;
187  if (extents_[i] != getUnderlyingViewExtent(numActiveDims_))
188  {
189  const int dataExtent = getUnderlyingViewExtent(numActiveDims_);
190  const int logicalExtent = extents_[i];
191  const int modulus = dataExtent;
192 
193  INTREPID2_TEST_FOR_EXCEPTION( dataExtent * (logicalExtent / dataExtent) != logicalExtent, std::invalid_argument, "data extent must evenly divide logical extent");
194 
195  variationModulus_[i] = modulus;
196  }
197  else
198  {
199  variationModulus_[i] = extents_[i];
200  }
201  activeDims_[numActiveDims_] = i;
202  numActiveDims_++;
203  }
204  else if (variationType_[i] == BLOCK_PLUS_DIAGONAL)
205  {
206  underlyingMatchesLogical_ = false;
207  blockPlusDiagonalCount++;
208  if (blockPlusDiagonalCount == 1) // first dimension thus marked --> active
209  {
210 
211 #ifdef HAVE_INTREPID2_DEBUG
212  const int numNondiagonalEntries = blockPlusDiagonalNumNondiagonalEntries(blockPlusDiagonalLastNonDiagonal_);
213  const int dataExtent = getUnderlyingViewExtent(numActiveDims_); // flat storage of all matrix entries
214  const int logicalExtent = extents_[i];
215  const int numDiagonalEntries = logicalExtent - (blockPlusDiagonalLastNonDiagonal_ + 1);
216  const int expectedDataExtent = numNondiagonalEntries + numDiagonalEntries;
217  INTREPID2_TEST_FOR_EXCEPTION(dataExtent != expectedDataExtent, std::invalid_argument, ("BLOCK_PLUS_DIAGONAL data extent of " + std::to_string(dataExtent) + " does not match expected based on blockPlusDiagonalLastNonDiagonal setting of " + std::to_string(blockPlusDiagonalLastNonDiagonal_)).c_str());
218 #endif
219 
220  activeDims_[numActiveDims_] = i;
221  numActiveDims_++;
222  }
223  variationModulus_[i] = getUnderlyingViewExtent(numActiveDims_);
224  INTREPID2_TEST_FOR_EXCEPTION(variationType_[i+1] != BLOCK_PLUS_DIAGONAL, std::invalid_argument, "BLOCK_PLUS_DIAGONAL ranks must be contiguous");
225  i++; // skip over the next BLOCK_PLUS_DIAGONAL
226  variationModulus_[i] = 1; // trivial modulus (should not ever be used)
227  INTREPID2_TEST_FOR_EXCEPTION(blockPlusDiagonalCount > 1, std::invalid_argument, "BLOCK_PLUS_DIAGONAL can only apply to two ranks");
228  }
229  else // CONSTANT
230  {
231  if (i < rank_)
232  {
233  underlyingMatchesLogical_ = false;
234  }
235  variationModulus_[i] = 1; // trivial modulus
236  }
237  }
238 
239  if (rank_ != dataRank_)
240  {
241  underlyingMatchesLogical_ = false;
242  }
243 
244  for (int d=numActiveDims_; d<7; d++)
245  {
246  // for *inactive* dims, the activeDims_ map just is the identity
247  // (this allows getEntry() to work even when the logical rank of the Data object is lower than that of the underlying View. This can happen for gradients in 1D.)
248  activeDims_[d] = d;
249  }
250  for (int d=0; d<7; d++)
251  {
252  INTREPID2_TEST_FOR_EXCEPTION(variationModulus_[d] == 0, std::logic_error, "variationModulus should not ever be 0");
253  }
254  }
255 
256  public:
258  template<class UnaryOperator>
259  void applyOperator(UnaryOperator unaryOperator)
260  {
261  using ExecutionSpace = typename DeviceType::execution_space;
262 
263  switch (dataRank_)
264  {
265  case 1:
266  {
267  const int dataRank = 1;
268  auto view = getUnderlyingView<dataRank>();
269 
270  const int dataExtent = this->getDataExtent(0);
271  Kokkos::RangePolicy<ExecutionSpace> policy(ExecutionSpace(),0,dataExtent);
272  Kokkos::parallel_for("apply operator in-place", policy,
273  KOKKOS_LAMBDA (const int &i0) {
274  view(i0) = unaryOperator(view(i0));
275  });
276 
277  }
278  break;
279  case 2:
280  {
281  const int dataRank = 2;
282  auto policy = dataExtentRangePolicy<dataRank>();
283  auto view = getUnderlyingView<dataRank>();
284 
285  Kokkos::parallel_for("apply operator in-place", policy,
286  KOKKOS_LAMBDA (const int &i0, const int &i1) {
287  view(i0,i1) = unaryOperator(view(i0,i1));
288  });
289  }
290  break;
291  case 3:
292  {
293  const int dataRank = 3;
294  auto policy = dataExtentRangePolicy<dataRank>();
295  auto view = getUnderlyingView<dataRank>();
296 
297  Kokkos::parallel_for("apply operator in-place", policy,
298  KOKKOS_LAMBDA (const int &i0, const int &i1, const int &i2) {
299  view(i0,i1,i2) = unaryOperator(view(i0,i1,i2));
300  });
301  }
302  break;
303  case 4:
304  {
305  const int dataRank = 4;
306  auto policy = dataExtentRangePolicy<dataRank>();
307  auto view = getUnderlyingView<dataRank>();
308 
309  Kokkos::parallel_for("apply operator in-place", policy,
310  KOKKOS_LAMBDA (const int &i0, const int &i1, const int &i2, const int &i3) {
311  view(i0,i1,i2,i3) = unaryOperator(view(i0,i1,i2,i3));
312  });
313  }
314  break;
315  case 5:
316  {
317  const int dataRank = 5;
318  auto policy = dataExtentRangePolicy<dataRank>();
319  auto view = getUnderlyingView<dataRank>();
320 
321  Kokkos::parallel_for("apply operator in-place", policy,
322  KOKKOS_LAMBDA (const int &i0, const int &i1, const int &i2, const int &i3, const int &i4) {
323  view(i0,i1,i2,i3,i4) = unaryOperator(view(i0,i1,i2,i3,i4));
324  });
325  }
326  break;
327  case 6:
328  {
329  const int dataRank = 6;
330  auto policy = dataExtentRangePolicy<dataRank>();
331  auto view = getUnderlyingView<dataRank>();
332 
333  Kokkos::parallel_for("apply operator in-place", policy,
334  KOKKOS_LAMBDA (const int &i0, const int &i1, const int &i2, const int &i3, const int &i4, const int &i5) {
335  view(i0,i1,i2,i3,i4,i5) = unaryOperator(view(i0,i1,i2,i3,i4,i5));
336  });
337  }
338  break;
339  case 7:
340  {
341  const int dataRank = 7;
342  auto policy6 = dataExtentRangePolicy<6>();
343  auto view = getUnderlyingView<dataRank>();
344 
345  const int dim_i6 = view.extent_int(6);
346 
347  Kokkos::parallel_for("apply operator in-place", policy6,
348  KOKKOS_LAMBDA (const int &i0, const int &i1, const int &i2, const int &i3, const int &i4, const int &i5) {
349  for (int i6=0; i6<dim_i6; i6++)
350  {
351  view(i0,i1,i2,i3,i4,i5,i6) = unaryOperator(view(i0,i1,i2,i3,i4,i5,i6));
352  }
353  });
354  }
355  break;
356  default:
357  INTREPID2_TEST_FOR_EXCEPTION(true,std::invalid_argument,"Unsupported data rank");
358  }
359  }
360 
362  template<class ...IntArgs>
363  KOKKOS_INLINE_FUNCTION
364  reference_type getWritableEntryWithPassThroughOption(const bool &passThroughBlockDiagonalArgs, const IntArgs... intArgs) const
365  {
366  #ifdef INTREPID2_HAVE_DEBUG
367  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(numArgs != rank_, std::invalid_argument, "getWritableEntry() should have the same number of arguments as the logical rank.");
368  #endif
369  constexpr int numArgs = sizeof...(intArgs);
370  if (underlyingMatchesLogical_)
371  {
372  // in this case, we require that numArgs == dataRank_
373  return getUnderlyingView<numArgs>()(intArgs...);
374  }
375 
376  // extract the type of the first argument; use that for the arrays below
377  using int_type = std::tuple_element_t<0, std::tuple<IntArgs...>>;
378 
379  const Kokkos::Array<int_type, numArgs+1> args {intArgs...,0}; // we pad with one extra entry (0) to avoid gcc compiler warnings about references beyond the bounds of the array (the [d+1]'s below)
380  Kokkos::Array<int_type, 7> refEntry;
381  for (int d=0; d<numArgs; d++)
382  {
383  switch (variationType_[d])
384  {
385  case CONSTANT: refEntry[d] = 0; break;
386  case GENERAL: refEntry[d] = args[d]; break;
387  case MODULAR: refEntry[d] = args[d] % variationModulus_[d]; break;
388  case BLOCK_PLUS_DIAGONAL:
389  {
390  if (passThroughBlockDiagonalArgs)
391  {
392  refEntry[d] = args[d];
393  refEntry[d+1] = args[d+1]; // this had better be == 0
394  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(args[d+1] != 0, std::invalid_argument, "getWritableEntry() called with passThroughBlockDiagonalArgs = true, but nonzero second matrix argument.");
395  }
396  else
397  {
398  const int numNondiagonalEntries = blockPlusDiagonalNumNondiagonalEntries(blockPlusDiagonalLastNonDiagonal_);
399 
400  const int_type &i = args[d];
401  if (d+1 >= numArgs)
402  {
403  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(true, std::invalid_argument, "BLOCK_PLUS_DIAGONAL must be present for two dimensions; here, encountered only one");
404  }
405  else
406  {
407  const int_type &j = args[d+1];
408 
409  if ((i > static_cast<int_type>(blockPlusDiagonalLastNonDiagonal_)) || (j > static_cast<int_type>(blockPlusDiagonalLastNonDiagonal_)))
410  {
411  if (i != j)
412  {
413  // off diagonal: zero
414  return zeroView_(0); // NOTE: this branches in an argument-dependent way; this is not great for CUDA performance. When using BLOCK_PLUS_DIAGONAL, should generally avoid calls to this getEntry() method. (Use methods that directly take advantage of the data packing instead.)
415  }
416  else
417  {
418  refEntry[d] = blockPlusDiagonalDiagonalEntryIndex(blockPlusDiagonalLastNonDiagonal_, numNondiagonalEntries, i);
419  }
420  }
421  else
422  {
423  refEntry[d] = blockPlusDiagonalBlockEntryIndex(blockPlusDiagonalLastNonDiagonal_, numNondiagonalEntries, i, j);
424  }
425 
426  // skip next d (this is required also to be BLOCK_PLUS_DIAGONAL, and we've consumed its arg as j above)
427  refEntry[d+1] = 0;
428  }
429  }
430  d++;
431  }
432  }
433  }
434  // refEntry should be zero-filled beyond numArgs, for cases when rank_ < dataRank_ (this only is allowed if the extra dimensions each has extent 1).
435  for (int d=numArgs; d<7; d++)
436  {
437  refEntry[d] = 0;
438  }
439 
440  if (dataRank_ == 1)
441  {
442  return get_fixed_view<View1>(underlyingView_)(refEntry[activeDims_[0]]);
443  }
444  else if (dataRank_ == 2)
445  {
446  return get_fixed_view<View2>(underlyingView_)(refEntry[activeDims_[0]],refEntry[activeDims_[1]]);
447  }
448  else if (dataRank_ == 3)
449  {
450  return get_fixed_view<View3>(underlyingView_)(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]]);
451  }
452  else if (dataRank_ == 4)
453  {
454  return get_fixed_view<View4>(underlyingView_)(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]],refEntry[activeDims_[3]]);
455  }
456  else if (dataRank_ == 5)
457  {
458  return get_fixed_view<View5>(underlyingView_)(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]],refEntry[activeDims_[3]],
459  refEntry[activeDims_[4]]);
460  }
461  else if (dataRank_ == 6)
462  {
463  return get_fixed_view<View6>(underlyingView_)(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]],refEntry[activeDims_[3]],
464  refEntry[activeDims_[4]],refEntry[activeDims_[5]]);
465  }
466  else // dataRank_ == 7
467  {
468  return get_fixed_view<View7>(underlyingView_)(refEntry[activeDims_[0]],refEntry[activeDims_[1]],refEntry[activeDims_[2]],refEntry[activeDims_[3]],
469  refEntry[activeDims_[4]],refEntry[activeDims_[5]],refEntry[activeDims_[6]]);
470  }
471 
472  }
473 
475  template<class ...IntArgs>
476  KOKKOS_INLINE_FUNCTION
477  reference_type getWritableEntry(const IntArgs... intArgs) const
478  {
479  return getWritableEntryWithPassThroughOption(false, intArgs...);
480  }
481  public:
483  template<class ToContainer, class FromContainer>
484  static void copyContainer(ToContainer to, FromContainer from)
485  {
486 // std::cout << "Entered copyContainer().\n";
487  auto policy = Kokkos::MDRangePolicy<execution_space,Kokkos::Rank<6>>({0,0,0,0,0,0},{from.extent_int(0),from.extent_int(1),from.extent_int(2), from.extent_int(3), from.extent_int(4), from.extent_int(5)});
488 
489  Kokkos::parallel_for("copyContainer", policy,
490  KOKKOS_LAMBDA (const int &i0, const int &i1, const int &i2, const int &i3, const int &i4, const int &i5) {
491  for (int i6=0; i6<from.extent_int(6); i6++)
492  {
493  to.access(i0,i1,i2,i3,i4,i5,i6) = from.access(i0,i1,i2,i3,i4,i5,i6);
494  }
495  });
496  }
497 
499  void allocateAndCopyFromDynRankView(ScalarView<DataScalar,DeviceType> data)
500  {
501 // std::cout << "Entered allocateAndCopyFromDynRankView().\n";
502  switch (dataRank_)
503  {
504  case 1: underlyingView_ = Kokkos::View<DataScalar*, DeviceType>("Intrepid2 Data", data.extent_int(0)); break;
505  case 2: underlyingView_ = Kokkos::View<DataScalar**, DeviceType>("Intrepid2 Data", data.extent_int(0), data.extent_int(1)); break;
506  case 3: underlyingView_ = Kokkos::View<DataScalar***, DeviceType>("Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2)); break;
507  case 4: underlyingView_ = Kokkos::View<DataScalar****, DeviceType>("Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2), data.extent_int(3)); break;
508  case 5: underlyingView_ = Kokkos::View<DataScalar*****, DeviceType>("Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2), data.extent_int(3), data.extent_int(4)); break;
509  case 6: underlyingView_ = Kokkos::View<DataScalar******, DeviceType>("Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2), data.extent_int(3), data.extent_int(4), data.extent_int(5)); break;
510  case 7: underlyingView_ = Kokkos::View<DataScalar*******, DeviceType>("Intrepid2 Data", data.extent_int(0), data.extent_int(1), data.extent_int(2), data.extent_int(3), data.extent_int(4), data.extent_int(5), data.extent_int(6)); break;
511  default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
512  }
513 
514  switch (dataRank_)
515  {
516  case 1: copyContainer(get_fixed_view<View1>(underlyingView_),data); break;
517  case 2: copyContainer(get_fixed_view<View2>(underlyingView_),data); break;
518  case 3: copyContainer(get_fixed_view<View3>(underlyingView_),data); break;
519  case 4: copyContainer(get_fixed_view<View4>(underlyingView_),data); break;
520  case 5: copyContainer(get_fixed_view<View5>(underlyingView_),data); break;
521  case 6: copyContainer(get_fixed_view<View6>(underlyingView_),data); break;
522  case 7: copyContainer(get_fixed_view<View7>(underlyingView_),data); break;
523  default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
524  }
525  }
526 
528  Data(std::vector<DimensionInfo> dimInfoVector)
529  :
530  // initialize member variables as if default constructor; if dimInfoVector is empty, we want default constructor behavior.
531  dataRank_(0), extents_({0,0,0,0,0,0,0}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(-1), rank_(dimInfoVector.size())
532  {
533  // If dimInfoVector is empty, the member initialization above is correct; otherwise, we set as below.
534  // Either way, once members are initialized, we must call setActiveDims().
535  if (dimInfoVector.size() != 0)
536  {
537  std::vector<int> dataExtents;
538 
539  bool blockPlusDiagonalEncountered = false;
540  for (int d=0; d<rank_; d++)
541  {
542  const DimensionInfo & dimInfo = dimInfoVector[d];
543  extents_[d] = dimInfo.logicalExtent;
544  variationType_[d] = dimInfo.variationType;
545  const bool isBlockPlusDiagonal = (variationType_[d] == BLOCK_PLUS_DIAGONAL);
546  const bool isSecondBlockPlusDiagonal = isBlockPlusDiagonal && blockPlusDiagonalEncountered;
547  if (isBlockPlusDiagonal)
548  {
549  blockPlusDiagonalEncountered = true;
550  blockPlusDiagonalLastNonDiagonal_ = dimInfo.blockPlusDiagonalLastNonDiagonal;
551  }
552  if ((variationType_[d] != CONSTANT) && (!isSecondBlockPlusDiagonal))
553  {
554  dataExtents.push_back(dimInfo.dataExtent);
555  }
556  }
557  if (dataExtents.size() == 0)
558  {
559  // constant data
560  dataExtents.push_back(1);
561  }
562  dataRank_ = dataExtents.size();
563  switch (dataRank_)
564  {
565  case 1: underlyingView_ = Kokkos::View<DataScalar*, DeviceType>("Intrepid2 Data", dataExtents[0]); break;
566  case 2: underlyingView_ = Kokkos::View<DataScalar**, DeviceType>("Intrepid2 Data", dataExtents[0], dataExtents[1]); break;
567  case 3: underlyingView_ = Kokkos::View<DataScalar***, DeviceType>("Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2]); break;
568  case 4: underlyingView_ = Kokkos::View<DataScalar****, DeviceType>("Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2], dataExtents[3]); break;
569  case 5: underlyingView_ = Kokkos::View<DataScalar*****, DeviceType>("Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2], dataExtents[3], dataExtents[4]); break;
570  case 6: underlyingView_ = Kokkos::View<DataScalar******, DeviceType>("Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2], dataExtents[3], dataExtents[4], dataExtents[5]); break;
571  case 7: underlyingView_ = Kokkos::View<DataScalar*******, DeviceType>("Intrepid2 Data", dataExtents[0], dataExtents[1], dataExtents[2], dataExtents[3], dataExtents[4], dataExtents[5], dataExtents[6]); break;
572  default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
573  }
574  }
575  setActiveDims();
576  }
577 
579  Data(const ScalarView<DataScalar,DeviceType> &data, int rank, Kokkos::Array<int,7> extents, Kokkos::Array<DataVariationType,7> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
580  :
581  dataRank_(data.rank()), extents_(extents), variationType_(variationType), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
582  {
584  setActiveDims();
585  }
586 
588  template<typename OtherDeviceType, class = typename std::enable_if< std::is_same<typename DeviceType::memory_space, typename OtherDeviceType::memory_space>::value>::type,
589  class = typename std::enable_if<!std::is_same<DeviceType,OtherDeviceType>::value>::type>
590  Data(const Data<DataScalar,OtherDeviceType> &data)
591  :
592  dataRank_(data.getUnderlyingViewRank()), extents_(data.getExtents()), variationType_(data.getVariationTypes()), blockPlusDiagonalLastNonDiagonal_(data.blockPlusDiagonalLastNonDiagonal()), rank_(data.rank())
593  {
594 // std::cout << "Entered copy-like Data constructor.\n";
595  if (dataRank_ != 0) // dataRank_ == 0 indicates an invalid Data object (a placeholder, can indicate zero value)
596  {
597  const auto view = data.getUnderlyingView();
598  switch (dataRank_)
599  {
600  case 1: underlyingView_ = data.getUnderlyingView1(); break;
601  case 2: underlyingView_ = data.getUnderlyingView2(); break;
602  case 3: underlyingView_ = data.getUnderlyingView3(); break;
603  case 4: underlyingView_ = data.getUnderlyingView4(); break;
604  case 5: underlyingView_ = data.getUnderlyingView5(); break;
605  case 6: underlyingView_ = data.getUnderlyingView6(); break;
606  case 7: underlyingView_ = data.getUnderlyingView7(); break;
607  default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
608  }
609  }
610  setActiveDims();
611  }
612 
614  template<typename OtherDeviceType, class = typename std::enable_if<!std::is_same<typename DeviceType::memory_space, typename OtherDeviceType::memory_space>::value>::type>
616  :
617  dataRank_(data.getUnderlyingViewRank()), extents_(data.getExtents()), variationType_(data.getVariationTypes()), blockPlusDiagonalLastNonDiagonal_(data.blockPlusDiagonalLastNonDiagonal()), rank_(data.rank())
618  {
619 // std::cout << "Entered copy-like Data constructor.\n";
620  if (dataRank_ != 0) // dataRank_ == 0 indicates an invalid Data object (a placeholder, can indicate zero value)
621  {
622  const auto view = data.getUnderlyingView();
623  switch (dataRank_)
624  {
625  case 1: underlyingView_ = Kokkos::View<DataScalar*, DeviceType>("Intrepid2 Data", view.extent_int(0)); break;
626  case 2: underlyingView_ = Kokkos::View<DataScalar**, DeviceType>("Intrepid2 Data", view.extent_int(0), view.extent_int(1)); break;
627  case 3: underlyingView_ = Kokkos::View<DataScalar***, DeviceType>("Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2)); break;
628  case 4: underlyingView_ = Kokkos::View<DataScalar****, DeviceType>("Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3)); break;
629  case 5: underlyingView_ = Kokkos::View<DataScalar*****, DeviceType>("Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4)); break;
630  case 6: underlyingView_ = Kokkos::View<DataScalar******, DeviceType>("Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4), view.extent_int(5)); break;
631  case 7: underlyingView_ = Kokkos::View<DataScalar*******, DeviceType>("Intrepid2 Data", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4), view.extent_int(5), view.extent_int(6)); break;
632  default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
633  }
634 
635  // copy
636  // (Note: Kokkos::deep_copy() will not generally work if the layouts are different; that's why we do a manual copy here once we have the data on the host):
637  // first, mirror and copy dataView; then copy to the appropriate data_ member
638  using MemorySpace = typename DeviceType::memory_space;
639  switch (dataRank_)
640  {
641  case 1: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView1()); copyContainer(get_fixed_view<View1>(underlyingView_), dataViewMirror);} break;
642  case 2: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView2()); copyContainer(get_fixed_view<View2>(underlyingView_), dataViewMirror);} break;
643  case 3: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView3()); copyContainer(get_fixed_view<View3>(underlyingView_), dataViewMirror);} break;
644  case 4: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView4()); copyContainer(get_fixed_view<View4>(underlyingView_), dataViewMirror);} break;
645  case 5: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView5()); copyContainer(get_fixed_view<View5>(underlyingView_), dataViewMirror);} break;
646  case 6: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView6()); copyContainer(get_fixed_view<View6>(underlyingView_), dataViewMirror);} break;
647  case 7: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView7()); copyContainer(get_fixed_view<View7>(underlyingView_), dataViewMirror);} break;
648  default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
649  }
650  }
651  setActiveDims();
652  }
653 
655 // Data(const Data<DataScalar,ExecSpaceType> &data)
656 // :
657 // dataRank_(data.getUnderlyingViewRank()), extents_(data.getExtents()), variationType_(data.getVariationTypes()), blockPlusDiagonalLastNonDiagonal_(data.blockPlusDiagonalLastNonDiagonal()), rank_(data.rank())
658 // {
659 // std::cout << "Entered Data copy constructor.\n";
660 // if (dataRank_ != 0) // dataRank_ == 0 indicates an invalid Data object (a placeholder, can indicate zero value)
661 // {
662 // const auto view = data.getUnderlyingView();
663 // switch (dataRank_)
664 // {
665 // case 1: underlyingView_ = Kokkos::View<DataScalar*, DeviceType>("Intrepid2 Data - explicit copy constructor(for debugging)", view.extent_int(0)); break;
666 // case 2: underlyingView_ = Kokkos::View<DataScalar**, DeviceType>("Intrepid2 Data - explicit copy constructor(for debugging)", view.extent_int(0), view.extent_int(1)); break;
667 // case 3: underlyingView_ = Kokkos::View<DataScalar***, DeviceType>("Intrepid2 Data - explicit copy constructor(for debugging)", view.extent_int(0), view.extent_int(1), view.extent_int(2)); break;
668 // case 4: underlyingView_ = Kokkos::View<DataScalar****, DeviceType>("Intrepid2 Data - explicit copy constructor(for debugging)", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3)); break;
669 // case 5: underlyingView_ = Kokkos::View<DataScalar*****, DeviceType>("Intrepid2 Data - explicit copy constructor(for debugging)", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4)); break;
670 // case 6: underlyingView_ = Kokkos::View<DataScalar******, DeviceType>("Intrepid2 Data - explicit copy constructor(for debugging)", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4), view.extent_int(5)); break;
671 // case 7: underlyingView_ = Kokkos::View<DataScalar*******, DeviceType>("Intrepid2 Data - explicit copy constructor(for debugging)", view.extent_int(0), view.extent_int(1), view.extent_int(2), view.extent_int(3), view.extent_int(4), view.extent_int(5), view.extent_int(6)); break;
672 // default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
673 // }
674 //
675 // // copy
676 // // (Note: Kokkos::deep_copy() will not generally work if the layouts are different; that's why we do a manual copy here once we have the data on the host):
677 // // first, mirror and copy dataView; then copy to the appropriate data_ member
678 // using MemorySpace = typename DeviceType::memory_space;
679 // switch (dataRank_)
680 // {
681 // case 1: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView1()); copyContainer(get_fixed_view<View1>(underlyingView_), dataViewMirror);} break;
682 // case 2: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView2()); copyContainer(get_fixed_view<View2>(underlyingView_), dataViewMirror);} break;
683 // case 3: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView3()); copyContainer(get_fixed_view<View3>(underlyingView_), dataViewMirror);} break;
684 // case 4: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView4()); copyContainer(get_fixed_view<View4>(underlyingView_), dataViewMirror);} break;
685 // case 5: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView5()); copyContainer(get_fixed_view<View5>(underlyingView_), dataViewMirror);} break;
686 // case 6: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView6()); copyContainer(get_fixed_view<View6>(underlyingView_), dataViewMirror);} break;
687 // case 7: {auto dataViewMirror = Kokkos::create_mirror_view_and_copy(MemorySpace(), data.getUnderlyingView7()); copyContainer(get_fixed_view<View7>(underlyingView_), dataViewMirror);} break;
688 // default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
689 // }
690 // }
691 //
692 // setActiveDims();
693 // }
694 
696  Data(ScalarView<DataScalar,DeviceType> data)
697  :
698  Data(data,
699  data.rank(),
700  Kokkos::Array<int,7> {data.extent_int(0),data.extent_int(1),data.extent_int(2),data.extent_int(3),data.extent_int(4),data.extent_int(5),data.extent_int(6)},
701  Kokkos::Array<DataVariationType,7> {GENERAL,GENERAL,GENERAL,GENERAL,GENERAL,GENERAL,GENERAL}, -1)
702  {}
703 
705  template<size_t rank, class ...DynRankViewProperties>
706  Data(const Kokkos::DynRankView<DataScalar,DeviceType, DynRankViewProperties...> &data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
707  :
708  dataRank_(data.rank()), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
709  {
710 // std::cout << "Entered a DynRankView Data() constructor.\n";
712  for (unsigned d=0; d<rank; d++)
713  {
714  extents_[d] = extents[d];
715  variationType_[d] = variationType[d];
716  }
717  setActiveDims();
718  }
719 
720  template<size_t rank, class ...ViewProperties>
721  Data(Kokkos::View<DataScalar*,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
722  :
723  dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
724  {
725  underlyingView_ = data;
726  for (unsigned d=0; d<rank; d++)
727  {
728  extents_[d] = extents[d];
729  variationType_[d] = variationType[d];
730  }
731  setActiveDims();
732  }
733 
734  template<size_t rank, class ...ViewProperties>
735  Data(Kokkos::View<DataScalar**,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
736  :
737  dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
738  {
739  underlyingView_ = data;
740  for (unsigned d=0; d<rank; d++)
741  {
742  extents_[d] = extents[d];
743  variationType_[d] = variationType[d];
744  }
745  setActiveDims();
746  }
747 
748  template<size_t rank, class ...ViewProperties>
749  Data(Kokkos::View<DataScalar***,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
750  :
751  dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
752  {
753  underlyingView_ = data;
754  for (unsigned d=0; d<rank; d++)
755  {
756  extents_[d] = extents[d];
757  variationType_[d] = variationType[d];
758  }
759  setActiveDims();
760  }
761 
762  template<size_t rank, class ...ViewProperties>
763  Data(Kokkos::View<DataScalar****,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
764  :
765  dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
766  {
767  underlyingView_ = data;
768  for (unsigned d=0; d<rank; d++)
769  {
770  extents_[d] = extents[d];
771  variationType_[d] = variationType[d];
772  }
773  setActiveDims();
774  }
775 
776  template<size_t rank, class ...ViewProperties>
777  Data(Kokkos::View<DataScalar*****,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
778  :
779  dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
780  {
781  underlyingView_ = data;
782  for (unsigned d=0; d<rank; d++)
783  {
784  extents_[d] = extents[d];
785  variationType_[d] = variationType[d];
786  }
787  setActiveDims();
788  }
789 
790  template<size_t rank, class ...ViewProperties>
791  Data(Kokkos::View<DataScalar******,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
792  :
793  dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
794  {
795  underlyingView_ = data;
796  for (unsigned d=0; d<rank; d++)
797  {
798  extents_[d] = extents[d];
799  variationType_[d] = variationType[d];
800  }
801  setActiveDims();
802  }
803 
804  template<size_t rank, class ...ViewProperties>
805  Data(Kokkos::View<DataScalar*******,DeviceType, ViewProperties...> data, Kokkos::Array<int,rank> extents, Kokkos::Array<DataVariationType,rank> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
806  :
807  dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
808  {
809  underlyingView_ = data;
810  for (unsigned d=0; d<rank; d++)
811  {
812  extents_[d] = extents[d];
813  variationType_[d] = variationType[d];
814  }
815  setActiveDims();
816  }
817 
819  template<class ViewScalar, class ...ViewProperties>
820  Data(const unsigned rank, Kokkos::View<ViewScalar,DeviceType, ViewProperties...> data, Kokkos::Array<int,7> extents, Kokkos::Array<DataVariationType,7> variationType, const int blockPlusDiagonalLastNonDiagonal = -1)
821  :
822  dataRank_(data.rank), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(blockPlusDiagonalLastNonDiagonal), rank_(rank)
823  {
824  underlyingView_ = data;
825  for (unsigned d=0; d<rank; d++)
826  {
827  extents_[d] = extents[d];
828  variationType_[d] = variationType[d];
829  }
830  setActiveDims();
831  }
832 
834  template<size_t rank>
835  Data(DataScalar constantValue, Kokkos::Array<int,rank> extents)
836  :
837  dataRank_(1), extents_({1,1,1,1,1,1,1}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(-1), rank_(rank)
838  {
839  underlyingView_ = Kokkos::View<DataScalar*,DeviceType>("Constant Data",1);
840  Kokkos::deep_copy(get_fixed_view<View1>(underlyingView_), constantValue);
841  for (unsigned d=0; d<rank; d++)
842  {
843  extents_[d] = extents[d];
844  }
845  setActiveDims();
846  }
847 
850  :
851  dataRank_(0), extents_({0,0,0,0,0,0,0}), variationType_({CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT,CONSTANT}), blockPlusDiagonalLastNonDiagonal_(-1), rank_(0)
852  {
853  setActiveDims();
854  }
855 
857  KOKKOS_INLINE_FUNCTION
859  {
860  return blockPlusDiagonalLastNonDiagonal_;
861  }
862 
864  KOKKOS_INLINE_FUNCTION
865  Kokkos::Array<int,7> getExtents() const
866  {
867  return extents_;
868  }
869 
871  KOKKOS_INLINE_FUNCTION
872  DimensionInfo getDimensionInfo(const int &dim) const
873  {
874  DimensionInfo dimInfo;
875 
876  dimInfo.logicalExtent = extent_int(dim);
877  dimInfo.variationType = variationType_[dim];
878  dimInfo.dataExtent = getDataExtent(dim);
879  dimInfo.variationModulus = variationModulus_[dim];
880 
881  if (dimInfo.variationType == BLOCK_PLUS_DIAGONAL)
882  {
883  dimInfo.blockPlusDiagonalLastNonDiagonal = blockPlusDiagonalLastNonDiagonal_;
884  }
885  return dimInfo;
886  }
887 
889  KOKKOS_INLINE_FUNCTION
890  DimensionInfo combinedDataDimensionInfo(const Data &otherData, const int &dim) const
891  {
892  const DimensionInfo myDimInfo = getDimensionInfo(dim);
893  const DimensionInfo otherDimInfo = otherData.getDimensionInfo(dim);
894 
895  return combinedDimensionInfo(myDimInfo, otherDimInfo);
896  }
897 
899  template<int rank>
900  KOKKOS_INLINE_FUNCTION
901  enable_if_t<rank==1, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
903  {
904  #ifdef HAVE_INTREPID2_DEBUG
905  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(dataRank_ != rank, std::invalid_argument, "getUnderlyingView() called for rank that does not match dataRank_");
906  #endif
907  return get_fixed_view<View1>(underlyingView_);
908  }
909 
911  template<int rank>
912  KOKKOS_INLINE_FUNCTION
913  enable_if_t<rank==2, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
915  {
916  #ifdef HAVE_INTREPID2_DEBUG
917  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(dataRank_ != rank, std::invalid_argument, "getUnderlyingView() called for rank that does not match dataRank_");
918  #endif
919  return get_fixed_view<View2>(underlyingView_);
920  }
921 
923  template<int rank>
924  KOKKOS_INLINE_FUNCTION
925  enable_if_t<rank==3, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
927  {
928  #ifdef HAVE_INTREPID2_DEBUG
929  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(dataRank_ != rank, std::invalid_argument, "getUnderlyingView() called for rank that does not match dataRank_");
930  #endif
931  return get_fixed_view<View3>(underlyingView_);
932  }
933 
935  template<int rank>
936  KOKKOS_INLINE_FUNCTION
937  enable_if_t<rank==4, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
939  {
940  #ifdef HAVE_INTREPID2_DEBUG
941  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(dataRank_ != rank, std::invalid_argument, "getUnderlyingView() called for rank that does not match dataRank_");
942  #endif
943  return get_fixed_view<View4>(underlyingView_);
944  }
945 
947  template<int rank>
948  KOKKOS_INLINE_FUNCTION
949  enable_if_t<rank==5, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
951  {
952  #ifdef HAVE_INTREPID2_DEBUG
953  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(dataRank_ != rank, std::invalid_argument, "getUnderlyingView() called for rank that does not match dataRank_");
954  #endif
955  return get_fixed_view<View5>(underlyingView_);
956  }
957 
959  template<int rank>
960  KOKKOS_INLINE_FUNCTION
961  enable_if_t<rank==6, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
963  {
964  #ifdef HAVE_INTREPID2_DEBUG
965  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(dataRank_ != rank, std::invalid_argument, "getUnderlyingView() called for rank that does not match dataRank_");
966  #endif
967  return get_fixed_view<View6>(underlyingView_);
968  }
969 
971  template<int rank>
972  KOKKOS_INLINE_FUNCTION
973  enable_if_t<rank==7, const Kokkos::View<typename RankExpander<DataScalar, rank>::value_type, DeviceType> &>
975  {
976  #ifdef HAVE_INTREPID2_DEBUG
977  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(dataRank_ != rank, std::invalid_argument, "getUnderlyingView() called for rank that does not match dataRank_");
978  #endif
979  return get_fixed_view<View7>(underlyingView_);
980  }
981 
983  KOKKOS_INLINE_FUNCTION
984  const Kokkos::View<DataScalar*, DeviceType> & getUnderlyingView1() const
985  {
986  return getUnderlyingView<1>();
987  }
988 
990  KOKKOS_INLINE_FUNCTION
991  const Kokkos::View<DataScalar**, DeviceType> & getUnderlyingView2() const
992  {
993  return getUnderlyingView<2>();
994  }
995 
997  KOKKOS_INLINE_FUNCTION
998  const Kokkos::View<DataScalar***, DeviceType> & getUnderlyingView3() const
999  {
1000  return getUnderlyingView<3>();
1001  }
1002 
1004  KOKKOS_INLINE_FUNCTION
1005  const Kokkos::View<DataScalar****, DeviceType> & getUnderlyingView4() const
1006  {
1007  return getUnderlyingView<4>();
1008  }
1009 
1011  KOKKOS_INLINE_FUNCTION
1012  const Kokkos::View<DataScalar*****, DeviceType> & getUnderlyingView5() const
1013  {
1014  return getUnderlyingView<5>();
1015  }
1016 
1018  KOKKOS_INLINE_FUNCTION
1019  const Kokkos::View<DataScalar******, DeviceType> & getUnderlyingView6() const
1020  {
1021  return getUnderlyingView<6>();
1022  }
1023 
1025  KOKKOS_INLINE_FUNCTION
1026  const Kokkos::View<DataScalar*******, DeviceType> & getUnderlyingView7() const
1027  {
1028  return getUnderlyingView<7>();
1029  }
1030 
1032  ScalarView<DataScalar,DeviceType> getUnderlyingView() const
1033  {
1034  switch (dataRank_)
1035  {
1036  case 1: return get_fixed_view<View1>(underlyingView_);
1037  case 2: return get_fixed_view<View2>(underlyingView_);
1038  case 3: return get_fixed_view<View3>(underlyingView_);
1039  case 4: return get_fixed_view<View4>(underlyingView_);
1040  case 5: return get_fixed_view<View5>(underlyingView_);
1041  case 6: return get_fixed_view<View6>(underlyingView_);
1042  case 7: return get_fixed_view<View7>(underlyingView_);
1043  default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
1044  }
1045  }
1046 
1048  KOKKOS_INLINE_FUNCTION
1049  ordinal_type getUnderlyingViewRank() const
1050  {
1051  return dataRank_;
1052  }
1053 
1055  KOKKOS_INLINE_FUNCTION
1056  ordinal_type getUnderlyingViewSize() const
1057  {
1058  ordinal_type size = 1;
1059  for (ordinal_type r=0; r<dataRank_; r++)
1060  {
1061  size *= getUnderlyingViewExtent(r);
1062  }
1063  return size;
1064  }
1065 
1067  ScalarView<DataScalar,DeviceType> allocateDynRankViewMatchingUnderlying() const
1068  {
1069  switch (dataRank_)
1070  {
1071  case 1: return getMatchingViewWithLabel(get_fixed_view<View1>(underlyingView_), "Intrepid2 Data", get_fixed_view<View1>(underlyingView_).extent_int(0));
1072  case 2: return getMatchingViewWithLabel(get_fixed_view<View2>(underlyingView_), "Intrepid2 Data", get_fixed_view<View2>(underlyingView_).extent_int(0), get_fixed_view<View2>(underlyingView_).extent_int(1));
1073  case 3: return getMatchingViewWithLabel(get_fixed_view<View3>(underlyingView_), "Intrepid2 Data", get_fixed_view<View3>(underlyingView_).extent_int(0), get_fixed_view<View3>(underlyingView_).extent_int(1), get_fixed_view<View3>(underlyingView_).extent_int(2));
1074  case 4: return getMatchingViewWithLabel(get_fixed_view<View4>(underlyingView_), "Intrepid2 Data", get_fixed_view<View4>(underlyingView_).extent_int(0), get_fixed_view<View4>(underlyingView_).extent_int(1), get_fixed_view<View4>(underlyingView_).extent_int(2), get_fixed_view<View4>(underlyingView_).extent_int(3));
1075  case 5: return getMatchingViewWithLabel(get_fixed_view<View5>(underlyingView_), "Intrepid2 Data", get_fixed_view<View5>(underlyingView_).extent_int(0), get_fixed_view<View5>(underlyingView_).extent_int(1), get_fixed_view<View5>(underlyingView_).extent_int(2), get_fixed_view<View5>(underlyingView_).extent_int(3), get_fixed_view<View5>(underlyingView_).extent_int(4));
1076  case 6: return getMatchingViewWithLabel(get_fixed_view<View6>(underlyingView_), "Intrepid2 Data", get_fixed_view<View6>(underlyingView_).extent_int(0), get_fixed_view<View6>(underlyingView_).extent_int(1), get_fixed_view<View6>(underlyingView_).extent_int(2), get_fixed_view<View6>(underlyingView_).extent_int(3), get_fixed_view<View6>(underlyingView_).extent_int(4), get_fixed_view<View6>(underlyingView_).extent_int(5));
1077  case 7: return getMatchingViewWithLabel(get_fixed_view<View7>(underlyingView_), "Intrepid2 Data", get_fixed_view<View7>(underlyingView_).extent_int(0), get_fixed_view<View7>(underlyingView_).extent_int(1), get_fixed_view<View7>(underlyingView_).extent_int(2), get_fixed_view<View7>(underlyingView_).extent_int(3), get_fixed_view<View7>(underlyingView_).extent_int(4), get_fixed_view<View7>(underlyingView_).extent_int(5), get_fixed_view<View7>(underlyingView_).extent_int(6));
1078  default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
1079  }
1080  }
1081 
1083  template<class ... DimArgs>
1084  ScalarView<DataScalar,DeviceType> allocateDynRankViewMatchingUnderlying(DimArgs... dims) const
1085  {
1086  switch (dataRank_)
1087  {
1088  case 1: return getMatchingViewWithLabel(get_fixed_view<View1>(underlyingView_), "Intrepid2 Data", dims...);
1089  case 2: return getMatchingViewWithLabel(get_fixed_view<View2>(underlyingView_), "Intrepid2 Data", dims...);
1090  case 3: return getMatchingViewWithLabel(get_fixed_view<View3>(underlyingView_), "Intrepid2 Data", dims...);
1091  case 4: return getMatchingViewWithLabel(get_fixed_view<View4>(underlyingView_), "Intrepid2 Data", dims...);
1092  case 5: return getMatchingViewWithLabel(get_fixed_view<View5>(underlyingView_), "Intrepid2 Data", dims...);
1093  case 6: return getMatchingViewWithLabel(get_fixed_view<View6>(underlyingView_), "Intrepid2 Data", dims...);
1094  case 7: return getMatchingViewWithLabel(get_fixed_view<View7>(underlyingView_), "Intrepid2 Data", dims...);
1095  default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
1096  }
1097  }
1098 
1100  void clear() const
1101  {
1102  switch (dataRank_)
1103  {
1104  case 1: Kokkos::deep_copy(get_fixed_view<View1>(underlyingView_), 0.0); break;
1105  case 2: Kokkos::deep_copy(get_fixed_view<View2>(underlyingView_), 0.0); break;
1106  case 3: Kokkos::deep_copy(get_fixed_view<View3>(underlyingView_), 0.0); break;
1107  case 4: Kokkos::deep_copy(get_fixed_view<View4>(underlyingView_), 0.0); break;
1108  case 5: Kokkos::deep_copy(get_fixed_view<View5>(underlyingView_), 0.0); break;
1109  case 6: Kokkos::deep_copy(get_fixed_view<View6>(underlyingView_), 0.0); break;
1110  case 7: Kokkos::deep_copy(get_fixed_view<View7>(underlyingView_), 0.0); break;
1111  default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
1112  }
1113  }
1114 
1116  void copyDataFromDynRankViewMatchingUnderlying(const ScalarView<DataScalar,DeviceType> &dynRankView) const
1117  {
1118 // std::cout << "Entered copyDataFromDynRankViewMatchingUnderlying().\n";
1119  switch (dataRank_)
1120  {
1121  case 1: copyContainer(get_fixed_view<View1>(underlyingView_),dynRankView); break;
1122  case 2: copyContainer(get_fixed_view<View2>(underlyingView_),dynRankView); break;
1123  case 3: copyContainer(get_fixed_view<View3>(underlyingView_),dynRankView); break;
1124  case 4: copyContainer(get_fixed_view<View4>(underlyingView_),dynRankView); break;
1125  case 5: copyContainer(get_fixed_view<View5>(underlyingView_),dynRankView); break;
1126  case 6: copyContainer(get_fixed_view<View6>(underlyingView_),dynRankView); break;
1127  case 7: copyContainer(get_fixed_view<View7>(underlyingView_),dynRankView); break;
1128  default: INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Invalid data rank");
1129  }
1130  }
1131 
1133  KOKKOS_INLINE_FUNCTION int getDataExtent(const ordinal_type &d) const
1134  {
1135  for (int i=0; i<numActiveDims_; i++)
1136  {
1137  if (activeDims_[i] == d)
1138  {
1139  return getUnderlyingViewExtent(i);
1140  }
1141  else if (activeDims_[i] > d)
1142  {
1143  return 1; // data does not vary in the specified dimension
1144  }
1145  }
1146  return 1; // data does not vary in the specified dimension
1147  }
1148 
1160  KOKKOS_INLINE_FUNCTION
1161  int getVariationModulus(const int &d) const
1162  {
1163  return variationModulus_[d];
1164  }
1165 
1167  KOKKOS_INLINE_FUNCTION
1168  const Kokkos::Array<DataVariationType,7> & getVariationTypes() const
1169  {
1170  return variationType_;
1171  }
1172 
1174  template<class ...IntArgs>
1175  KOKKOS_INLINE_FUNCTION
1176  return_type getEntryWithPassThroughOption(const bool &passThroughBlockDiagonalArgs, const IntArgs&... intArgs) const
1177  {
1178  return getWritableEntryWithPassThroughOption(passThroughBlockDiagonalArgs, intArgs...);
1179  }
1180 
1182  template<class ...IntArgs>
1183  KOKKOS_INLINE_FUNCTION
1184  return_type getEntry(const IntArgs&... intArgs) const
1185  {
1186  return getEntryWithPassThroughOption(false, intArgs...);
1187  }
1188 
1189  template <bool...> struct bool_pack;
1190 
1191  template <bool... v>
1192  using all_true = std::is_same<bool_pack<true, v...>, bool_pack<v..., true>>;
1193 
1194  template <class ...IntArgs>
1195  using valid_args = all_true<std::is_integral<IntArgs>{}...>;
1196 
1197  static_assert(valid_args<int,long,unsigned>::value, "valid args works");
1198 
1200  template <class ...IntArgs>
1201  KOKKOS_INLINE_FUNCTION
1202 #ifndef __INTEL_COMPILER
1203  // icc has a bug that prevents compilation with this enable_if_t
1204  // (possibly the same as https://community.intel.com/t5/Intel-C-Compiler/Intel-Compiler-bug-while-deducing-template-arguments-inside/m-p/1164358)
1205  // so with icc we'll just skip the argument type/count check
1206  enable_if_t<valid_args<IntArgs...>::value && (sizeof...(IntArgs) <= 7),return_type>
1207 #else
1208  return_type
1209 #endif
1210  operator()(const IntArgs&... intArgs) const {
1211  return getEntry(intArgs...);
1212  }
1213 
1215  KOKKOS_INLINE_FUNCTION
1216  int extent_int(const int& r) const
1217  {
1218  return extents_[r];
1219  }
1220 
1221  template <typename iType>
1222  KOKKOS_INLINE_FUNCTION constexpr
1223  typename std::enable_if<std::is_integral<iType>::value, size_t>::type
1224  extent(const iType& r) const {
1225  return extents_[r];
1226  }
1227 
1229  KOKKOS_INLINE_FUNCTION bool isDiagonal() const
1230  {
1231  if (blockPlusDiagonalLastNonDiagonal_ >= 1) return false;
1232  int numBlockPlusDiagonalTypes = 0;
1233  for (unsigned r = 0; r<variationType_.size(); r++)
1234  {
1235  const auto &entryType = variationType_[r];
1236  if (entryType == BLOCK_PLUS_DIAGONAL) numBlockPlusDiagonalTypes++;
1237  }
1238  // 2 BLOCK_PLUS_DIAGONAL entries, combined with blockPlusDiagonalLastNonDiagonal being -1 or 0 indicates diagonal
1239  if (numBlockPlusDiagonalTypes == 2) return true;
1240  else if (numBlockPlusDiagonalTypes == 0) return false; // no BLOCK_PLUS_DIAGONAL --> not a diagonal matrix
1241  else INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(true, std::invalid_argument, "Unexpected number of ranks marked as BLOCK_PLUS_DIAGONAL (should be 0 or 2)");
1242  return false; // statement should be unreachable; included because compilers don't necessarily recognize that fact...
1243  }
1244 
1249  {
1250  return Data<DataScalar,DeviceType>(value, this->getExtents());
1251  }
1252 
1259  {
1260  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A.rank() != B.rank(), std::invalid_argument, "A and B must have the same logical shape");
1261  const int rank = A.rank();
1262  std::vector<DimensionInfo> dimInfo(rank);
1263  for (int d=0; d<rank; d++)
1264  {
1265  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A.extent_int(d) != B.extent_int(d), std::invalid_argument, "A and B must have the same logical shape");
1266  dimInfo[d] = A.combinedDataDimensionInfo(B, d);
1267  }
1268  Data<DataScalar,DeviceType> result(dimInfo);
1269  return result;
1270  }
1271 
1278  static Data<DataScalar,DeviceType> allocateMatMatResult( const bool transposeA, const Data<DataScalar,DeviceType> &A_MatData, const bool transposeB, const Data<DataScalar,DeviceType> &B_MatData )
1279  {
1280  // we treat last two logical dimensions of matData as the matrix; last dimension of vecData as the vector
1281  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A_MatData.rank() != B_MatData.rank(), std::invalid_argument, "AmatData and BmatData have incompatible ranks");
1282 
1283  const int D1_DIM = A_MatData.rank() - 2;
1284  const int D2_DIM = A_MatData.rank() - 1;
1285 
1286  const int A_rows = A_MatData.extent_int(D1_DIM);
1287  const int A_cols = A_MatData.extent_int(D2_DIM);
1288  const int B_rows = B_MatData.extent_int(D1_DIM);
1289  const int B_cols = B_MatData.extent_int(D2_DIM);
1290 
1291  const int leftRows = transposeA ? A_cols : A_rows;
1292  const int leftCols = transposeA ? A_rows : A_cols;
1293  const int rightRows = transposeB ? B_cols : B_rows;
1294  const int rightCols = transposeB ? B_rows : B_cols;
1295 
1296  INTREPID2_TEST_FOR_EXCEPTION(leftCols != rightRows, std::invalid_argument, "incompatible matrix dimensions");
1297 
1298  Kokkos::Array<int,7> resultExtents; // logical extents
1299  Kokkos::Array<DataVariationType,7> resultVariationTypes; // for each dimension, whether the data varies in that dimension
1300 
1301  resultExtents[D1_DIM] = leftRows;
1302  resultExtents[D2_DIM] = rightCols;
1303  int resultBlockPlusDiagonalLastNonDiagonal = -1;
1304  if ( (A_MatData.getVariationTypes()[D1_DIM] == BLOCK_PLUS_DIAGONAL) && (B_MatData.getVariationTypes()[D1_DIM] == BLOCK_PLUS_DIAGONAL) )
1305  {
1306  // diagonal times diagonal is diagonal; the result will have the maximum of A and B's non-diagonal block size
1307  resultVariationTypes[D1_DIM] = BLOCK_PLUS_DIAGONAL;
1308  resultVariationTypes[D2_DIM] = BLOCK_PLUS_DIAGONAL;
1309  resultBlockPlusDiagonalLastNonDiagonal = std::max(A_MatData.blockPlusDiagonalLastNonDiagonal(), B_MatData.blockPlusDiagonalLastNonDiagonal());
1310  }
1311 
1312  const int resultRank = A_MatData.rank();
1313 
1314  auto A_VariationTypes = A_MatData.getVariationTypes();
1315  auto B_VariationTypes = B_MatData.getVariationTypes();
1316 
1317  Kokkos::Array<int,7> resultActiveDims;
1318  Kokkos::Array<int,7> resultDataDims;
1319  int resultNumActiveDims = 0; // how many of the 7 entries are actually filled in
1320  // the following loop is over the dimensions *prior* to matrix dimensions
1321  for (int i=0; i<resultRank-2; i++)
1322  {
1323  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A_MatData.extent_int(i) != B_MatData.extent_int(i), std::invalid_argument, "A and B extents must match in each non-matrix dimension");
1324 
1325  resultExtents[i] = A_MatData.extent_int(i);
1326 
1327  const DataVariationType &A_VariationType = A_VariationTypes[i];
1328  const DataVariationType &B_VariationType = B_VariationTypes[i];
1329 
1330  // BLOCK_PLUS_DIAGONAL should only occur in matData, and only in the matrix (final) dimensions
1331  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A_VariationType == BLOCK_PLUS_DIAGONAL, std::invalid_argument, "unsupported variationType");
1332  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(B_VariationType == BLOCK_PLUS_DIAGONAL, std::invalid_argument, "unsupported variationType");
1333 
1334  int dataSize = 0;
1335  DataVariationType resultVariationType;
1336  if ((A_VariationType == GENERAL) || (B_VariationType == GENERAL))
1337  {
1338  resultVariationType = GENERAL;
1339  dataSize = resultExtents[i];
1340  }
1341  else if ((B_VariationType == CONSTANT) && (A_VariationType == CONSTANT))
1342  {
1343  resultVariationType = CONSTANT;
1344  dataSize = 1;
1345  }
1346  else if ((B_VariationType == MODULAR) && (A_VariationType == CONSTANT))
1347  {
1348  resultVariationType = MODULAR;
1349  dataSize = B_MatData.getVariationModulus(i);
1350  }
1351  else if ((B_VariationType == CONSTANT) && (A_VariationType == MODULAR))
1352  {
1353  resultVariationType = MODULAR;
1354  dataSize = A_MatData.getVariationModulus(i);
1355  }
1356  else
1357  {
1358  // both are modular. We allow this if they agree on the modulus
1359  auto A_Modulus = A_MatData.getVariationModulus(i);
1360  auto B_Modulus = B_MatData.getVariationModulus(i);
1361  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A_Modulus != B_Modulus, std::invalid_argument, "If both matrices have variation type MODULAR, they must agree on the modulus");
1362  resultVariationType = MODULAR;
1363  dataSize = A_Modulus;
1364  }
1365  resultVariationTypes[i] = resultVariationType;
1366 
1367  if (resultVariationType != CONSTANT)
1368  {
1369  resultActiveDims[resultNumActiveDims] = i;
1370  resultDataDims[resultNumActiveDims] = dataSize;
1371  resultNumActiveDims++;
1372  }
1373  }
1374 
1375  // set things for final dimensions:
1376  resultExtents[D1_DIM] = leftRows;
1377  resultExtents[D2_DIM] = rightCols;
1378 
1379  if ( (A_MatData.getVariationTypes()[D1_DIM] == BLOCK_PLUS_DIAGONAL) && (B_MatData.getVariationTypes()[D1_DIM] == BLOCK_PLUS_DIAGONAL) )
1380  {
1381  // diagonal times diagonal is diagonal; the result will have the maximum of A and B's non-diagonal block size
1382  resultVariationTypes[D1_DIM] = BLOCK_PLUS_DIAGONAL;
1383  resultVariationTypes[D2_DIM] = BLOCK_PLUS_DIAGONAL;
1384  resultBlockPlusDiagonalLastNonDiagonal = std::max(A_MatData.blockPlusDiagonalLastNonDiagonal(), B_MatData.blockPlusDiagonalLastNonDiagonal());
1385 
1386  resultActiveDims[resultNumActiveDims] = resultRank - 2;
1387 
1388  const int numDiagonalEntries = leftRows - (resultBlockPlusDiagonalLastNonDiagonal + 1);
1389  const int numNondiagonalEntries = (resultBlockPlusDiagonalLastNonDiagonal + 1) * (resultBlockPlusDiagonalLastNonDiagonal + 1);
1390 
1391  resultDataDims[resultNumActiveDims] = numDiagonalEntries + numNondiagonalEntries;
1392  resultNumActiveDims++;
1393  }
1394  else
1395  {
1396  // pretty much the only variation types that make sense for matrix dims are GENERAL and BLOCK_PLUS_DIAGONAL
1397  resultVariationTypes[D1_DIM] = GENERAL;
1398  resultVariationTypes[D2_DIM] = GENERAL;
1399 
1400  resultActiveDims[resultNumActiveDims] = resultRank - 2;
1401  resultActiveDims[resultNumActiveDims+1] = resultRank - 1;
1402 
1403  resultDataDims[resultNumActiveDims] = leftRows;
1404  resultDataDims[resultNumActiveDims+1] = rightCols;
1405  resultNumActiveDims += 2;
1406  }
1407 
1408  for (int i=resultRank; i<7; i++)
1409  {
1410  resultVariationTypes[i] = CONSTANT;
1411  resultExtents[i] = 1;
1412  }
1413 
1414  ScalarView<DataScalar,DeviceType> data; // new view will match this one in layout and fad dimension, if any
1415  auto viewToMatch = A_MatData.getUnderlyingView();
1416  if (resultNumActiveDims == 1)
1417  {
1418  data = getMatchingViewWithLabel(viewToMatch, "Data mat-mat result", resultDataDims[0]);
1419  }
1420  else if (resultNumActiveDims == 2)
1421  {
1422  data = getMatchingViewWithLabel(viewToMatch, "Data mat-mat result", resultDataDims[0], resultDataDims[1]);
1423  }
1424  else if (resultNumActiveDims == 3)
1425  {
1426  data = getMatchingViewWithLabel(viewToMatch, "Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2]);
1427  }
1428  else if (resultNumActiveDims == 4)
1429  {
1430  data = getMatchingViewWithLabel(viewToMatch, "Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2],
1431  resultDataDims[3]);
1432  }
1433  else if (resultNumActiveDims == 5)
1434  {
1435  data = getMatchingViewWithLabel(viewToMatch, "Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2],
1436  resultDataDims[3], resultDataDims[4]);
1437  }
1438  else if (resultNumActiveDims == 6)
1439  {
1440  data = getMatchingViewWithLabel(viewToMatch, "Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2],
1441  resultDataDims[3], resultDataDims[4], resultDataDims[5]);
1442  }
1443  else // resultNumActiveDims == 7
1444  {
1445  data = getMatchingViewWithLabel(viewToMatch, "Data mat-mat result", resultDataDims[0], resultDataDims[1], resultDataDims[2],
1446  resultDataDims[3], resultDataDims[4], resultDataDims[5], resultDataDims[6]);
1447  }
1448 
1449  return Data<DataScalar,DeviceType>(data,resultRank,resultExtents,resultVariationTypes,resultBlockPlusDiagonalLastNonDiagonal);
1450  }
1451 
1459  {
1460  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A.rank() != B.rank(), std::invalid_argument, "A and B must have the same logical shape");
1461  const int rank = A.rank();
1462  const int resultRank = rank - numContractionDims;
1463  std::vector<DimensionInfo> dimInfo(resultRank);
1464  for (int d=0; d<resultRank; d++)
1465  {
1466  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A.extent_int(d) != B.extent_int(d), std::invalid_argument, "A and B must have the same logical shape");
1467  dimInfo[d] = A.combinedDataDimensionInfo(B, d);
1468  }
1469  Data<DataScalar,DeviceType> result(dimInfo);
1470  return result;
1471  }
1472 
1479  {
1480  return allocateContractionResult(A, B, 1);
1481  }
1482 
1485  static Data<DataScalar,DeviceType> allocateMatVecResult( const Data<DataScalar,DeviceType> &matData, const Data<DataScalar,DeviceType> &vecData, const bool transposeMatrix = false )
1486  {
1487  // we treat last two logical dimensions of matData as the matrix; last dimension of vecData as the vector
1488  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(matData.rank() != vecData.rank() + 1, std::invalid_argument, "matData and vecData have incompatible ranks");
1489  const int vecDim = vecData.extent_int(vecData.rank() - 1);
1490 
1491  const int D1_DIM = matData.rank() - 2;
1492  const int D2_DIM = matData.rank() - 1;
1493 
1494  const int matRows = matData.extent_int(D1_DIM);
1495  const int matCols = matData.extent_int(D2_DIM);
1496 
1497  const int rows = transposeMatrix ? matCols : matRows;
1498  const int cols = transposeMatrix ? matRows : matCols;
1499 
1500  const int resultRank = vecData.rank();
1501 
1502  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(cols != vecDim, std::invalid_argument, "matData column count != vecData dimension");
1503 
1504  Kokkos::Array<int,7> resultExtents; // logical extents
1505  Kokkos::Array<DataVariationType,7> resultVariationTypes; // for each dimension, whether the data varies in that dimension
1506  auto vecVariationTypes = vecData.getVariationTypes();
1507  auto matVariationTypes = matData.getVariationTypes();
1508 
1509  Kokkos::Array<int,7> resultActiveDims;
1510  Kokkos::Array<int,7> resultDataDims;
1511  int resultNumActiveDims = 0; // how many of the 7 entries are actually filled in
1512  // the following loop is over the dimensions *prior* to matrix/vector dimensions
1513  for (int i=0; i<resultRank-1; i++)
1514  {
1515  resultExtents[i] = vecData.extent_int(i);
1516  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(vecData.extent_int(i) != matData.extent_int(i), std::invalid_argument, "matData and vecData extents must match in each non-matrix/vector dimension");
1517 
1518  const DataVariationType &vecVariationType = vecVariationTypes[i];
1519  const DataVariationType &matVariationType = matVariationTypes[i];
1520 
1521  // BLOCK_PLUS_DIAGONAL should only occur in matData, and only in the matrix (final) dimensions
1522  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(vecVariationType == BLOCK_PLUS_DIAGONAL, std::invalid_argument, "unsupported variationType");
1523  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(matVariationType == BLOCK_PLUS_DIAGONAL, std::invalid_argument, "unsupported variationType");
1524 
1525  int dataSize = 0;
1526  DataVariationType resultVariationType;
1527  if ((vecVariationType == GENERAL) || (matVariationType == GENERAL))
1528  {
1529  resultVariationType = GENERAL;
1530  dataSize = resultExtents[i];
1531  }
1532  else if ((matVariationType == CONSTANT) && (vecVariationType == CONSTANT))
1533  {
1534  resultVariationType = CONSTANT;
1535  dataSize = 1;
1536  }
1537  else if ((matVariationType == MODULAR) && (vecVariationType == CONSTANT))
1538  {
1539  resultVariationType = MODULAR;
1540  dataSize = matData.getVariationModulus(i);
1541  }
1542  else if ((matVariationType == CONSTANT) && (vecVariationType == MODULAR))
1543  {
1544  resultVariationType = MODULAR;
1545  dataSize = matData.getVariationModulus(i);
1546  }
1547  else
1548  {
1549  // both are modular. We allow this if they agree on the modulus
1550  auto matModulus = matData.getVariationModulus(i);
1551  auto vecModulus = vecData.getVariationModulus(i);
1552  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(matModulus != vecModulus, std::invalid_argument, "If matrix and vector both have variation type MODULAR, they must agree on the modulus");
1553  resultVariationType = MODULAR;
1554  dataSize = matModulus;
1555  }
1556  resultVariationTypes[i] = resultVariationType;
1557 
1558  if (resultVariationType != CONSTANT)
1559  {
1560  resultActiveDims[resultNumActiveDims] = i;
1561  resultDataDims[resultNumActiveDims] = dataSize;
1562  resultNumActiveDims++;
1563  }
1564  }
1565  // for the final dimension, the variation type is always GENERAL
1566  // (Some combinations, e.g. CONSTANT/CONSTANT *would* generate a CONSTANT result, but constant matrices don't make a lot of sense beyond 1x1 matrices…)
1567  resultActiveDims[resultNumActiveDims] = resultRank - 1;
1568  resultDataDims[resultNumActiveDims] = rows;
1569  resultNumActiveDims++;
1570 
1571  for (int i=resultRank; i<7; i++)
1572  {
1573  resultVariationTypes[i] = CONSTANT;
1574  resultExtents[i] = 1;
1575  }
1576  resultVariationTypes[resultRank-1] = GENERAL;
1577  resultExtents[resultRank-1] = rows;
1578 
1579  ScalarView<DataScalar,DeviceType> data;
1580  if (resultNumActiveDims == 1)
1581  {
1582  data = matData.allocateDynRankViewMatchingUnderlying(resultDataDims[0]);
1583  }
1584  else if (resultNumActiveDims == 2)
1585  {
1586  data = matData.allocateDynRankViewMatchingUnderlying(resultDataDims[0], resultDataDims[1]);
1587  }
1588  else if (resultNumActiveDims == 3)
1589  {
1590  data = matData.allocateDynRankViewMatchingUnderlying(resultDataDims[0], resultDataDims[1], resultDataDims[2]);
1591  }
1592  else if (resultNumActiveDims == 4)
1593  {
1594  data = matData.allocateDynRankViewMatchingUnderlying(resultDataDims[0], resultDataDims[1], resultDataDims[2],
1595  resultDataDims[3]);
1596  }
1597  else if (resultNumActiveDims == 5)
1598  {
1599  data = matData.allocateDynRankViewMatchingUnderlying(resultDataDims[0], resultDataDims[1], resultDataDims[2],
1600  resultDataDims[3], resultDataDims[4]);
1601  }
1602  else if (resultNumActiveDims == 6)
1603  {
1604  data = matData.allocateDynRankViewMatchingUnderlying(resultDataDims[0], resultDataDims[1], resultDataDims[2],
1605  resultDataDims[3], resultDataDims[4], resultDataDims[5]);
1606  }
1607  else // resultNumActiveDims == 7
1608  {
1609  data = matData.allocateDynRankViewMatchingUnderlying(resultDataDims[0], resultDataDims[1], resultDataDims[2],
1610  resultDataDims[3], resultDataDims[4], resultDataDims[5], resultDataDims[6]);
1611  }
1612 
1613  return Data<DataScalar,DeviceType>(data,resultRank,resultExtents,resultVariationTypes);
1614  }
1615 
1617  template<int rank>
1618  enable_if_t<(rank!=1) && (rank!=7), Kokkos::MDRangePolicy<typename DeviceType::execution_space,Kokkos::Rank<rank>> >
1620  {
1621  using ExecutionSpace = typename DeviceType::execution_space;
1622  Kokkos::Array<int,rank> startingOrdinals;
1623  Kokkos::Array<int,rank> extents;
1624 
1625  for (int d=0; d<rank; d++)
1626  {
1627  startingOrdinals[d] = 0;
1628  extents[d] = getDataExtent(d);
1629  }
1630  auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<rank>>(startingOrdinals,extents);
1631  return policy;
1632  }
1633 
1635  template<int rank>
1636  enable_if_t<rank==7, Kokkos::MDRangePolicy<typename DeviceType::execution_space,Kokkos::Rank<6>> >
1638  {
1639  using ExecutionSpace = typename DeviceType::execution_space;
1640  Kokkos::Array<int,6> startingOrdinals;
1641  Kokkos::Array<int,6> extents;
1642 
1643  for (int d=0; d<6; d++)
1644  {
1645  startingOrdinals[d] = 0;
1646  extents[d] = getDataExtent(d);
1647  }
1648  auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<6>>(startingOrdinals,extents);
1649  return policy;
1650  }
1651 
1652  template<int rank>
1653  inline
1654  enable_if_t<rank==1, Kokkos::RangePolicy<typename DeviceType::execution_space> >
1656  {
1657  using ExecutionSpace = typename DeviceType::execution_space;
1658  Kokkos::RangePolicy<ExecutionSpace> policy(ExecutionSpace(),0,getDataExtent(0));
1659  return policy;
1660  }
1661 
1663  Data shallowCopy(const int rank, const Kokkos::Array<int,7> &extents, const Kokkos::Array<DataVariationType,7> &variationTypes) const
1664  {
1665  switch (dataRank_)
1666  {
1667  case 1: return Data(rank, get_fixed_view<View1>(underlyingView_), extents, variationTypes);
1668  case 2: return Data(rank, get_fixed_view<View2>(underlyingView_), extents, variationTypes);
1669  case 3: return Data(rank, get_fixed_view<View3>(underlyingView_), extents, variationTypes);
1670  case 4: return Data(rank, get_fixed_view<View4>(underlyingView_), extents, variationTypes);
1671  case 5: return Data(rank, get_fixed_view<View5>(underlyingView_), extents, variationTypes);
1672  case 6: return Data(rank, get_fixed_view<View6>(underlyingView_), extents, variationTypes);
1673  case 7: return Data(rank, get_fixed_view<View7>(underlyingView_), extents, variationTypes);
1674  default:
1675  INTREPID2_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Unhandled dataRank_");
1676  }
1677  }
1678 
1681  {
1682  const int D_DIM = A.rank() - 1;
1683  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A.extent_int(D_DIM) != B.extent_int(D_DIM), std::invalid_argument, "A and B have different extents");
1684  const int vectorComponents = A.extent_int(D_DIM);
1685 
1686  // shallow copy of this to avoid implicit references to this in call to getWritableEntry() below
1687  Data<DataScalar,DeviceType> thisData = *this;
1688 
1689  using ExecutionSpace = typename DeviceType::execution_space;
1690  // note the use of getDataExtent() below: we only range over the possibly-distinct entries
1691  if (rank_ == 1) // contraction result rank; e.g., (P)
1692  {
1693  Kokkos::parallel_for("compute dot product", getDataExtent(0),
1694  KOKKOS_LAMBDA (const int &pointOrdinal) {
1695  auto & val = thisData.getWritableEntry(pointOrdinal);
1696  val = 0;
1697  for (int i=0; i<vectorComponents; i++)
1698  {
1699  val += A(pointOrdinal,i) * B(pointOrdinal,i);
1700  }
1701  });
1702  }
1703  else if (rank_ == 2) // contraction result rank; e.g., (C,P)
1704  {
1705  // typical case for e.g. gradient data: (C,P,D)
1706  auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<2>>({0,0},{getDataExtent(0),getDataExtent(1)});
1707  Kokkos::parallel_for("compute dot product", policy,
1708  KOKKOS_LAMBDA (const int &cellOrdinal, const int &pointOrdinal) {
1709  auto & val = thisData.getWritableEntry(cellOrdinal, pointOrdinal);
1710  val = 0;
1711  for (int i=0; i<vectorComponents; i++)
1712  {
1713  val += A(cellOrdinal,pointOrdinal,i) * B(cellOrdinal,pointOrdinal,i);
1714  }
1715  });
1716  }
1717  else if (rank_ == 3)
1718  {
1719  auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<3>>({0,0,0},{getDataExtent(0),getDataExtent(1),getDataExtent(2)});
1720  Kokkos::parallel_for("compute dot product", policy,
1721  KOKKOS_LAMBDA (const int &cellOrdinal, const int &pointOrdinal, const int &d) {
1722  auto & val = thisData.getWritableEntry(cellOrdinal, pointOrdinal,d);
1723  val = 0;
1724  for (int i=0; i<vectorComponents; i++)
1725  {
1726  val += A(cellOrdinal,pointOrdinal,d,i) * B(cellOrdinal,pointOrdinal,d,i);
1727  }
1728  });
1729  }
1730  else
1731  {
1732  // TODO: handle other cases
1733  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(true, std::logic_error, "rank not yet supported");
1734  }
1735  }
1736 
1738  template<class BinaryOperator>
1739  void storeInPlaceCombination(const Data<DataScalar,DeviceType> &A, const Data<DataScalar,DeviceType> &B, BinaryOperator binaryOperator);
1740 
1743  {
1745  storeInPlaceCombination(A, B, sum);
1746  }
1747 
1750  {
1752  storeInPlaceCombination(A, B, product);
1753  }
1754 
1757  {
1759  storeInPlaceCombination(A, B, difference);
1760  }
1761 
1764  {
1766  storeInPlaceCombination(A, B, quotient);
1767  }
1768 
1770  void storeMatVec( const Data<DataScalar,DeviceType> &matData, const Data<DataScalar,DeviceType> &vecData, const bool transposeMatrix = false )
1771  {
1772  // TODO: add a compile-time (SFINAE-type) guard against DataScalar types that do not support arithmetic operations. (We support Orientation as a DataScalar type; it might suffice just to compare DataScalar to Orientation, and eliminate this method for that case.)
1773  // TODO: check for invalidly shaped containers.
1774 
1775  const int D1_DIM = matData.rank() - 2;
1776  const int D2_DIM = matData.rank() - 1;
1777 
1778  const int matRows = matData.extent_int(D1_DIM);
1779  const int matCols = matData.extent_int(D2_DIM);
1780 
1781  const int rows = transposeMatrix ? matCols : matRows;
1782  const int cols = transposeMatrix ? matRows : matCols;
1783 
1784  // shallow copy of this to avoid implicit references to this in call to getWritableEntry() below
1785  Data<DataScalar,DeviceType> thisData = *this;
1786 
1787  using ExecutionSpace = typename DeviceType::execution_space;
1788  // note the use of getDataExtent() below: we only range over the possibly-distinct entries
1789  if (rank_ == 3)
1790  {
1791  // typical case for e.g. gradient data: (C,P,D)
1792  auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<3>>({0,0,0},{getDataExtent(0),getDataExtent(1),rows});
1793  Kokkos::parallel_for("compute mat-vec", policy,
1794  KOKKOS_LAMBDA (const int &cellOrdinal, const int &pointOrdinal, const int &i) {
1795  auto & val_i = thisData.getWritableEntry(cellOrdinal, pointOrdinal, i);
1796  val_i = 0;
1797  for (int j=0; j<cols; j++)
1798  {
1799  const auto & mat_ij = transposeMatrix ? matData(cellOrdinal,pointOrdinal,j,i) : matData(cellOrdinal,pointOrdinal,i,j);
1800  val_i += mat_ij * vecData(cellOrdinal,pointOrdinal,j);
1801  }
1802  });
1803  }
1804  else if (rank_ == 2)
1805  {
1806  //
1807  auto policy = Kokkos::MDRangePolicy<ExecutionSpace,Kokkos::Rank<2>>({0,0},{getDataExtent(0),rows});
1808  Kokkos::parallel_for("compute mat-vec", policy,
1809  KOKKOS_LAMBDA (const int &vectorOrdinal, const int &i) {
1810  auto & val_i = thisData.getWritableEntry(vectorOrdinal, i);
1811  val_i = 0;
1812  for (int j=0; j<cols; j++)
1813  {
1814  const auto & mat_ij = transposeMatrix ? matData(vectorOrdinal,j,i) : matData(vectorOrdinal,i,j);
1815  val_i += mat_ij * vecData(vectorOrdinal,j);
1816  }
1817  });
1818  }
1819  else if (rank_ == 1)
1820  {
1821  // single-vector case
1822  Kokkos::RangePolicy<ExecutionSpace> policy(0,rows);
1823  Kokkos::parallel_for("compute mat-vec", policy,
1824  KOKKOS_LAMBDA (const int &i) {
1825  auto & val_i = thisData.getWritableEntry(i);
1826  val_i = 0;
1827  for (int j=0; j<cols; j++)
1828  {
1829  const auto & mat_ij = transposeMatrix ? matData(j,i) : matData(i,j);
1830  val_i += mat_ij * vecData(j);
1831  }
1832  });
1833  }
1834  else
1835  {
1836  // TODO: handle other cases
1837  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(true, std::logic_error, "rank not yet supported");
1838  }
1839  }
1840 
1847  void storeMatMat( const bool transposeA, const Data<DataScalar,DeviceType> &A_MatData, const bool transposeB, const Data<DataScalar,DeviceType> &B_MatData )
1848  {
1849  // TODO: add a compile-time (SFINAE-type) guard against DataScalar types that do not support arithmetic operations. (We support Orientation as a DataScalar type; it might suffice just to compare DataScalar to Orientation, and eliminate this method for that case.)
1850  // TODO: check for invalidly shaped containers.
1851 
1852  // we treat last two logical dimensions of matData as the matrix; last dimension of vecData as the vector
1853  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(A_MatData.rank() != B_MatData.rank(), std::invalid_argument, "AmatData and BmatData have incompatible ranks");
1854 
1855  const int D1_DIM = A_MatData.rank() - 2;
1856  const int D2_DIM = A_MatData.rank() - 1;
1857 
1858  const int A_rows = A_MatData.extent_int(D1_DIM);
1859  const int A_cols = A_MatData.extent_int(D2_DIM);
1860  const int B_rows = B_MatData.extent_int(D1_DIM);
1861  const int B_cols = B_MatData.extent_int(D2_DIM);
1862 
1863  const int leftRows = transposeA ? A_cols : A_rows;
1864  const int leftCols = transposeA ? A_rows : A_cols;
1865  const int rightCols = transposeB ? B_rows : B_cols;
1866 
1867 #ifdef INTREPID2_HAVE_DEBUG
1868  const int rightRows = transposeB ? B_cols : B_rows;
1869  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(leftCols != rightRows, std::invalid_argument, "inner dimensions do not match");
1870 #endif
1871 
1872  // shallow copy of this to avoid implicit references to this in call to getWritableEntry() below
1873  Data<DataScalar,DeviceType> thisData = *this;
1874 
1875  using ExecutionSpace = typename DeviceType::execution_space;
1876 
1877  const int diagonalStart = (variationType_[D1_DIM] == BLOCK_PLUS_DIAGONAL) ? blockPlusDiagonalLastNonDiagonal_ + 1 : leftRows;
1878  // note the use of getDataExtent() below: we only range over the possibly-distinct entries
1879  if (rank_ == 3)
1880  {
1881  // (C,D,D), say
1882  auto policy = Kokkos::RangePolicy<ExecutionSpace>(0,getDataExtent(0));
1883  Kokkos::parallel_for("compute mat-mat", policy,
1884  KOKKOS_LAMBDA (const int &matrixOrdinal) {
1885  for (int i=0; i<diagonalStart; i++)
1886  {
1887  for (int j=0; j<rightCols; j++)
1888  {
1889  auto & val_ij = thisData.getWritableEntry(matrixOrdinal, i, j);
1890  val_ij = 0;
1891  for (int k=0; k<leftCols; k++)
1892  {
1893  const auto & left = transposeA ? A_MatData(matrixOrdinal,k,i) : A_MatData(matrixOrdinal,i,k);
1894  const auto & right = transposeB ? B_MatData(matrixOrdinal,j,k) : B_MatData(matrixOrdinal,k,j);
1895  val_ij += left * right;
1896  }
1897  }
1898  }
1899  for (int i=diagonalStart; i<leftRows; i++)
1900  {
1901  auto & val_ii = thisData.getWritableEntry(matrixOrdinal, i, i);
1902  const auto & left = A_MatData(matrixOrdinal,i,i);
1903  const auto & right = B_MatData(matrixOrdinal,i,i);
1904  val_ii = left * right;
1905  }
1906  });
1907  }
1908  else if (rank_ == 4)
1909  {
1910  // (C,P,D,D), perhaps
1911  auto policy = Kokkos::MDRangePolicy<ExecutionSpace, Kokkos::Rank<2> >({0,0},{getDataExtent(0),getDataExtent(1)});
1912  if (underlyingMatchesLogical_) // receiving data object is completely expanded
1913  {
1914  Kokkos::parallel_for("compute mat-mat", policy,
1915  KOKKOS_LAMBDA (const int &cellOrdinal, const int &pointOrdinal) {
1916  for (int i=0; i<leftRows; i++)
1917  {
1918  for (int j=0; j<rightCols; j++)
1919  {
1920  auto & val_ij = thisData.getUnderlyingView4()(cellOrdinal,pointOrdinal, i, j);
1921  val_ij = 0;
1922  for (int k=0; k<leftCols; k++)
1923  {
1924  const auto & left = transposeA ? A_MatData(cellOrdinal,pointOrdinal,k,i) : A_MatData(cellOrdinal,pointOrdinal,i,k);
1925  const auto & right = transposeB ? B_MatData(cellOrdinal,pointOrdinal,j,k) : B_MatData(cellOrdinal,pointOrdinal,k,j);
1926  val_ij += left * right;
1927  }
1928  }
1929  }
1930  });
1931  }
1932  else
1933  {
1934  Kokkos::parallel_for("compute mat-mat", policy,
1935  KOKKOS_LAMBDA (const int &cellOrdinal, const int &pointOrdinal) {
1936  for (int i=0; i<diagonalStart; i++)
1937  {
1938  for (int j=0; j<rightCols; j++)
1939  {
1940  auto & val_ij = thisData.getWritableEntry(cellOrdinal,pointOrdinal, i, j);
1941  val_ij = 0;
1942  for (int k=0; k<leftCols; k++)
1943  {
1944  const auto & left = transposeA ? A_MatData(cellOrdinal,pointOrdinal,k,i) : A_MatData(cellOrdinal,pointOrdinal,i,k);
1945  const auto & right = transposeB ? B_MatData(cellOrdinal,pointOrdinal,j,k) : B_MatData(cellOrdinal,pointOrdinal,k,j);
1946  val_ij += left * right;
1947  }
1948  }
1949  }
1950  for (int i=diagonalStart; i<leftRows; i++)
1951  {
1952  auto & val_ii = thisData.getWritableEntry(cellOrdinal,pointOrdinal, i, i);
1953  const auto & left = A_MatData(cellOrdinal,pointOrdinal,i,i);
1954  const auto & right = B_MatData(cellOrdinal,pointOrdinal,i,i);
1955  val_ii = left * right;
1956  }
1957  });
1958  }
1959  }
1960  else
1961  {
1962  // TODO: handle other cases
1963  INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(true, std::logic_error, "rank not yet supported");
1964  }
1965  }
1966 
1968  KOKKOS_INLINE_FUNCTION constexpr bool isValid() const
1969  {
1970  return extents_[0] > 0;
1971  }
1972 
1974  KOKKOS_INLINE_FUNCTION
1975  unsigned rank() const
1976  {
1977  return rank_;
1978  }
1979 
1986  void setExtent(const ordinal_type &d, const ordinal_type &newExtent)
1987  {
1988  INTREPID2_TEST_FOR_EXCEPTION(variationType_[d] == BLOCK_PLUS_DIAGONAL, std::invalid_argument, "setExtent is not supported for BLOCK_PLUS_DIAGONAL dimensions");
1989 
1990  if (variationType_[d] == MODULAR)
1991  {
1992  bool dividesEvenly = ((newExtent / variationModulus_[d]) * variationModulus_[d] == newExtent);
1993  INTREPID2_TEST_FOR_EXCEPTION(!dividesEvenly, std::invalid_argument, "when setExtent is called on dimenisions with MODULAR variation, the modulus must divide the new extent evenly");
1994  }
1995 
1996  if ((newExtent != extents_[d]) && (variationType_[d] == GENERAL))
1997  {
1998  // then we need to resize; let's determine the full set of new extents
1999  std::vector<ordinal_type> newExtents(dataRank_,-1);
2000  for (int r=0; r<dataRank_; r++)
2001  {
2002  if (activeDims_[r] == d)
2003  {
2004  // this is the changed dimension
2005  newExtents[r] = newExtent;
2006  }
2007  else
2008  {
2009  // unchanged; copy from existing
2010  newExtents[r] = getUnderlyingViewExtent(r);
2011  }
2012  }
2013 
2014  switch (dataRank_)
2015  {
2016  case 1: Kokkos::resize(std::get<View1>(underlyingView_),newExtents[0]);
2017  break;
2018  case 2: Kokkos::resize(std::get<View2>(underlyingView_),newExtents[0],newExtents[1]);
2019  break;
2020  case 3: Kokkos::resize(std::get<View3>(underlyingView_),newExtents[0],newExtents[1],newExtents[2]);
2021  break;
2022  case 4: Kokkos::resize(std::get<View4>(underlyingView_),newExtents[0],newExtents[1],newExtents[2],newExtents[3]);
2023  break;
2024  case 5: Kokkos::resize(std::get<View5>(underlyingView_),newExtents[0],newExtents[1],newExtents[2],newExtents[3],newExtents[4]);
2025  break;
2026  case 6: Kokkos::resize(std::get<View6>(underlyingView_),newExtents[0],newExtents[1],newExtents[2],newExtents[3],newExtents[4],newExtents[5]);
2027  break;
2028  case 7: Kokkos::resize(std::get<View7>(underlyingView_),newExtents[0],newExtents[1],newExtents[2],newExtents[3],newExtents[4],newExtents[5],newExtents[6]);
2029  break;
2030  default: INTREPID2_TEST_FOR_EXCEPTION(true, std::logic_error, "Unexpected dataRank_ value");
2031  }
2032  }
2033 
2034  extents_[d] = newExtent;
2035  }
2036 
2038  KOKKOS_INLINE_FUNCTION
2040  {
2041  return underlyingMatchesLogical_;
2042  }
2043  };
2044 
2045  template<class DataScalar, typename DeviceType>
2046  KOKKOS_INLINE_FUNCTION constexpr unsigned rank(const Data<DataScalar, DeviceType>& D) {
2047  return D.rank();
2048  }
2049 }
2050 
2051 // we do ETI for doubles and default ExecutionSpace's device_type
2053 
2054 #include "Intrepid2_DataDef.hpp"
2055 
2056 #endif /* Intrepid2_Data_h */
enable_if_t< rank==7, Kokkos::MDRangePolicy< typename DeviceType::execution_space, Kokkos::Rank< 6 > > > dataExtentRangePolicy()
returns an MDRangePolicy over the first six underlying data extents (but with the logical shape)...
void clear() const
Copies 0.0 to the underlying View.
void copyDataFromDynRankViewMatchingUnderlying(const ScalarView< DataScalar, DeviceType > &dynRankView) const
Copies from the provided DynRankView into the underlying Kokkos::View container storing the unique da...
KOKKOS_INLINE_FUNCTION reference_type getWritableEntry(const IntArgs...intArgs) const
Returns an l-value reference to the specified logical entry in the underlying view. Note that for variation types other than GENERAL, multiple valid argument sets will refer to the same memory location. Intended for Intrepid2 developers and expert users only. If passThroughBlockDiagonalArgs is TRUE, the corresponding arguments are interpreted as entries in the 1D packed matrix rather than as logical 2D matrix row and column.
KOKKOS_INLINE_FUNCTION ordinal_type getUnderlyingViewRank() const
returns the rank of the View that stores the unique data
Header file with various static argument-extractor classes. These are useful for writing efficient...
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar *******, DeviceType > & getUnderlyingView7() const
returns the View that stores the unique data. For rank-7 underlying containers.
ScalarView< DataScalar, DeviceType > allocateDynRankViewMatchingUnderlying(DimArgs...dims) const
Returns a DynRankView that matches the underlying Kokkos::View object value_type and layout...
void applyOperator(UnaryOperator unaryOperator)
applies the specified unary operator to each entry
static KOKKOS_INLINE_FUNCTION int blockPlusDiagonalDiagonalEntryIndex(const int &lastNondiagonal, const int &numNondiagonalEntries, const int &i)
Returns flattened index of the specified (i,i) matrix entry, assuming that i &gt; lastNondiagonal. Only applicable for BLOCK_PLUS_DIAGONAL DataVariationType.
void storeDotProduct(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
Places the result of a contraction along the final dimension of A and B into this data container...
static KOKKOS_INLINE_FUNCTION int blockPlusDiagonalBlockEntryIndex(const int &lastNondiagonal, const int &numNondiagonalEntries, const int &i, const int &j)
//! Returns flattened index of the specified (i,j) matrix entry, assuming that i,j ≤ lastNondiagonal...
static Data< DataScalar, DeviceType > allocateInPlaceCombinationResult(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
static Data< DataScalar, DeviceType > allocateMatVecResult(const Data< DataScalar, DeviceType > &matData, const Data< DataScalar, DeviceType > &vecData, const bool transposeMatrix=false)
void storeMatVec(const Data< DataScalar, DeviceType > &matData, const Data< DataScalar, DeviceType > &vecData, const bool transposeMatrix=false)
Places the result of a matrix-vector multiply corresponding to the two provided containers into this ...
Data(const unsigned rank, Kokkos::View< ViewScalar, DeviceType, ViewProperties...> data, Kokkos::Array< int, 7 > extents, Kokkos::Array< DataVariationType, 7 > variationType, const int blockPlusDiagonalLastNonDiagonal=-1)
constructor with run-time rank (requires full-length extents, variationType inputs; those beyond the ...
KOKKOS_INLINE_FUNCTION enable_if_t< rank==6, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 6...
KOKKOS_INLINE_FUNCTION enable_if_t< rank==5, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 5...
static void copyContainer(ToContainer to, FromContainer from)
Generic data copying method to allow construction of Data object from DynRankViews for which deep_cop...
#define INTREPID2_TEST_FOR_EXCEPTION_DEVICE_SAFE(test, x, msg)
KOKKOS_INLINE_FUNCTION int getVariationModulus(const int &d) const
Variation modulus accessor.
KOKKOS_INLINE_FUNCTION enable_if_t< rank==7, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 7...
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar *****, DeviceType > & getUnderlyingView5() const
returns the View that stores the unique data. For rank-5 underlying containers.
KOKKOS_INLINE_FUNCTION DimensionInfo combinedDataDimensionInfo(const Data &otherData, const int &dim) const
Returns (DataVariationType, data extent) in the specified dimension for a Data container that combine...
ScalarView< DataScalar, DeviceType > getUnderlyingView() const
Returns a DynRankView constructed atop the same underlying data as the fixed-rank Kokkos::View used i...
KOKKOS_INLINE_FUNCTION Kokkos::Array< int, 7 > getExtents() const
Returns an array containing the logical extents in each dimension.
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar *, DeviceType > & getUnderlyingView1() const
returns the View that stores the unique data. For rank-1 underlying containers.
KOKKOS_INLINE_FUNCTION const Kokkos::Array< DataVariationType, 7 > & getVariationTypes() const
Returns an array with the variation types in each logical dimension.
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar ******, DeviceType > & getUnderlyingView6() const
returns the View that stores the unique data. For rank-6 underlying containers.
KOKKOS_INLINE_FUNCTION enable_if_t< rank==1, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 1...
Defines implementations for the Data class that are not present in the declaration.
KOKKOS_INLINE_FUNCTION bool underlyingMatchesLogical() const
Returns true if the underlying container has exactly the same rank and extents as the logical contain...
Wrapper around a Kokkos::View that allows data that is constant or repeating in various logical dimen...
void storeInPlaceCombination(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B, BinaryOperator binaryOperator)
Places the result of an in-place combination (e.g., entrywise sum) into this data container...
Header function for Intrepid2::Util class and other utility functions.
KOKKOS_INLINE_FUNCTION bool isDiagonal() const
returns true for containers that have two dimensions marked as BLOCK_PLUS_DIAGONAL for which the non-...
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar ****, DeviceType > & getUnderlyingView4() const
returns the View that stores the unique data. For rank-4 underlying containers.
Data(std::vector< DimensionInfo > dimInfoVector)
Constructor in terms of DimensionInfo for each logical dimension; does not require a View to be speci...
Struct expressing all variation information about a Data object in a single dimension, including its logical extent and storage extent.
KOKKOS_INLINE_FUNCTION return_type getEntryWithPassThroughOption(const bool &passThroughBlockDiagonalArgs, const IntArgs &...intArgs) const
Returns a (read-only) value corresponding to the specified logical data location. If passThroughBlock...
Data shallowCopy(const int rank, const Kokkos::Array< int, 7 > &extents, const Kokkos::Array< DataVariationType, 7 > &variationTypes) const
Creates a new Data object with the same underlying view, but with the specified logical rank...
void storeInPlaceDifference(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
stores the in-place (entrywise) difference, A .- B, into this container.
Defines functors for use with Data objects: so far, we include simple arithmetical functors for sum...
KOKKOS_INLINE_FUNCTION DimensionInfo getDimensionInfo(const int &dim) const
Returns an object fully specifying the indicated dimension. This is used in determining appropriate s...
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar ***, DeviceType > & getUnderlyingView3() const
returns the View that stores the unique data. For rank-3 underlying containers.
Data(DataScalar constantValue, Kokkos::Array< int, rank > extents)
constructor for everywhere-constant data
enable_if_t<(rank!=1)&&(rank!=7), Kokkos::MDRangePolicy< typename DeviceType::execution_space, Kokkos::Rank< rank > > > dataExtentRangePolicy()
returns an MDRangePolicy over the underlying data extents (but with the logical shape).
KOKKOS_INLINE_FUNCTION reference_type getWritableEntryWithPassThroughOption(const bool &passThroughBlockDiagonalArgs, const IntArgs...intArgs) const
Returns an l-value reference to the specified logical entry in the underlying view. Note that for variation types other than GENERAL, multiple valid argument sets will refer to the same memory location. Intended for Intrepid2 developers and expert users only. If passThroughBlockDiagonalArgs is TRUE, the corresponding arguments are interpreted as entries in the 1D packed matrix rather than as logical 2D matrix row and column.
Data< DataScalar, DeviceType > allocateConstantData(const DataScalar &value)
KOKKOS_INLINE_FUNCTION constexpr bool isValid() const
returns true for containers that have data; false for those that don&#39;t (namely, those that have been ...
KOKKOS_INLINE_FUNCTION enable_if_t< valid_args< IntArgs...>::value &&(sizeof...(IntArgs)<=7), return_type > operator()(const IntArgs &...intArgs) const
Returns a value corresponding to the specified logical data location.
ScalarView< DataScalar, DeviceType > allocateDynRankViewMatchingUnderlying() const
Returns a DynRankView that matches the underlying Kokkos::View object in value_type, layout, and dimension.
void storeInPlaceQuotient(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
stores the in-place (entrywise) quotient, A ./ B, into this container.
static Data< DataScalar, DeviceType > allocateDotProductResult(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
void storeInPlaceSum(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
stores the in-place (entrywise) sum, A .+ B, into this container.
Data(const ScalarView< DataScalar, DeviceType > &data, int rank, Kokkos::Array< int, 7 > extents, Kokkos::Array< DataVariationType, 7 > variationType, const int blockPlusDiagonalLastNonDiagonal=-1)
DynRankView constructor. Will copy to a View of appropriate rank.
void storeMatMat(const bool transposeA, const Data< DataScalar, DeviceType > &A_MatData, const bool transposeB, const Data< DataScalar, DeviceType > &B_MatData)
KOKKOS_INLINE_FUNCTION ordinal_type getUnderlyingViewSize() const
returns the number of entries in the View that stores the unique data
void allocateAndCopyFromDynRankView(ScalarView< DataScalar, DeviceType > data)
allocate an underlying View that matches the provided DynRankView in dimensions, and copy...
static KOKKOS_INLINE_FUNCTION int blockPlusDiagonalNumNondiagonalEntries(const int &lastNondiagonal)
Returns the number of non-diagonal entries based on the last non-diagonal. Only applicable for BLOCK_...
KOKKOS_INLINE_FUNCTION enable_if_t< rank==3, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 3...
void setActiveDims()
class initialization method. Called by constructors.
Data()
default constructor (empty data)
void setExtent(const ordinal_type &d, const ordinal_type &newExtent)
sets the logical extent in the specified dimension. If needed, the underlying data container is resiz...
A singleton class for a DynRankView containing exactly one zero entry. (Technically, the entry is DataScalar(), the default value for the scalar type.) This allows View-wrapping classes to return a reference to zero, even when that zero is not explicitly stored in the wrapped views.
void storeInPlaceProduct(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B)
stores the in-place (entrywise) product, A .* B, into this container.
KOKKOS_INLINE_FUNCTION enable_if_t< rank==4, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 4...
KOKKOS_INLINE_FUNCTION unsigned rank() const
Returns the logical rank of the Data container.
KOKKOS_INLINE_FUNCTION int extent_int(const int &r) const
Returns the logical extent in the specified dimension.
Defines DimensionInfo struct that allows specification of a dimension within a Data object...
Defines DataVariationType enum that specifies the types of variation possible within a Data object...
KOKKOS_INLINE_FUNCTION enable_if_t< rank==2, const Kokkos::View< typename RankExpander< DataScalar, rank >::value_type, DeviceType > & > getUnderlyingView() const
Returns the underlying view. Throws an exception if the underlying view is not rank 2...
KOKKOS_INLINE_FUNCTION int getDataExtent(const ordinal_type &d) const
returns the true extent of the data corresponding to the logical dimension provided; if the data does...
KOKKOS_INLINE_FUNCTION int getUnderlyingViewExtent(const int &dim) const
Returns the extent of the underlying view in the specified dimension.
Data(ScalarView< DataScalar, DeviceType > data)
copy constructor modeled after the copy-like constructor above. Not as efficient as the implicit copy...
static Data< DataScalar, DeviceType > allocateContractionResult(const Data< DataScalar, DeviceType > &A, const Data< DataScalar, DeviceType > &B, const int &numContractionDims)
KOKKOS_INLINE_FUNCTION const int & blockPlusDiagonalLastNonDiagonal() const
For a Data object containing data with variation type BLOCK_PLUS_DIAGONAL, returns the row and column...
KOKKOS_INLINE_FUNCTION const Kokkos::View< DataScalar **, DeviceType > & getUnderlyingView2() const
returns the View that stores the unique data. For rank-2 underlying containers.
KOKKOS_INLINE_FUNCTION return_type getEntry(const IntArgs &...intArgs) const
Returns a (read-only) value corresponding to the specified logical data location. ...
static Data< DataScalar, DeviceType > allocateMatMatResult(const bool transposeA, const Data< DataScalar, DeviceType > &A_MatData, const bool transposeB, const Data< DataScalar, DeviceType > &B_MatData)
Data(const Data< DataScalar, OtherDeviceType > &data)
copy-like constructor for differing execution spaces. This does a deep_copy of the underlying view...