40 #ifndef TPETRA_DETAILS_WRAPPEDDUALVIEW_HPP
41 #define TPETRA_DETAILS_WRAPPEDDUALVIEW_HPP
43 #include <Tpetra_Access.hpp>
44 #include <Tpetra_Details_temporaryViewUtils.hpp>
45 #include <Kokkos_DualView.hpp>
46 #include "Teuchos_TestForException.hpp"
52 #ifdef DEBUG_UVM_REMOVAL
54 #define DEBUG_UVM_REMOVAL_ARGUMENT ,const char* callerstr = __builtin_FUNCTION(),const char * filestr=__builtin_FILE(),const int linnum = __builtin_LINE()
56 #define DEBUG_UVM_REMOVAL_PRINT_CALLER(fn) \
58 auto envVarSet = std::getenv("TPETRA_UVM_REMOVAL"); \
59 if (envVarSet && (std::strcmp(envVarSet,"1") == 0)) \
60 std::cout << (fn) << " called from " << callerstr \
61 << " at " << filestr << ":"<<linnum \
62 << " host cnt " << dualView.h_view.use_count() \
63 << " device cnt " << dualView.d_view.use_count() \
69 #define DEBUG_UVM_REMOVAL_ARGUMENT
70 #define DEBUG_UVM_REMOVAL_PRINT_CALLER(fn)
78 template<
typename SC,
typename LO,
typename GO,
typename NO>
88 template <
typename DualViewType>
90 using valueType =
typename DualViewType::value_type;
91 using constValueType =
typename DualViewType::const_value_type;
92 static constexpr
bool value = std::is_same<valueType, constValueType>::value;
95 template <
typename DualViewType>
96 using enableIfConstData = std::enable_if_t<hasConstData<DualViewType>::value>;
98 template <
typename DualViewType>
99 using enableIfNonConstData = std::enable_if_t<!hasConstData<DualViewType>::value>;
101 template <
typename DualViewType>
102 enableIfNonConstData<DualViewType>
103 sync_host(DualViewType dualView) {
105 dualView.sync_host();
108 template <
typename DualViewType>
109 enableIfConstData<DualViewType>
110 sync_host(DualViewType dualView) { }
112 template <
typename DualViewType>
113 enableIfNonConstData<DualViewType>
114 sync_device(DualViewType dualView) {
116 dualView.sync_device();
119 template <
typename DualViewType>
120 enableIfConstData<DualViewType>
121 sync_device(DualViewType dualView) { }
142 template <
typename DualViewType>
146 using DVT = DualViewType;
147 using t_host =
typename DualViewType::t_host;
148 using t_dev =
typename DualViewType::t_dev;
150 using HostType =
typename t_host::device_type;
151 using DeviceType =
typename t_dev::device_type;
154 static constexpr
bool dualViewHasNonConstData = !impl::hasConstData<DualViewType>::value;
155 static constexpr
bool deviceMemoryIsHostAccessible =
156 Kokkos::SpaceAccessibility<Kokkos::DefaultHostExecutionSpace, typename t_dev::memory_space>::accessible;
166 : originalDualView(dualV),
167 dualView(originalDualView)
171 template <
class SrcDualViewType>
173 : originalDualView(src.originalDualView),
174 dualView(src.dualView)
178 template <
class SrcDualViewType>
180 originalDualView = src.originalDualView;
181 dualView = src.dualView;
191 : originalDualView(origDualV),
196 WrappedDualView(
const t_dev deviceView) {
197 TEUCHOS_TEST_FOR_EXCEPTION(
198 deviceView.data() !=
nullptr && deviceView.use_count() == 0,
199 std::invalid_argument,
200 "Tpetra::Details::WrappedDualView: cannot construct with a device view that\n"
201 "does not own its memory (i.e. constructed with a raw pointer and dimensions)\n"
202 "because the WrappedDualView needs to assume ownership of the memory.");
206 if(deviceView.use_count() != 0)
208 hostView = Kokkos::create_mirror_view(
209 Kokkos::WithoutInitializing,
210 typename t_host::memory_space(),
213 originalDualView = DualViewType(deviceView, hostView);
214 originalDualView.clear_sync_state();
215 originalDualView.modify_device();
216 dualView = originalDualView;
220 WrappedDualView(
const WrappedDualView parent,
int offset,
int numEntries) {
221 originalDualView = parent.originalDualView;
222 dualView = getSubview(parent.dualView, offset, numEntries);
227 WrappedDualView(
const WrappedDualView parent,
const Kokkos::pair<size_t,size_t>& rowRng,
const Kokkos::ALL_t& colRng) {
228 originalDualView = parent.originalDualView;
229 dualView = getSubview2D(parent.dualView,rowRng,colRng);
232 WrappedDualView(
const WrappedDualView parent,
const Kokkos::ALL_t &rowRng,
const Kokkos::pair<size_t,size_t>& colRng) {
233 originalDualView = parent.originalDualView;
234 dualView = getSubview2D(parent.dualView,rowRng,colRng);
237 WrappedDualView(
const WrappedDualView parent,
const Kokkos::pair<size_t,size_t>& rowRng,
const Kokkos::pair<size_t,size_t>& colRng) {
238 originalDualView = parent.originalDualView;
239 dualView = getSubview2D(parent.dualView,rowRng,colRng);
242 size_t extent(
const int i)
const {
243 return dualView.h_view.extent(i);
246 void stride(
size_t * stride_)
const {
247 dualView.stride(stride_);
251 size_t origExtent(
const int i)
const {
252 return originalDualView.h_view.extent(i);
255 const char * label()
const {
256 return dualView.d_view.label();
260 typename t_host::const_type
261 getHostView(Access::ReadOnlyStruct
262 DEBUG_UVM_REMOVAL_ARGUMENT
265 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getHostViewReadOnly");
267 if(needsSyncPath()) {
268 throwIfDeviceViewAlive();
269 impl::sync_host(originalDualView);
271 return dualView.view_host();
275 getHostView(Access::ReadWriteStruct
276 DEBUG_UVM_REMOVAL_ARGUMENT
279 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getHostViewReadWrite");
280 static_assert(dualViewHasNonConstData,
281 "ReadWrite views are not available for DualView with const data");
282 if(needsSyncPath()) {
283 throwIfDeviceViewAlive();
284 impl::sync_host(originalDualView);
285 originalDualView.modify_host();
288 return dualView.view_host();
292 getHostView(Access::OverwriteAllStruct
293 DEBUG_UVM_REMOVAL_ARGUMENT
296 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getHostViewOverwriteAll");
297 static_assert(dualViewHasNonConstData,
298 "OverwriteAll views are not available for DualView with const data");
300 return getHostView(Access::ReadWrite);
302 if(needsSyncPath()) {
303 throwIfDeviceViewAlive();
304 if (deviceMemoryIsHostAccessible) Kokkos::fence(
"WrappedDualView::getHostView");
305 dualView.clear_sync_state();
306 dualView.modify_host();
308 return dualView.view_host();
311 typename t_dev::const_type
312 getDeviceView(Access::ReadOnlyStruct
313 DEBUG_UVM_REMOVAL_ARGUMENT
316 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getDeviceViewReadOnly");
317 if(needsSyncPath()) {
318 throwIfHostViewAlive();
319 impl::sync_device(originalDualView);
321 return dualView.view_device();
325 getDeviceView(Access::ReadWriteStruct
326 DEBUG_UVM_REMOVAL_ARGUMENT
329 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getDeviceViewReadWrite");
330 static_assert(dualViewHasNonConstData,
331 "ReadWrite views are not available for DualView with const data");
332 if(needsSyncPath()) {
333 throwIfHostViewAlive();
334 impl::sync_device(originalDualView);
335 originalDualView.modify_device();
337 return dualView.view_device();
341 getDeviceView(Access::OverwriteAllStruct
342 DEBUG_UVM_REMOVAL_ARGUMENT
345 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getDeviceViewOverwriteAll");
346 static_assert(dualViewHasNonConstData,
347 "OverwriteAll views are not available for DualView with const data");
349 return getDeviceView(Access::ReadWrite);
351 if(needsSyncPath()) {
352 throwIfHostViewAlive();
353 if (deviceMemoryIsHostAccessible) Kokkos::fence(
"WrappedDualView::getDeviceView");
354 dualView.clear_sync_state();
355 dualView.modify_device();
357 return dualView.view_device();
360 template<
class TargetDeviceType>
361 typename std::remove_reference<decltype(std::declval<DualViewType>().
template view<TargetDeviceType>())>::type::const_type
362 getView (Access::ReadOnlyStruct s DEBUG_UVM_REMOVAL_ARGUMENT)
const {
363 using ReturnViewType =
typename std::remove_reference<decltype(std::declval<DualViewType>().
template view<TargetDeviceType>())>::type::const_type;
364 using ReturnDeviceType =
typename ReturnViewType::device_type;
365 constexpr
bool returnDevice = std::is_same<ReturnDeviceType, DeviceType>::value;
367 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getView<Device>ReadOnly");
368 if(needsSyncPath()) {
369 throwIfHostViewAlive();
370 impl::sync_device(originalDualView);
374 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getView<Host>ReadOnly");
375 if(needsSyncPath()) {
376 throwIfDeviceViewAlive();
377 impl::sync_host(originalDualView);
381 return dualView.template view<TargetDeviceType>();
385 template<
class TargetDeviceType>
386 typename std::remove_reference<decltype(std::declval<DualViewType>().
template view<TargetDeviceType>())>::type
387 getView (Access::ReadWriteStruct s DEBUG_UVM_REMOVAL_ARGUMENT)
const {
388 using ReturnViewType =
typename std::remove_reference<decltype(std::declval<DualViewType>().
template view<TargetDeviceType>())>::type;
389 using ReturnDeviceType =
typename ReturnViewType::device_type;
390 constexpr
bool returnDevice = std::is_same<ReturnDeviceType, DeviceType>::value;
393 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getView<Device>ReadWrite");
394 static_assert(dualViewHasNonConstData,
395 "ReadWrite views are not available for DualView with const data");
396 if(needsSyncPath()) {
397 throwIfHostViewAlive();
398 impl::sync_device(originalDualView);
399 originalDualView.modify_device();
403 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getView<Host>ReadWrite");
404 static_assert(dualViewHasNonConstData,
405 "ReadWrite views are not available for DualView with const data");
406 if(needsSyncPath()) {
407 throwIfDeviceViewAlive();
408 impl::sync_host(originalDualView);
409 originalDualView.modify_host();
413 return dualView.template view<TargetDeviceType>();
417 template<
class TargetDeviceType>
418 typename std::remove_reference<decltype(std::declval<DualViewType>().
template view<TargetDeviceType>())>::type
419 getView (Access::OverwriteAllStruct s DEBUG_UVM_REMOVAL_ARGUMENT)
const {
420 using ReturnViewType =
typename std::remove_reference<decltype(std::declval<DualViewType>().
template view<TargetDeviceType>())>::type;
421 using ReturnDeviceType =
typename ReturnViewType::device_type;
424 return getView<TargetDeviceType>(Access::ReadWrite);
426 constexpr
bool returnDevice = std::is_same<ReturnDeviceType, DeviceType>::value;
429 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getView<Device>OverwriteAll");
430 static_assert(dualViewHasNonConstData,
431 "OverwriteAll views are not available for DualView with const data");
432 if(needsSyncPath()) {
433 throwIfHostViewAlive();
434 dualView.clear_sync_state();
435 dualView.modify_host();
439 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getView<Host>OverwriteAll");
440 static_assert(dualViewHasNonConstData,
441 "OverwriteAll views are not available for DualView with const data");
442 if(needsSyncPath()) {
443 throwIfDeviceViewAlive();
444 dualView.clear_sync_state();
445 dualView.modify_device();
449 return dualView.template view<TargetDeviceType>();
453 typename t_host::const_type
454 getHostSubview(
int offset,
int numEntries, Access::ReadOnlyStruct
455 DEBUG_UVM_REMOVAL_ARGUMENT
458 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getHostSubviewReadOnly");
459 if(needsSyncPath()) {
460 throwIfDeviceViewAlive();
461 impl::sync_host(originalDualView);
463 return getSubview(dualView.view_host(), offset, numEntries);
467 getHostSubview(
int offset,
int numEntries, Access::ReadWriteStruct
468 DEBUG_UVM_REMOVAL_ARGUMENT
471 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getHostSubviewReadWrite");
472 static_assert(dualViewHasNonConstData,
473 "ReadWrite views are not available for DualView with const data");
474 if(needsSyncPath()) {
475 throwIfDeviceViewAlive();
476 impl::sync_host(originalDualView);
477 originalDualView.modify_host();
479 return getSubview(dualView.view_host(), offset, numEntries);
483 getHostSubview(
int offset,
int numEntries, Access::OverwriteAllStruct
484 DEBUG_UVM_REMOVAL_ARGUMENT
487 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getHostSubviewOverwriteAll");
488 static_assert(dualViewHasNonConstData,
489 "OverwriteAll views are not available for DualView with const data");
490 return getHostSubview(offset, numEntries, Access::ReadWrite);
493 typename t_dev::const_type
494 getDeviceSubview(
int offset,
int numEntries, Access::ReadOnlyStruct
495 DEBUG_UVM_REMOVAL_ARGUMENT
498 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getDeviceSubviewReadOnly");
499 if(needsSyncPath()) {
500 throwIfHostViewAlive();
501 impl::sync_device(originalDualView);
503 return getSubview(dualView.view_device(), offset, numEntries);
507 getDeviceSubview(
int offset,
int numEntries, Access::ReadWriteStruct
508 DEBUG_UVM_REMOVAL_ARGUMENT
511 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getDeviceSubviewReadWrite");
512 static_assert(dualViewHasNonConstData,
513 "ReadWrite views are not available for DualView with const data");
514 if(needsSyncPath()) {
515 throwIfHostViewAlive();
516 impl::sync_device(originalDualView);
517 originalDualView.modify_device();
519 return getSubview(dualView.view_device(), offset, numEntries);
523 getDeviceSubview(
int offset,
int numEntries, Access::OverwriteAllStruct
524 DEBUG_UVM_REMOVAL_ARGUMENT
527 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getDeviceSubviewOverwriteAll");
528 static_assert(dualViewHasNonConstData,
529 "OverwriteAll views are not available for DualView with const data");
530 return getDeviceSubview(offset, numEntries, Access::ReadWrite);
535 typename t_host::HostMirror getHostCopy()
const {
536 auto X_dev = dualView.view_host();
537 if(X_dev.span_is_contiguous()) {
538 auto mirror = Kokkos::create_mirror_view(X_dev);
543 auto X_contig = Tpetra::Details::TempView::toLayout<decltype(X_dev), Kokkos::LayoutLeft>(X_dev);
544 auto mirror = Kokkos::create_mirror_view(X_contig);
550 typename t_dev::HostMirror getDeviceCopy()
const {
551 auto X_dev = dualView.view_device();
552 if(X_dev.span_is_contiguous()) {
553 auto mirror = Kokkos::create_mirror_view(X_dev);
558 auto X_contig = Tpetra::Details::TempView::toLayout<decltype(X_dev), Kokkos::LayoutLeft>(X_dev);
559 auto mirror = Kokkos::create_mirror_view(X_contig);
566 bool is_valid_host()
const {
567 return dualView.view_host().size() == 0 || dualView.view_host().data();
570 bool is_valid_device()
const {
571 return dualView.view_device().size() == 0 || dualView.view_device().data();
575 bool need_sync_host()
const {
576 return originalDualView.need_sync_host();
579 bool need_sync_device()
const {
580 return originalDualView.need_sync_device();
583 int host_view_use_count()
const {
584 return originalDualView.h_view.use_count();
587 int device_view_use_count()
const {
588 return originalDualView.d_view.use_count();
594 template<
typename SC,
typename LO,
typename GO,
typename NO>
595 friend class ::Tpetra::MultiVector;
601 DualViewType getOriginalDualView()
const {
602 return originalDualView;
605 DualViewType getDualView()
const {
609 template <
typename ViewType>
610 ViewType getSubview(ViewType view,
int offset,
int numEntries)
const {
611 return Kokkos::subview(view, Kokkos::pair<int, int>(offset, offset+numEntries));
614 template <
typename ViewType,
typename int_type>
615 ViewType getSubview2D(ViewType view, Kokkos::pair<int_type,int_type> offset0,
const Kokkos::ALL_t&)
const {
616 return Kokkos::subview(view,offset0,Kokkos::ALL());
619 template <
typename ViewType,
typename int_type>
620 ViewType getSubview2D(ViewType view,
const Kokkos::ALL_t&, Kokkos::pair<int_type,int_type> offset1)
const {
621 return Kokkos::subview(view,Kokkos::ALL(),offset1);
624 template <
typename ViewType,
typename int_type>
625 ViewType getSubview2D(ViewType view, Kokkos::pair<int_type,int_type> offset0, Kokkos::pair<int_type,int_type> offset1)
const {
626 return Kokkos::subview(view,offset0,offset1);
629 bool memoryIsAliased()
const {
630 return deviceMemoryIsHostAccessible && dualView.h_view.data() == dualView.d_view.data();
651 bool needsSyncPath()
const {
657 return !memoryIsAliased() || Spaces::is_gpu_exec_space<typename DualViewType::execution_space>();
661 void throwIfViewsAreDifferentSizes()
const {
664 if(dualView.d_view.size() != dualView.h_view.size()) {
665 std::ostringstream msg;
666 msg <<
"Tpetra::Details::WrappedDualView (name = " << dualView.d_view.label()
667 <<
"; host and device views are different sizes: "
668 << dualView.h_view.size() <<
" vs " <<dualView.h_view.size();
669 throw std::runtime_error(msg.str());
673 void throwIfHostViewAlive()
const {
674 throwIfViewsAreDifferentSizes();
675 if (dualView.h_view.use_count() > dualView.d_view.use_count()) {
676 std::ostringstream msg;
677 msg <<
"Tpetra::Details::WrappedDualView (name = " << dualView.d_view.label()
678 <<
"; host use_count = " << dualView.h_view.use_count()
679 <<
"; device use_count = " << dualView.d_view.use_count() <<
"): "
680 <<
"Cannot access data on device while a host view is alive";
681 throw std::runtime_error(msg.str());
685 void throwIfDeviceViewAlive()
const {
686 throwIfViewsAreDifferentSizes();
687 if (dualView.d_view.use_count() > dualView.h_view.use_count()) {
688 std::ostringstream msg;
689 msg <<
"Tpetra::Details::WrappedDualView (name = " << dualView.d_view.label()
690 <<
"; host use_count = " << dualView.h_view.use_count()
691 <<
"; device use_count = " << dualView.d_view.use_count() <<
"): "
692 <<
"Cannot access data on host while a device view is alive";
693 throw std::runtime_error(msg.str());
698 return originalDualView.h_view != dualView.h_view;
701 mutable DualViewType originalDualView;
702 mutable DualViewType dualView;
One or more distributed dense vectors.
bool wdvTrackingEnabled
Whether WrappedDualView reference count checking is enabled. Initially true. Since the DualView sync ...
A wrapper around Kokkos::DualView to safely manage data that might be replicated between host and dev...
WrappedDualView(const WrappedDualView< SrcDualViewType > &src)
Conversion copy constructor.
void deep_copy(MultiVector< DS, DL, DG, DN > &dst, const MultiVector< SS, SL, SG, SN > &src)
Copy the contents of the MultiVector src into dst.
WrappedDualView & operator=(const WrappedDualView< SrcDualViewType > &src)
Conversion assignment operator.
void enableWDVTracking()
Enable WrappedDualView reference-count tracking and syncing. Call this after exiting a host-parallel ...
void disableWDVTracking()
Disable WrappedDualView reference-count tracking and syncing. Call this before entering a host-parall...