10 #ifndef TPETRA_CRSGRAPH_DEF_HPP
11 #define TPETRA_CRSGRAPH_DEF_HPP
20 #include "Tpetra_Details_getGraphDiagOffsets.hpp"
21 #include "Tpetra_Details_getGraphOffRankOffsets.hpp"
22 #include "Tpetra_Details_makeColMap.hpp"
26 #include "Tpetra_Distributor.hpp"
27 #include "Teuchos_SerialDenseMatrix.hpp"
28 #include "Tpetra_Vector.hpp"
31 #include "Tpetra_Details_packCrsGraph.hpp"
32 #include "Tpetra_Details_unpackCrsGraphAndCombine.hpp"
33 #include "Tpetra_Details_CrsPadding.hpp"
40 #include <type_traits>
48 template<
class MapIter>
50 verbosePrintMap(std::ostream& out,
56 using ::Tpetra::Details::Behavior;
59 out << mapName <<
": {";
60 const size_t maxNumToPrint =
62 if (maxNumToPrint == 0) {
68 const size_t numToPrint = numEnt > maxNumToPrint ?
69 maxNumToPrint : numEnt;
71 for (MapIter it = beg; it != end; ++it) {
72 out <<
"(" << (*it).first <<
", ";
76 if (count +
size_t(1) < numToPrint) {
88 template<
class LO,
class GO,
class Node>
89 Teuchos::ArrayView<GO>
91 std::vector<GO>& gblColIndsStorage,
92 const RowGraph<LO, GO, Node>& graph,
95 size_t origNumEnt = graph.getNumEntriesInGlobalRow(gblRowInd);
96 if (gblColIndsStorage.size() < origNumEnt) {
97 gblColIndsStorage.resize(origNumEnt);
99 typename CrsGraph<LO,GO,Node>::nonconst_global_inds_host_view_type gblColInds(gblColIndsStorage.data(),
101 graph.getGlobalRowCopy(gblRowInd, gblColInds, origNumEnt);
102 Teuchos::ArrayView<GO> retval(gblColIndsStorage.data(),origNumEnt);
106 template<
class LO,
class GO,
class DT,
class OffsetType,
class NumEntType>
107 class ConvertColumnIndicesFromGlobalToLocal {
109 ConvertColumnIndicesFromGlobalToLocal (const ::Kokkos::View<LO*, DT>& lclColInds,
110 const ::Kokkos::View<const GO*, DT>& gblColInds,
111 const ::Kokkos::View<const OffsetType*, DT>& ptr,
112 const ::Tpetra::Details::LocalMap<LO, GO, DT>& lclColMap,
113 const ::Kokkos::View<const NumEntType*, DT>& numRowEnt) :
114 lclColInds_ (lclColInds),
115 gblColInds_ (gblColInds),
117 lclColMap_ (lclColMap),
118 numRowEnt_ (numRowEnt)
122 operator () (
const LO& lclRow, OffsetType& curNumBad)
const
124 const OffsetType offset = ptr_(lclRow);
128 const LO numEnt =
static_cast<LO
> (numRowEnt_(lclRow));
129 for (LO j = 0; j < numEnt; ++j) {
130 const GO gid = gblColInds_(offset + j);
131 const LO lid = lclColMap_.getLocalElement (gid);
132 lclColInds_(offset + j) = lid;
133 if (lid == ::Tpetra::Details::OrdinalTraits<LO>::invalid ()) {
140 run (const ::Kokkos::View<LO*, DT>& lclColInds,
141 const ::Kokkos::View<const GO*, DT>& gblColInds,
142 const ::Kokkos::View<const OffsetType*, DT>& ptr,
143 const ::Tpetra::Details::LocalMap<LO, GO, DT>& lclColMap,
144 const ::Kokkos::View<const NumEntType*, DT>& numRowEnt)
146 typedef ::Kokkos::RangePolicy<typename DT::execution_space, LO> range_type;
147 typedef ConvertColumnIndicesFromGlobalToLocal<LO, GO, DT, OffsetType, NumEntType> functor_type;
149 const LO lclNumRows = ptr.extent (0) == 0 ?
150 static_cast<LO
> (0) : static_cast<LO> (ptr.extent (0) - 1);
151 OffsetType numBad = 0;
153 ::Kokkos::parallel_reduce (range_type (0, lclNumRows),
154 functor_type (lclColInds, gblColInds, ptr,
155 lclColMap, numRowEnt),
161 ::Kokkos::View<LO*, DT> lclColInds_;
162 ::Kokkos::View<const GO*, DT> gblColInds_;
163 ::Kokkos::View<const OffsetType*, DT> ptr_;
165 ::Kokkos::View<const NumEntType*, DT> numRowEnt_;
184 template<
class LO,
class GO,
class DT,
class OffsetType,
class NumEntType>
187 const Kokkos::View<const GO*, DT>& gblColInds,
188 const Kokkos::View<const OffsetType*, DT>& ptr,
190 const Kokkos::View<const NumEntType*, DT>& numRowEnt)
192 using Impl::ConvertColumnIndicesFromGlobalToLocal;
193 typedef ConvertColumnIndicesFromGlobalToLocal<LO, GO, DT, OffsetType, NumEntType> impl_type;
194 return impl_type::run (lclColInds, gblColInds, ptr, lclColMap, numRowEnt);
197 template<
class ViewType,
class LO>
198 class MaxDifference {
200 MaxDifference (
const ViewType& ptr) : ptr_ (ptr) {}
202 KOKKOS_INLINE_FUNCTION
void init (LO& dst)
const {
206 KOKKOS_INLINE_FUNCTION
void
207 join (LO& dst,
const LO& src)
const
209 dst = (src > dst) ? src : dst;
212 KOKKOS_INLINE_FUNCTION
void
213 operator () (
const LO lclRow, LO& maxNumEnt)
const
215 const LO numEnt =
static_cast<LO
> (ptr_(lclRow+1) - ptr_(lclRow));
216 maxNumEnt = (numEnt > maxNumEnt) ? numEnt : maxNumEnt;
219 typename ViewType::const_type ptr_;
222 template<
class ViewType,
class LO>
223 typename ViewType::non_const_value_type
224 maxDifference (
const char kernelLabel[],
228 if (lclNumRows == 0) {
231 return static_cast<LO
> (0);
234 using execution_space =
typename ViewType::execution_space;
235 using range_type = Kokkos::RangePolicy<execution_space, LO>;
237 Kokkos::parallel_reduce (kernelLabel,
238 range_type (0, lclNumRows),
239 MaxDifference<ViewType, LO> (ptr),
247 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
249 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
254 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
256 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
261 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
262 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
263 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
264 const size_t maxNumEntriesPerRow,
265 const Teuchos::RCP<Teuchos::ParameterList>& params) :
268 , numAllocForAllRows_ (maxNumEntriesPerRow)
270 const char tfecfFuncName[] =
271 "CrsGraph(rowMap,maxNumEntriesPerRow,params): ";
273 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
274 (maxNumEntriesPerRow == Teuchos::OrdinalTraits<size_t>::invalid (),
275 std::invalid_argument,
"The allocation hint maxNumEntriesPerRow must be "
276 "a valid size_t value, which in this case means it must not be "
277 "Teuchos::OrdinalTraits<size_t>::invalid().");
282 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
284 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
285 const Teuchos::RCP<const map_type>& colMap,
286 const size_t maxNumEntriesPerRow,
287 const Teuchos::RCP<Teuchos::ParameterList>& params) :
291 , numAllocForAllRows_ (maxNumEntriesPerRow)
293 const char tfecfFuncName[] =
294 "CrsGraph(rowMap,colMap,maxNumEntriesPerRow,params): ";
296 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
297 maxNumEntriesPerRow == Teuchos::OrdinalTraits<size_t>::invalid (),
298 std::invalid_argument,
"The allocation hint maxNumEntriesPerRow must be "
299 "a valid size_t value, which in this case means it must not be "
300 "Teuchos::OrdinalTraits<size_t>::invalid().");
306 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
308 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
309 const Teuchos::ArrayView<const size_t>& numEntPerRow,
310 const Teuchos::RCP<Teuchos::ParameterList>& params) :
313 , numAllocForAllRows_ (0)
315 const char tfecfFuncName[] =
316 "CrsGraph(rowMap,numEntPerRow,params): ";
319 const size_t lclNumRows = rowMap.is_null () ?
320 static_cast<size_t> (0) : rowMap->getLocalNumElements ();
321 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
322 static_cast<size_t> (numEntPerRow.size ()) != lclNumRows,
323 std::invalid_argument,
"numEntPerRow has length " << numEntPerRow.size ()
324 <<
" != the local number of rows " << lclNumRows <<
" as specified by "
325 "the input row Map.");
328 for (
size_t r = 0; r < lclNumRows; ++r) {
329 const size_t curRowCount = numEntPerRow[r];
330 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
331 (curRowCount == Teuchos::OrdinalTraits<size_t>::invalid (),
332 std::invalid_argument,
"numEntPerRow(" << r <<
") "
333 "specifies an invalid number of entries "
334 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
342 typedef typename out_view_type::non_const_type nc_view_type;
343 typedef Kokkos::View<
const size_t*,
344 typename nc_view_type::array_layout,
346 Kokkos::MemoryUnmanaged> in_view_type;
347 in_view_type numAllocPerRowIn (numEntPerRow.getRawPtr (), lclNumRows);
348 nc_view_type numAllocPerRowOut (
"Tpetra::CrsGraph::numAllocPerRow",
351 using exec_space =
typename nc_view_type::execution_space;
361 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
363 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
364 const Kokkos::DualView<const size_t*, device_type>& numEntPerRow,
365 const Teuchos::RCP<Teuchos::ParameterList>& params) :
368 , k_numAllocPerRow_ (numEntPerRow.h_view)
369 , numAllocForAllRows_ (0)
371 const char tfecfFuncName[] =
372 "CrsGraph(rowMap,numEntPerRow,params): ";
375 const size_t lclNumRows = rowMap.is_null () ?
376 static_cast<size_t> (0) : rowMap->getLocalNumElements ();
377 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
378 static_cast<size_t> (numEntPerRow.extent (0)) != lclNumRows,
379 std::invalid_argument,
"numEntPerRow has length " <<
380 numEntPerRow.extent (0) <<
" != the local number of rows " <<
381 lclNumRows <<
" as specified by " "the input row Map.");
384 for (
size_t r = 0; r < lclNumRows; ++r) {
385 const size_t curRowCount = numEntPerRow.h_view(r);
386 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
387 (curRowCount == Teuchos::OrdinalTraits<size_t>::invalid (),
388 std::invalid_argument,
"numEntPerRow(" << r <<
") "
389 "specifies an invalid number of entries "
390 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
399 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
401 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
402 const Teuchos::RCP<const map_type>& colMap,
403 const Kokkos::DualView<const size_t*, device_type>& numEntPerRow,
404 const Teuchos::RCP<Teuchos::ParameterList>& params) :
408 , k_numAllocPerRow_ (numEntPerRow.h_view)
409 , numAllocForAllRows_ (0)
411 const char tfecfFuncName[] =
412 "CrsGraph(rowMap,colMap,numEntPerRow,params): ";
415 const size_t lclNumRows = rowMap.is_null () ?
416 static_cast<size_t> (0) : rowMap->getLocalNumElements ();
417 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
418 static_cast<size_t> (numEntPerRow.extent (0)) != lclNumRows,
419 std::invalid_argument,
"numEntPerRow has length " <<
420 numEntPerRow.extent (0) <<
" != the local number of rows " <<
421 lclNumRows <<
" as specified by " "the input row Map.");
424 for (
size_t r = 0; r < lclNumRows; ++r) {
425 const size_t curRowCount = numEntPerRow.h_view(r);
426 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
427 (curRowCount == Teuchos::OrdinalTraits<size_t>::invalid (),
428 std::invalid_argument,
"numEntPerRow(" << r <<
") "
429 "specifies an invalid number of entries "
430 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
439 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
441 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
442 const Teuchos::RCP<const map_type>& colMap,
443 const Teuchos::ArrayView<const size_t>& numEntPerRow,
444 const Teuchos::RCP<Teuchos::ParameterList>& params) :
448 , numAllocForAllRows_ (0)
450 const char tfecfFuncName[] =
451 "CrsGraph(rowMap,colMap,numEntPerRow,params): ";
454 const size_t lclNumRows = rowMap.is_null () ?
455 static_cast<size_t> (0) : rowMap->getLocalNumElements ();
456 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
457 static_cast<size_t> (numEntPerRow.size ()) != lclNumRows,
458 std::invalid_argument,
"numEntPerRow has length " << numEntPerRow.size ()
459 <<
" != the local number of rows " << lclNumRows <<
" as specified by "
460 "the input row Map.");
463 for (
size_t r = 0; r < lclNumRows; ++r) {
464 const size_t curRowCount = numEntPerRow[r];
465 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
466 (curRowCount == Teuchos::OrdinalTraits<size_t>::invalid (),
467 std::invalid_argument,
"numEntPerRow(" << r <<
") "
468 "specifies an invalid number of entries "
469 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
477 typedef typename out_view_type::non_const_type nc_view_type;
478 typedef Kokkos::View<
const size_t*,
479 typename nc_view_type::array_layout,
481 Kokkos::MemoryUnmanaged> in_view_type;
482 in_view_type numAllocPerRowIn (numEntPerRow.getRawPtr (), lclNumRows);
483 nc_view_type numAllocPerRowOut (
"Tpetra::CrsGraph::numAllocPerRow",
486 using exec_space =
typename nc_view_type::execution_space;
495 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
498 const Teuchos::RCP<const map_type>& rowMap,
499 const Teuchos::RCP<Teuchos::ParameterList>& params) :
502 , colMap_(originalGraph.colMap_)
503 , numAllocForAllRows_(originalGraph.numAllocForAllRows_)
504 , storageStatus_(originalGraph.storageStatus_)
505 , indicesAreAllocated_(originalGraph.indicesAreAllocated_)
506 , indicesAreLocal_(originalGraph.indicesAreLocal_)
507 , indicesAreSorted_(originalGraph.indicesAreSorted_)
511 int numRows = rowMap->getLocalNumElements();
513 auto rowsToUse = Kokkos::pair<size_t, size_t>(0, numRows+1);
519 if (indicesAreLocal_) {
530 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
532 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
533 const Teuchos::RCP<const map_type>& colMap,
534 const typename local_graph_device_type::row_map_type& rowPointers,
535 const typename local_graph_device_type::entries_type::non_const_type& columnIndices,
536 const Teuchos::RCP<Teuchos::ParameterList>& params) :
540 , numAllocForAllRows_(0)
541 , storageStatus_(Details::STORAGE_1D_PACKED)
542 , indicesAreAllocated_(true)
543 , indicesAreLocal_(true)
546 if (! params.is_null() && params->isParameter(
"sorted") &&
547 ! params->get<
bool>(
"sorted")) {
557 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
559 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
560 const Teuchos::RCP<const map_type>& colMap,
561 const Teuchos::ArrayRCP<size_t>& rowPointers,
562 const Teuchos::ArrayRCP<LocalOrdinal> & columnIndices,
563 const Teuchos::RCP<Teuchos::ParameterList>& params) :
567 , numAllocForAllRows_ (0)
568 , storageStatus_ (Details::STORAGE_1D_PACKED)
569 , indicesAreAllocated_ (true)
570 , indicesAreLocal_ (true)
573 if (! params.is_null() && params->isParameter(
"sorted") &&
574 ! params->get<
bool>(
"sorted")) {
584 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
586 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
587 const Teuchos::RCP<const map_type>& colMap,
589 const Teuchos::RCP<Teuchos::ParameterList>& params)
598 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
601 const Teuchos::RCP<const map_type>& rowMap,
602 const Teuchos::RCP<const map_type>& colMap,
603 const Teuchos::RCP<const map_type>& domainMap,
604 const Teuchos::RCP<const map_type>& rangeMap,
605 const Teuchos::RCP<Teuchos::ParameterList>& params)
609 , numAllocForAllRows_ (0)
610 , storageStatus_ (Details::STORAGE_1D_PACKED)
611 , indicesAreAllocated_ (true)
612 , indicesAreLocal_ (true)
615 const char tfecfFuncName[] =
"CrsGraph(Kokkos::LocalStaticCrsGraph,Map,Map,Map,Map)";
617 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
618 colMap.is_null (), std::runtime_error,
619 ": The input column Map must be nonnull.");
620 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
621 k_local_graph_.numRows () != rowMap->getLocalNumElements (),
623 ": The input row Map and the input local graph need to have the same "
624 "number of rows. The row Map claims " << rowMap->getLocalNumElements ()
625 <<
" row(s), but the local graph claims " << k_local_graph_.numRows ()
635 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
637 ": cannot have 1D data structures allocated.");
639 if(! params.is_null() && params->isParameter(
"sorted") &&
640 ! params->get<
bool>(
"sorted")) {
648 rangeMap .is_null() ?
rowMap_ : rangeMap);
649 Teuchos::Array<int> remotePIDs (0);
654 this->setRowPtrs(k_local_graph_.row_map);
656 set_need_sync_host_uvm_access();
658 const bool callComputeGlobalConstants = params.get () ==
nullptr ||
659 params->get (
"compute global constants",
true);
661 if (callComputeGlobalConstants) {
664 this->fillComplete_ =
true;
668 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
671 const Teuchos::RCP<const map_type>& rowMap,
672 const Teuchos::RCP<const map_type>& colMap,
673 const Teuchos::RCP<const map_type>& domainMap,
674 const Teuchos::RCP<const map_type>& rangeMap,
675 const Teuchos::RCP<const import_type>& importer,
676 const Teuchos::RCP<const export_type>& exporter,
677 const Teuchos::RCP<Teuchos::ParameterList>& params) :
681 rangeMap_ (rangeMap.is_null () ? rowMap : rangeMap),
682 domainMap_ (domainMap.is_null () ? rowMap : domainMap),
683 importer_ (importer),
684 exporter_ (exporter),
685 numAllocForAllRows_ (0),
686 storageStatus_ (Details::STORAGE_1D_PACKED),
687 indicesAreAllocated_ (true),
688 indicesAreLocal_ (true)
691 const char tfecfFuncName[] =
"Tpetra::CrsGraph(local_graph_device_type,"
692 "Map,Map,Map,Map,Import,Export,params): ";
694 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
695 (colMap.is_null (), std::runtime_error,
696 "The input column Map must be nonnull.");
700 setRowPtrs(lclGraph.row_map);
702 set_need_sync_host_uvm_access();
704 if (! params.is_null() && params->isParameter(
"sorted") &&
705 ! params->get<
bool>(
"sorted")) {
712 const bool callComputeGlobalConstants =
713 params.get () ==
nullptr ||
714 params->get (
"compute global constants",
true);
715 if (callComputeGlobalConstants) {
718 fillComplete_ =
true;
722 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
724 CrsGraph (
const row_ptrs_device_view_type& rowPointers,
726 const Teuchos::RCP<const map_type>& rowMap,
727 const Teuchos::RCP<const map_type>& colMap,
728 const Teuchos::RCP<const map_type>& domainMap,
729 const Teuchos::RCP<const map_type>& rangeMap,
730 const Teuchos::RCP<const import_type>& importer,
731 const Teuchos::RCP<const export_type>& exporter,
732 const Teuchos::RCP<Teuchos::ParameterList>& params) :
736 rangeMap_ (rangeMap.is_null () ? rowMap : rangeMap),
737 domainMap_ (domainMap.is_null () ? rowMap : domainMap),
738 importer_ (importer),
739 exporter_ (exporter),
740 numAllocForAllRows_ (0),
741 storageStatus_ (Details::STORAGE_1D_PACKED),
742 indicesAreAllocated_ (true),
743 indicesAreLocal_ (true)
746 const char tfecfFuncName[] =
"Tpetra::CrsGraph(row_ptrs_device_view_type,local_inds_wdv_type"
747 "Map,Map,Map,Map,Import,Export,params): ";
749 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
750 (colMap.is_null (), std::runtime_error,
751 "The input column Map must be nonnull.");
755 setRowPtrs(rowPointers);
757 set_need_sync_host_uvm_access();
759 if (! params.is_null() && params->isParameter(
"sorted") &&
760 ! params->get<
bool>(
"sorted")) {
767 const bool callComputeGlobalConstants =
768 params.get () ==
nullptr ||
769 params->get (
"compute global constants",
true);
770 if (callComputeGlobalConstants) {
773 fillComplete_ =
true;
777 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
778 Teuchos::RCP<const Teuchos::ParameterList>
783 using Teuchos::ParameterList;
784 using Teuchos::parameterList;
786 RCP<ParameterList> params = parameterList (
"Tpetra::CrsGraph");
789 RCP<ParameterList> importSublist = parameterList (
"Import");
801 Distributor distributor (rowMap_->getComm (), importSublist);
802 params->set (
"Import", *importSublist,
"How the Import performs communication.");
808 params->set (
"Export", *importSublist,
"How the Export performs communication.");
813 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
818 Teuchos::RCP<const Teuchos::ParameterList> validParams =
819 getValidParameters ();
820 params->validateParametersAndSetDefaults (*validParams);
821 this->setMyParamList (params);
824 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
829 return rowMap_->getGlobalNumElements ();
832 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
837 const char tfecfFuncName[] =
"getGlobalNumCols: ";
838 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
839 ! isFillComplete () || getDomainMap ().is_null (), std::runtime_error,
840 "The graph does not have a domain Map. You may not call this method in "
842 return getDomainMap ()->getGlobalNumElements ();
846 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
851 return this->rowMap_.is_null () ?
852 static_cast<size_t> (0) :
853 this->rowMap_->getLocalNumElements ();
857 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
862 const char tfecfFuncName[] =
"getLocalNumCols: ";
863 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
864 ! hasColMap (), std::runtime_error,
865 "The graph does not have a column Map. You may not call this method "
866 "unless the graph has a column Map. This requires either that a custom "
867 "column Map was given to the constructor, or that fillComplete() has "
869 return colMap_.is_null () ?
static_cast<size_t> (0) :
870 colMap_->getLocalNumElements ();
875 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
876 Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
883 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
884 Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
891 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
892 Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
899 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
900 Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
907 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
908 Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::import_type>
915 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
916 Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::export_type>
923 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
928 return ! colMap_.is_null ();
931 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
939 const bool isOpt = indicesAreAllocated_ &&
940 k_numRowEntries_.extent (0) == 0 &&
941 getLocalNumRows () > 0;
947 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
952 const char tfecfFuncName[] =
"getGlobalNumEntries: ";
953 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
954 (! this->haveGlobalConstants_, std::logic_error,
955 "The graph does not have global constants computed, "
956 "but the user has requested them.");
958 return globalNumEntries_;
962 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
967 const char tfecfFuncName[] =
"getLocalNumEntries: ";
968 typedef LocalOrdinal LO;
970 if (this->indicesAreAllocated_) {
971 const LO lclNumRows = this->getLocalNumRows ();
972 if (lclNumRows == 0) {
973 return static_cast<size_t> (0);
977 auto numEntPerRow = this->k_numRowEntries_;
978 const LO numNumEntPerRow = numEntPerRow.extent (0);
979 if (numNumEntPerRow == 0) {
980 if (static_cast<LO> (this->getRowPtrsPackedDevice().extent (0)) <
981 static_cast<LO> (lclNumRows + 1)) {
982 return static_cast<size_t> (0);
989 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
990 (this->getRowPtrsPackedHost()(lclNumRows) != lclIndsPacked_wdv.extent(0), std::logic_error,
991 "Final entry of packed host rowptrs doesn't match the length of lclIndsPacked");
993 return lclIndsPacked_wdv.extent(0);
1003 typedef typename num_row_entries_type::execution_space
1005 typedef Kokkos::RangePolicy<host_exec_space, LO> range_type;
1007 const LO upperLoopBound = lclNumRows < numNumEntPerRow ?
1010 size_t nodeNumEnt = 0;
1011 Kokkos::parallel_reduce (
"Tpetra::CrsGraph::getNumNodeEntries",
1012 range_type (0, upperLoopBound),
1013 [=] (
const LO& k,
size_t& lclSum) {
1014 lclSum += numEntPerRow(k);
1021 return static_cast<size_t> (0);
1025 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1030 const char tfecfFuncName[] =
"getGlobalMaxNumRowEntries: ";
1031 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1032 (! this->haveGlobalConstants_, std::logic_error,
1033 "The graph does not have global constants computed, "
1034 "but the user has requested them.");
1036 return globalMaxNumRowEntries_;
1039 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1044 return nodeMaxNumRowEntries_;
1047 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1052 return fillComplete_;
1055 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1060 return ! fillComplete_;
1064 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1069 return indicesAreLocal_;
1072 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1077 return indicesAreGlobal_;
1080 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1085 typedef LocalOrdinal LO;
1087 if (this->indicesAreAllocated_) {
1088 const LO lclNumRows = this->getLocalNumRows ();
1089 if (lclNumRows == 0) {
1090 return static_cast<size_t> (0);
1092 else if (storageStatus_ == Details::STORAGE_1D_PACKED) {
1093 if (static_cast<LO> (this->getRowPtrsPackedDevice().extent (0)) <
1094 static_cast<LO
> (lclNumRows + 1)) {
1095 return static_cast<size_t> (0);
1098 if(this->isLocallyIndexed())
1099 return lclIndsPacked_wdv.extent(0);
1101 return gblInds_wdv.extent(0);
1104 else if (storageStatus_ == Details::STORAGE_1D_UNPACKED) {
1105 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
1106 if (rowPtrsUnpacked_host.extent (0) == 0) {
1107 return static_cast<size_t> (0);
1110 if(this->isLocallyIndexed())
1111 return lclIndsUnpacked_wdv.extent(0);
1113 return gblInds_wdv.extent(0);
1117 return static_cast<size_t> (0);
1121 return Tpetra::Details::OrdinalTraits<size_t>::invalid ();
1125 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1126 Teuchos::RCP<const Teuchos::Comm<int> >
1130 return this->rowMap_.is_null () ? Teuchos::null : this->rowMap_->getComm ();
1133 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1138 return rowMap_->getIndexBase ();
1141 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1146 return indicesAreAllocated_;
1149 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1154 return indicesAreSorted_;
1157 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1162 return noRedundancies_;
1165 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1175 indicesAreSorted_ =
false;
1176 noRedundancies_ =
false;
1180 haveLocalConstants_ =
false;
1183 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1189 using Teuchos::arcp;
1190 using Teuchos::Array;
1191 using Teuchos::ArrayRCP;
1193 typedef Teuchos::ArrayRCP<size_t>::size_type size_type;
1194 typedef typename local_graph_device_type::row_map_type::non_const_type
1195 non_const_row_map_type;
1196 const char tfecfFuncName[] =
"allocateIndices: ";
1197 const char suffix[] =
1198 " Please report this bug to the Tpetra developers.";
1199 ProfilingRegion profRegion(
"Tpetra::CrsGraph::allocateIndices");
1201 std::unique_ptr<std::string> prefix;
1203 prefix = this->createPrefix(
"CrsGraph", tfecfFuncName);
1204 std::ostringstream os;
1205 os << *prefix <<
"Start: lg="
1206 << (lg == GlobalIndices ?
"GlobalIndices" :
"LocalIndices")
1207 <<
", numRows: " << this->getLocalNumRows() << endl;
1208 std::cerr << os.str();
1214 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1215 (isLocallyIndexed () && lg == GlobalIndices, std::logic_error,
1216 ": The graph is locally indexed, but Tpetra code is calling "
1217 "this method with lg=GlobalIndices." << suffix);
1218 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1219 (isGloballyIndexed () && lg == LocalIndices, std::logic_error,
1220 ": The graph is globally indexed, but Tpetra code is calling "
1221 "this method with lg=LocalIndices." << suffix);
1222 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1223 (indicesAreAllocated (), std::logic_error,
": The graph's "
1224 "indices are already allocated, but Tpetra is calling "
1225 "allocateIndices again." << suffix);
1226 const size_t numRows = this->getLocalNumRows ();
1231 size_type numInds = 0;
1234 std::ostringstream os;
1235 os << *prefix <<
"Allocate k_rowPtrs: " << (numRows+1) << endl;
1236 std::cerr << os.str();
1238 non_const_row_map_type k_rowPtrs (
"Tpetra::CrsGraph::ptr", numRows + 1);
1240 if (this->k_numAllocPerRow_.extent (0) != 0) {
1245 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1246 (this->k_numAllocPerRow_.extent (0) != numRows,
1247 std::invalid_argument,
"k_numAllocPerRow_ is allocated, that is, "
1248 "has nonzero length " << this->k_numAllocPerRow_.extent (0)
1249 <<
", but its length != numRows = " << numRows <<
".");
1267 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1268 (this->numAllocForAllRows_ ==
1269 Tpetra::Details::OrdinalTraits<size_t>::invalid (),
1270 std::invalid_argument,
"numAllocForAllRows_ has an invalid value, "
1271 "namely Tpetra::Details::OrdinalTraits<size_t>::invalid() = " <<
1272 Tpetra::Details::OrdinalTraits<size_t>::invalid () <<
".");
1278 setRowPtrsUnpacked(k_rowPtrs);
1281 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1282 (numInds != size_type(this->getRowPtrsUnpackedHost()(numRows)), std::logic_error,
1283 ": Number of indices produced by computeOffsetsFrom[Constant]Counts "
1284 "does not match final entry of rowptrs unpacked");
1288 if (lg == LocalIndices) {
1290 std::ostringstream os;
1291 os << *prefix <<
"Allocate local column indices "
1292 "lclIndsUnpacked_wdv: " << numInds << endl;
1293 std::cerr << os.str();
1295 lclIndsUnpacked_wdv = local_inds_wdv_type (
1296 local_inds_dualv_type(
"Tpetra::CrsGraph::lclInd",numInds));
1300 std::ostringstream os;
1301 os << *prefix <<
"Allocate global column indices "
1302 "gblInds_wdv: " << numInds << endl;
1303 std::cerr << os.str();
1305 gblInds_wdv = global_inds_wdv_type (
1306 global_inds_dualv_type(
"Tpetra::CrsGraph::gblInd",numInds));
1308 storageStatus_ = Details::STORAGE_1D_UNPACKED;
1310 this->indicesAreLocal_ = (lg == LocalIndices);
1311 this->indicesAreGlobal_ = (lg == GlobalIndices);
1314 using Kokkos::ViewAllocateWithoutInitializing;
1315 const char label[] =
"Tpetra::CrsGraph::numRowEntries";
1317 std::ostringstream os;
1318 os << *prefix <<
"Allocate k_numRowEntries_: " << numRows
1320 std::cerr << os.str();
1322 num_row_entries_type numRowEnt (ViewAllocateWithoutInitializing (label), numRows);
1325 Kokkos::fence(
"CrsGraph::allocateIndices");
1326 this->k_numRowEntries_ = numRowEnt;
1330 this->numAllocForAllRows_ = 0;
1331 this->k_numAllocPerRow_ = decltype (k_numAllocPerRow_) ();
1332 this->indicesAreAllocated_ =
true;
1335 this->checkInternalState ();
1337 catch (std::logic_error& e) {
1338 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1339 (
true, std::logic_error,
"At end of allocateIndices, "
1340 "checkInternalState threw std::logic_error: "
1343 catch (std::exception& e) {
1344 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1345 (
true, std::runtime_error,
"At end of allocateIndices, "
1346 "checkInternalState threw std::exception: "
1350 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1351 (
true, std::runtime_error,
"At end of allocateIndices, "
1352 "checkInternalState threw an exception "
1353 "not a subclass of std::exception.");
1357 std::ostringstream os;
1358 os << *prefix <<
"Done" << endl;
1359 std::cerr << os.str();
1363 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1364 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1365 local_inds_dualv_type::t_host::const_type
1369 if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
1370 return typename local_inds_dualv_type::t_host::const_type ();
1372 return lclIndsUnpacked_wdv.getHostSubview(rowinfo.offset1D,
1377 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1379 local_inds_dualv_type::t_host
1383 if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
1384 return typename local_inds_dualv_type::t_host ();
1386 return lclIndsUnpacked_wdv.getHostSubview(rowinfo.offset1D,
1391 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1393 global_inds_dualv_type::t_host::const_type
1397 if (rowinfo.allocSize == 0 || gblInds_wdv.extent(0) == 0)
1398 return typename global_inds_dualv_type::t_host::const_type ();
1400 return gblInds_wdv.getHostSubview(rowinfo.offset1D,
1405 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1407 local_inds_dualv_type::t_dev::const_type
1411 if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
1412 return typename local_inds_dualv_type::t_dev::const_type ();
1414 return lclIndsUnpacked_wdv.getDeviceSubview(rowinfo.offset1D,
1419 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1421 global_inds_dualv_type::t_dev::const_type
1425 if (rowinfo.allocSize == 0 || gblInds_wdv.extent(0) == 0)
1426 return typename global_inds_dualv_type::t_dev::const_type ();
1428 return gblInds_wdv.getDeviceSubview(rowinfo.offset1D,
1434 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1439 const size_t STINV = Teuchos::OrdinalTraits<size_t>::invalid ();
1441 if (this->rowMap_.is_null () || ! this->rowMap_->isNodeLocalElement (myRow)) {
1442 ret.localRow = STINV;
1445 ret.offset1D = STINV;
1449 ret.localRow =
static_cast<size_t> (myRow);
1450 if (this->indicesAreAllocated ()) {
1451 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
1453 if (rowPtrsUnpacked_host.extent (0) == 0) {
1458 ret.offset1D = rowPtrsUnpacked_host(myRow);
1459 ret.allocSize = rowPtrsUnpacked_host(myRow+1) - rowPtrsUnpacked_host(myRow);
1462 ret.numEntries = (this->k_numRowEntries_.extent (0) == 0) ?
1464 this->k_numRowEntries_(myRow);
1470 ret.allocSize = (this->k_numAllocPerRow_.extent (0) != 0) ?
1471 this->k_numAllocPerRow_(myRow) :
1472 this->numAllocForAllRows_;
1474 ret.offset1D = STINV;
1481 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1486 const size_t STINV = Teuchos::OrdinalTraits<size_t>::invalid ();
1488 if (this->rowMap_.is_null ()) {
1489 ret.localRow = STINV;
1492 ret.offset1D = STINV;
1495 const LocalOrdinal myRow = this->rowMap_->getLocalElement (gblRow);
1496 if (myRow == Teuchos::OrdinalTraits<LocalOrdinal>::invalid ()) {
1497 ret.localRow = STINV;
1500 ret.offset1D = STINV;
1504 ret.localRow =
static_cast<size_t> (myRow);
1505 if (this->indicesAreAllocated ()) {
1509 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
1510 if (rowPtrsUnpacked_host.extent (0) == 0) {
1515 ret.offset1D = rowPtrsUnpacked_host(myRow);
1516 ret.allocSize = rowPtrsUnpacked_host(myRow+1) - rowPtrsUnpacked_host(myRow);
1519 ret.numEntries = (this->k_numRowEntries_.extent (0) == 0) ?
1521 this->k_numRowEntries_(myRow);
1527 ret.allocSize = (this->k_numAllocPerRow_.extent (0) != 0) ?
1528 this->k_numAllocPerRow_(myRow) :
1529 this->numAllocForAllRows_;
1531 ret.offset1D = STINV;
1538 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1543 using Teuchos::OrdinalTraits;
1544 typedef LocalOrdinal LO;
1545 typedef GlobalOrdinal GO;
1551 static_assert (
sizeof (GlobalOrdinal) >=
sizeof (LocalOrdinal),
1552 "Tpetra::CrsGraph: sizeof(GlobalOrdinal) must be >= sizeof(LocalOrdinal).");
1555 static_assert (
sizeof (
size_t) >=
sizeof (LocalOrdinal),
1556 "Tpetra::CrsGraph: sizeof(size_t) must be >= sizeof(LocalOrdinal).");
1557 static_assert (
sizeof(GST) >=
sizeof(
size_t),
1558 "Tpetra::CrsGraph: sizeof(Tpetra::global_size_t) must be >= sizeof(size_t).");
1566 const char msg[] =
"Tpetra::CrsGraph: Object cannot be created with the "
1567 "given template arguments: size assumptions are not valid.";
1568 TEUCHOS_TEST_FOR_EXCEPTION(
1569 static_cast<size_t> (Teuchos::OrdinalTraits<LO>::max ()) > Teuchos::OrdinalTraits<size_t>::max (),
1570 std::runtime_error, msg);
1571 TEUCHOS_TEST_FOR_EXCEPTION(
1572 static_cast<GST> (Teuchos::OrdinalTraits<LO>::max ()) > static_cast<GST> (Teuchos::OrdinalTraits<GO>::max ()),
1573 std::runtime_error, msg);
1574 TEUCHOS_TEST_FOR_EXCEPTION(
1575 static_cast<size_t> (Teuchos::OrdinalTraits<GO>::max ()) > Teuchos::OrdinalTraits<GST>::max(),
1576 std::runtime_error, msg);
1577 TEUCHOS_TEST_FOR_EXCEPTION(
1578 Teuchos::OrdinalTraits<size_t>::max () > Teuchos::OrdinalTraits<GST>::max (),
1579 std::runtime_error, msg);
1583 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1587 const SLocalGlobalViews &newInds,
1588 const ELocalGlobal lg,
1589 const ELocalGlobal I)
1591 using Teuchos::ArrayView;
1592 typedef LocalOrdinal LO;
1593 typedef GlobalOrdinal GO;
1594 const char tfecfFuncName[] =
"insertIndices: ";
1596 size_t oldNumEnt = 0;
1598 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1599 (lg != GlobalIndices && lg != LocalIndices, std::invalid_argument,
1600 "lg must be either GlobalIndices or LocalIndices.");
1601 oldNumEnt = this->getNumEntriesInLocalRow (rowinfo.localRow);
1604 size_t numNewInds = 0;
1605 if (lg == GlobalIndices) {
1606 ArrayView<const GO> new_ginds = newInds.ginds;
1607 numNewInds = new_ginds.size();
1608 if (I == GlobalIndices) {
1609 auto gind_view = gblInds_wdv.getHostView(Access::ReadWrite);
1611 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1612 (static_cast<size_t> (gind_view.size ()) <
1613 rowinfo.numEntries + numNewInds, std::logic_error,
1614 "gind_view.size() = " << gind_view.size ()
1615 <<
" < rowinfo.numEntries (= " << rowinfo.numEntries
1616 <<
") + numNewInds (= " << numNewInds <<
").");
1618 GO*
const gblColInds_out = gind_view.data () + rowinfo.offset1D
1619 + rowinfo.numEntries;
1620 for (
size_t k = 0; k < numNewInds; ++k) {
1621 gblColInds_out[k] = new_ginds[k];
1624 else if (I == LocalIndices) {
1625 auto lind_view = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
1627 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1628 (static_cast<size_t> (lind_view.size ()) <
1629 rowinfo.numEntries + numNewInds, std::logic_error,
1630 "lind_view.size() = " << lind_view.size ()
1631 <<
" < rowinfo.numEntries (= " << rowinfo.numEntries
1632 <<
") + numNewInds (= " << numNewInds <<
").");
1634 LO*
const lclColInds_out = lind_view.data () + rowinfo.offset1D
1635 + rowinfo.numEntries;
1636 for (
size_t k = 0; k < numNewInds; ++k) {
1637 lclColInds_out[k] = colMap_->getLocalElement (new_ginds[k]);
1641 else if (lg == LocalIndices) {
1642 ArrayView<const LO> new_linds = newInds.linds;
1643 numNewInds = new_linds.size();
1644 if (I == LocalIndices) {
1645 auto lind_view = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
1647 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1648 (static_cast<size_t> (lind_view.size ()) <
1649 rowinfo.numEntries + numNewInds, std::logic_error,
1650 "lind_view.size() = " << lind_view.size ()
1651 <<
" < rowinfo.numEntries (= " << rowinfo.numEntries
1652 <<
") + numNewInds (= " << numNewInds <<
").");
1654 LO*
const lclColInds_out = lind_view.data () + rowinfo.offset1D
1655 + rowinfo.numEntries;
1656 for (
size_t k = 0; k < numNewInds; ++k) {
1657 lclColInds_out[k] = new_linds[k];
1660 else if (I == GlobalIndices) {
1661 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1662 (
true, std::logic_error,
"The case where the input indices are local "
1663 "and the indices to write are global (lg=LocalIndices, I="
1664 "GlobalIndices) is not implemented, because it does not make sense."
1665 << std::endl <<
"If you have correct local column indices, that "
1666 "means the graph has a column Map. In that case, you should be "
1667 "storing local indices.");
1671 rowinfo.numEntries += numNewInds;
1672 this->k_numRowEntries_(rowinfo.localRow) += numNewInds;
1673 this->setLocallyModified ();
1676 const size_t chkNewNumEnt =
1677 this->getNumEntriesInLocalRow (rowinfo.localRow);
1678 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1679 (chkNewNumEnt != oldNumEnt + numNewInds, std::logic_error,
1680 "chkNewNumEnt = " << chkNewNumEnt
1681 <<
" != oldNumEnt (= " << oldNumEnt
1682 <<
") + numNewInds (= " << numNewInds <<
").");
1688 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1692 const GlobalOrdinal inputGblColInds[],
1693 const size_t numInputInds)
1695 return this->insertGlobalIndicesImpl (this->getRowInfo (lclRow),
1696 inputGblColInds, numInputInds);
1699 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1703 const GlobalOrdinal inputGblColInds[],
1704 const size_t numInputInds,
1705 std::function<
void(
const size_t,
const size_t,
const size_t)> fun)
1709 using Kokkos::subview;
1710 using Kokkos::MemoryUnmanaged;
1711 using Teuchos::ArrayView;
1712 using LO = LocalOrdinal;
1713 using GO = GlobalOrdinal;
1714 const char tfecfFuncName[] =
"insertGlobalIndicesImpl: ";
1715 const LO lclRow =
static_cast<LO
> (rowInfo.localRow);
1717 auto numEntries = rowInfo.numEntries;
1718 using inp_view_type = View<const GO*, Kokkos::HostSpace, MemoryUnmanaged>;
1719 inp_view_type inputInds(inputGblColInds, numInputInds);
1722 auto gblIndsHostView = this->gblInds_wdv.getHostView(Access::ReadWrite);
1725 numEntries, inputInds, fun);
1728 const bool insertFailed =
1729 numInserted == Teuchos::OrdinalTraits<size_t>::invalid();
1731 constexpr
size_t ONE (1);
1732 const int myRank = this->getComm()->getRank();
1733 std::ostringstream os;
1735 os <<
"Proc " << myRank <<
": Not enough capacity to insert "
1737 <<
" ind" << (numInputInds != ONE ?
"ices" :
"ex")
1738 <<
" into local row " << lclRow <<
", which currently has "
1739 << rowInfo.numEntries
1740 <<
" entr" << (rowInfo.numEntries != ONE ?
"ies" :
"y")
1741 <<
" and total allocation size " << rowInfo.allocSize
1743 const size_t maxNumToPrint =
1745 ArrayView<const GO> inputGblColIndsView(inputGblColInds,
1748 "column indices", maxNumToPrint);
1750 auto curGblColInds = getGlobalIndsViewHost(rowInfo);
1751 ArrayView<const GO> curGblColIndsView(curGblColInds.data(),
1752 rowInfo.numEntries);
1754 "column indices", maxNumToPrint);
1755 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1756 (
true, std::runtime_error, os.str());
1759 this->k_numRowEntries_(lclRow) += numInserted;
1761 this->setLocallyModified();
1766 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1770 const Teuchos::ArrayView<const LocalOrdinal>& indices,
1771 std::function<
void(
const size_t,
const size_t,
const size_t)> fun)
1773 using Kokkos::MemoryUnmanaged;
1774 using Kokkos::subview;
1776 using LO = LocalOrdinal;
1777 const char tfecfFuncName[] =
"insertLocallIndicesImpl: ";
1779 const RowInfo rowInfo = this->getRowInfo(myRow);
1781 size_t numNewInds = 0;
1782 size_t newNumEntries = 0;
1784 auto numEntries = rowInfo.numEntries;
1786 using inp_view_type = View<const LO*, Kokkos::HostSpace, MemoryUnmanaged>;
1787 inp_view_type inputInds(indices.getRawPtr(), indices.size());
1788 size_t numInserted = 0;
1790 auto lclInds = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
1792 numEntries, inputInds, fun);
1795 const bool insertFailed =
1796 numInserted == Teuchos::OrdinalTraits<size_t>::invalid();
1798 constexpr
size_t ONE (1);
1799 const size_t numInputInds(indices.size());
1800 const int myRank = this->getComm()->getRank();
1801 std::ostringstream os;
1802 os <<
"On MPI Process " << myRank <<
": Not enough capacity to "
1803 "insert " << numInputInds
1804 <<
" ind" << (numInputInds != ONE ?
"ices" :
"ex")
1805 <<
" into local row " << myRow <<
", which currently has "
1806 << rowInfo.numEntries
1807 <<
" entr" << (rowInfo.numEntries != ONE ?
"ies" :
"y")
1808 <<
" and total allocation size " << rowInfo.allocSize <<
".";
1809 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1810 (
true, std::runtime_error, os.str());
1812 numNewInds = numInserted;
1813 newNumEntries = rowInfo.numEntries + numNewInds;
1815 this->k_numRowEntries_(myRow) += numNewInds;
1816 this->setLocallyModified ();
1819 const size_t chkNewNumEntries = this->getNumEntriesInLocalRow (myRow);
1820 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1821 (chkNewNumEntries != newNumEntries, std::logic_error,
1822 "getNumEntriesInLocalRow(" << myRow <<
") = " << chkNewNumEntries
1823 <<
" != newNumEntries = " << newNumEntries
1824 <<
". Please report this bug to the Tpetra developers.");
1828 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1832 const Teuchos::ArrayView<const GlobalOrdinal>& indices,
1833 std::function<
void(
const size_t,
const size_t,
const size_t)> fun)
const
1835 using GO = GlobalOrdinal;
1837 using Kokkos::MemoryUnmanaged;
1838 auto invalidCount = Teuchos::OrdinalTraits<size_t>::invalid();
1840 using inp_view_type = View<const GO*, Kokkos::HostSpace, MemoryUnmanaged>;
1841 inp_view_type inputInds(indices.getRawPtr(), indices.size());
1843 size_t numFound = 0;
1844 LocalOrdinal lclRow = rowInfo.localRow;
1845 if (this->isLocallyIndexed())
1847 if (this->colMap_.is_null())
1848 return invalidCount;
1849 const auto& colMap = *(this->colMap_);
1850 auto map = [&](GO
const gblInd){
return colMap.getLocalElement(gblInd);};
1853 lclIndsUnpacked_wdv.getHostView(Access::ReadOnly), inputInds, map, fun);
1855 else if (this->isGloballyIndexed())
1859 gblInds_wdv.getHostView(Access::ReadOnly), inputInds, fun);
1865 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1872 const size_t origNumEnt = rowInfo.numEntries;
1873 if (origNumEnt != Tpetra::Details::OrdinalTraits<size_t>::invalid () &&
1875 auto lclColInds = this->getLocalIndsViewHostNonConst (rowInfo);
1877 LocalOrdinal*
const lclColIndsRaw = lclColInds.data ();
1879 std::sort (lclColIndsRaw, lclColIndsRaw + origNumEnt);
1883 LocalOrdinal*
const beg = lclColIndsRaw;
1884 LocalOrdinal*
const end = beg + rowInfo.numEntries;
1885 LocalOrdinal*
const newend = std::unique (beg, end);
1886 const size_t newNumEnt = newend - beg;
1889 this->k_numRowEntries_(rowInfo.localRow) = newNumEnt;
1890 return origNumEnt - newNumEnt;
1893 return static_cast<size_t> (0);
1897 return static_cast<size_t> (0);
1902 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1906 const Teuchos::RCP<const map_type>& rangeMap)
1909 if (domainMap_ != domainMap) {
1910 domainMap_ = domainMap;
1911 importer_ = Teuchos::null;
1913 if (rangeMap_ != rangeMap) {
1914 rangeMap_ = rangeMap;
1915 exporter_ = Teuchos::null;
1920 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1925 const auto INV = Teuchos::OrdinalTraits<global_size_t>::invalid();
1927 globalNumEntries_ = INV;
1928 globalMaxNumRowEntries_ = INV;
1929 haveGlobalConstants_ =
false;
1933 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1940 const char tfecfFuncName[] =
"checkInternalState: ";
1941 const char suffix[] =
" Please report this bug to the Tpetra developers.";
1943 std::unique_ptr<std::string> prefix;
1945 prefix = this->createPrefix(
"CrsGraph",
"checkInternalState");
1946 std::ostringstream os;
1947 os << *prefix <<
"Start" << endl;
1948 std::cerr << os.str();
1951 const global_size_t GSTI = Teuchos::OrdinalTraits<global_size_t>::invalid ();
1957 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1958 (this->rowMap_.is_null (), std::logic_error,
1959 "Row Map is null." << suffix);
1962 const LocalOrdinal lclNumRows =
1963 static_cast<LocalOrdinal
> (this->getLocalNumRows ());
1965 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1966 (this->isFillActive () == this->isFillComplete (), std::logic_error,
1967 "Graph cannot be both fill active and fill complete." << suffix);
1968 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1969 (this->isFillComplete () &&
1970 (this->colMap_.is_null () ||
1971 this->rangeMap_.is_null () ||
1972 this->domainMap_.is_null ()),
1974 "Graph is full complete, but at least one of {column, range, domain} "
1975 "Map is null." << suffix);
1976 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1977 (this->isStorageOptimized () && ! this->indicesAreAllocated (),
1978 std::logic_error,
"Storage is optimized, but indices are not "
1979 "allocated, not even trivially." << suffix);
1981 size_t nodeAllocSize = 0;
1983 nodeAllocSize = this->getLocalAllocationSize ();
1985 catch (std::logic_error& e) {
1986 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1987 (
true, std::runtime_error,
"getLocalAllocationSize threw "
1988 "std::logic_error: " << e.what ());
1990 catch (std::exception& e) {
1991 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1992 (
true, std::runtime_error,
"getLocalAllocationSize threw an "
1993 "std::exception: " << e.what ());
1996 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1997 (
true, std::runtime_error,
"getLocalAllocationSize threw an exception "
1998 "not a subclass of std::exception.");
2001 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2002 (this->isStorageOptimized () &&
2003 nodeAllocSize != this->getLocalNumEntries (),
2004 std::logic_error,
"Storage is optimized, but "
2005 "this->getLocalAllocationSize() = " << nodeAllocSize
2006 <<
" != this->getLocalNumEntries() = " << this->getLocalNumEntries ()
2008 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2009 (! this->haveGlobalConstants_ &&
2010 (this->globalNumEntries_ != GSTI ||
2011 this->globalMaxNumRowEntries_ != GSTI),
2012 std::logic_error,
"Graph claims not to have global constants, but "
2013 "some of the global constants are not marked as invalid." << suffix);
2014 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2015 (this->haveGlobalConstants_ &&
2016 (this->globalNumEntries_ == GSTI ||
2017 this->globalMaxNumRowEntries_ == GSTI),
2018 std::logic_error,
"Graph claims to have global constants, but "
2019 "some of them are marked as invalid." << suffix);
2020 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2021 (this->haveGlobalConstants_ &&
2022 (this->globalNumEntries_ < this->getLocalNumEntries () ||
2023 this->globalMaxNumRowEntries_ < this->nodeMaxNumRowEntries_),
2024 std::logic_error,
"Graph claims to have global constants, and "
2025 "all of the values of the global constants are valid, but "
2026 "some of the local constants are greater than "
2027 "their corresponding global constants." << suffix);
2028 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2029 (this->indicesAreAllocated () &&
2030 (this->numAllocForAllRows_ != 0 ||
2031 this->k_numAllocPerRow_.extent (0) != 0),
2032 std::logic_error,
"The graph claims that its indices are allocated, but "
2033 "either numAllocForAllRows_ (= " << this->numAllocForAllRows_ <<
") is "
2034 "nonzero, or k_numAllocPerRow_ has nonzero dimension. In other words, "
2035 "the graph is supposed to release its \"allocation specifications\" "
2036 "when it allocates its indices." << suffix);
2037 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
2038 auto rowPtrsUnpacked_dev = this->getRowPtrsUnpackedDevice();
2039 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2040 (rowPtrsUnpacked_host.extent(0) != rowPtrsUnpacked_dev.extent(0),
2041 std::logic_error,
"The host and device views of k_rowPtrs_ have "
2042 "different sizes; rowPtrsUnpacked_host_ has size "
2043 << rowPtrsUnpacked_host.extent(0)
2044 <<
", but rowPtrsUnpacked_dev_ has size "
2045 << rowPtrsUnpacked_dev.extent(0)
2047 if (isGloballyIndexed() && rowPtrsUnpacked_host.extent(0) != 0) {
2048 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2049 (
size_t(rowPtrsUnpacked_host.extent(0)) !=
size_t(lclNumRows + 1),
2050 std::logic_error,
"The graph is globally indexed and "
2051 "k_rowPtrs has nonzero size " << rowPtrsUnpacked_host.extent(0)
2052 <<
", but that size does not equal lclNumRows+1 = "
2053 << (lclNumRows+1) <<
"." << suffix);
2054 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2055 (rowPtrsUnpacked_host(lclNumRows) !=
size_t(gblInds_wdv.extent(0)),
2056 std::logic_error,
"The graph is globally indexed and "
2057 "k_rowPtrs_ has nonzero size " << rowPtrsUnpacked_host.extent(0)
2058 <<
", but k_rowPtrs_(lclNumRows=" << lclNumRows <<
")="
2059 << rowPtrsUnpacked_host(lclNumRows)
2060 <<
" != gblInds_wdv.extent(0)="
2061 << gblInds_wdv.extent(0) <<
"." << suffix);
2063 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2064 (this->isLocallyIndexed () &&
2065 rowPtrsUnpacked_host.extent (0) != 0 &&
2066 (
static_cast<size_t> (rowPtrsUnpacked_host.extent (0)) !=
2067 static_cast<size_t> (lclNumRows + 1) ||
2068 rowPtrsUnpacked_host(lclNumRows) !=
2069 static_cast<size_t> (this->lclIndsUnpacked_wdv.extent (0))),
2070 std::logic_error,
"If k_rowPtrs_ has nonzero size and "
2071 "the graph is locally indexed, then "
2072 "k_rowPtrs_ must have N+1 rows, and "
2073 "k_rowPtrs_(N) must equal lclIndsUnpacked_wdv.extent(0)." << suffix);
2075 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2076 (this->indicesAreAllocated () &&
2077 nodeAllocSize > 0 &&
2078 this->lclIndsUnpacked_wdv.extent (0) == 0 &&
2079 this->gblInds_wdv.extent (0) == 0,
2080 std::logic_error,
"Graph is allocated nontrivially, but "
2081 "but 1-D allocations are not present." << suffix);
2083 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2084 (! this->indicesAreAllocated () &&
2085 ((rowPtrsUnpacked_host.extent (0) != 0 ||
2086 this->k_numRowEntries_.extent (0) != 0) ||
2087 this->lclIndsUnpacked_wdv.extent (0) != 0 ||
2088 this->gblInds_wdv.extent (0) != 0),
2089 std::logic_error,
"If indices are not allocated, "
2090 "then none of the buffers should be." << suffix);
2094 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2095 ((this->indicesAreLocal_ || this->indicesAreGlobal_) &&
2096 ! this->indicesAreAllocated_,
2097 std::logic_error,
"Indices may be local or global only if they are "
2098 "allocated." << suffix);
2099 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2100 (this->indicesAreLocal_ && this->indicesAreGlobal_,
2101 std::logic_error,
"Indices may not be both local and global." << suffix);
2102 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2103 (indicesAreLocal_ && gblInds_wdv.extent (0) != 0,
2104 std::logic_error,
"Indices are local, but "
2105 "gblInds_wdv.extent(0) (= " << gblInds_wdv.extent (0)
2106 <<
") != 0. In other words, if indices are local, then "
2107 "allocations of global indices should not be present."
2109 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2110 (indicesAreGlobal_ && lclIndsUnpacked_wdv.extent (0) != 0,
2111 std::logic_error,
"Indices are global, but "
2112 "lclIndsUnpacked_wdv.extent(0) (= " << lclIndsUnpacked_wdv.extent(0)
2113 <<
") != 0. In other words, if indices are global, "
2114 "then allocations for local indices should not be present."
2116 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2117 (indicesAreLocal_ && nodeAllocSize > 0 &&
2118 lclIndsUnpacked_wdv.extent (0) == 0 && getLocalNumRows () > 0,
2119 std::logic_error,
"Indices are local and "
2120 "getLocalAllocationSize() = " << nodeAllocSize <<
" > 0, but "
2121 "lclIndsUnpacked_wdv.extent(0) = 0 and getLocalNumRows() = "
2122 << getLocalNumRows () <<
" > 0." << suffix);
2123 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2124 (indicesAreGlobal_ && nodeAllocSize > 0 &&
2125 gblInds_wdv.extent (0) == 0 && getLocalNumRows () > 0,
2126 std::logic_error,
"Indices are global and "
2127 "getLocalAllocationSize() = " << nodeAllocSize <<
" > 0, but "
2128 "gblInds_wdv.extent(0) = 0 and getLocalNumRows() = "
2129 << getLocalNumRows () <<
" > 0." << suffix);
2131 if (this->indicesAreAllocated () &&
2132 rowPtrsUnpacked_host.extent (0) != 0) {
2133 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2134 (static_cast<size_t> (rowPtrsUnpacked_host.extent (0)) !=
2135 this->getLocalNumRows () + 1,
2136 std::logic_error,
"Indices are allocated and "
2137 "k_rowPtrs_ has nonzero length, but rowPtrsUnpacked_host_.extent(0) = "
2138 << rowPtrsUnpacked_host.extent (0) <<
" != getLocalNumRows()+1 = "
2139 << (this->getLocalNumRows () + 1) <<
"." << suffix);
2140 const size_t actualNumAllocated =
2141 rowPtrsUnpacked_host(this->getLocalNumRows());
2142 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2143 (this->isLocallyIndexed () &&
2144 static_cast<size_t> (this->lclIndsUnpacked_wdv.extent (0)) != actualNumAllocated,
2145 std::logic_error,
"Graph is locally indexed, indices are "
2146 "are allocated, and k_rowPtrs_ has nonzero length, but "
2147 "lclIndsUnpacked_wdv.extent(0) = " << this->lclIndsUnpacked_wdv.extent (0)
2148 <<
" != actualNumAllocated = " << actualNumAllocated << suffix);
2149 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2150 (this->isGloballyIndexed () &&
2151 static_cast<size_t> (this->gblInds_wdv.extent (0)) != actualNumAllocated,
2152 std::logic_error,
"Graph is globally indexed, indices "
2153 "are allocated, and k_rowPtrs_ has nonzero length, but "
2154 "gblInds_wdv.extent(0) = " << this->gblInds_wdv.extent (0)
2155 <<
" != actualNumAllocated = " << actualNumAllocated << suffix);
2159 std::ostringstream os;
2160 os << *prefix <<
"Done" << endl;
2161 std::cerr << os.str();
2167 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2172 const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex (globalRow);
2173 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid ()) {
2174 return Teuchos::OrdinalTraits<size_t>::invalid ();
2177 return rowInfo.numEntries;
2182 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2187 const RowInfo rowInfo = this->getRowInfo (localRow);
2188 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid ()) {
2189 return Teuchos::OrdinalTraits<size_t>::invalid ();
2192 return rowInfo.numEntries;
2197 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2202 const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex (globalRow);
2203 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid ()) {
2204 return Teuchos::OrdinalTraits<size_t>::invalid ();
2207 return rowInfo.allocSize;
2212 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2217 const RowInfo rowInfo = this->getRowInfo (localRow);
2218 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid ()) {
2219 return Teuchos::OrdinalTraits<size_t>::invalid ();
2222 return rowInfo.allocSize;
2227 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2228 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::row_ptrs_host_view_type
2232 return getRowPtrsPackedHost();
2235 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2236 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::row_ptrs_device_view_type
2240 return getRowPtrsPackedDevice();
2244 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2245 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::local_inds_host_view_type
2249 return lclIndsPacked_wdv.getHostView(Access::ReadOnly);
2252 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2257 return lclIndsPacked_wdv.getDeviceView(Access::ReadOnly);
2260 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2264 nonconst_local_inds_host_view_type & indices,
2265 size_t& numEntries)
const
2267 using Teuchos::ArrayView;
2268 const char tfecfFuncName[] =
"getLocalRowCopy: ";
2270 TEUCHOS_TEST_FOR_EXCEPTION(
2271 isGloballyIndexed () && ! hasColMap (), std::runtime_error,
2272 "Tpetra::CrsGraph::getLocalRowCopy: The graph is globally indexed and "
2273 "does not have a column Map yet. That means we don't have local indices "
2274 "for columns yet, so it doesn't make sense to call this method. If the "
2275 "graph doesn't have a column Map yet, you should call fillComplete on "
2280 const RowInfo rowinfo = this->getRowInfo (localRow);
2282 const size_t theNumEntries = rowinfo.numEntries;
2283 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2284 (static_cast<size_t> (indices.size ()) < theNumEntries,std::runtime_error,
2285 "Specified storage (size==" << indices.size () <<
") does not suffice "
2286 "to hold all " << theNumEntries <<
" entry/ies for this row.");
2287 numEntries = theNumEntries;
2289 if (rowinfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid ()) {
2290 if (isLocallyIndexed ()) {
2291 auto lclInds = getLocalIndsViewHost(rowinfo);
2292 for (
size_t j = 0; j < theNumEntries; ++j) {
2293 indices[j] = lclInds(j);
2296 else if (isGloballyIndexed ()) {
2297 auto gblInds = getGlobalIndsViewHost(rowinfo);
2298 for (
size_t j = 0; j < theNumEntries; ++j) {
2299 indices[j] = colMap_->getLocalElement (gblInds(j));
2306 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2310 nonconst_global_inds_host_view_type &indices,
2311 size_t& numEntries)
const
2313 using Teuchos::ArrayView;
2314 const char tfecfFuncName[] =
"getGlobalRowCopy: ";
2318 const RowInfo rowinfo = getRowInfoFromGlobalRowIndex (globalRow);
2319 const size_t theNumEntries = rowinfo.numEntries;
2320 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2321 static_cast<size_t> (indices.size ()) < theNumEntries, std::runtime_error,
2322 "Specified storage (size==" << indices.size () <<
") does not suffice "
2323 "to hold all " << theNumEntries <<
" entry/ies for this row.");
2324 numEntries = theNumEntries;
2326 if (rowinfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid ()) {
2327 if (isLocallyIndexed ()) {
2328 auto lclInds = getLocalIndsViewHost(rowinfo);
2329 for (
size_t j = 0; j < theNumEntries; ++j) {
2330 indices[j] = colMap_->getGlobalElement (lclInds(j));
2333 else if (isGloballyIndexed ()) {
2334 auto gblInds = getGlobalIndsViewHost(rowinfo);
2335 for (
size_t j = 0; j < theNumEntries; ++j) {
2336 indices[j] = gblInds(j);
2343 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2347 const LocalOrdinal localRow,
2348 local_inds_host_view_type &indices)
const
2350 const char tfecfFuncName[] =
"getLocalRowView: ";
2352 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2353 (isGloballyIndexed (), std::runtime_error,
"The graph's indices are "
2354 "currently stored as global indices, so we cannot return a view with "
2355 "local column indices, whether or not the graph has a column Map. If "
2356 "the graph _does_ have a column Map, use getLocalRowCopy() instead.");
2358 const RowInfo rowInfo = getRowInfo (localRow);
2359 if (rowInfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid () &&
2360 rowInfo.numEntries > 0) {
2361 indices = lclIndsUnpacked_wdv.getHostSubview(rowInfo.offset1D,
2368 indices = local_inds_host_view_type();
2372 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2373 (static_cast<size_t> (indices.size ()) !=
2374 getNumEntriesInLocalRow (localRow), std::logic_error,
"indices.size() "
2375 "= " << indices.extent(0) <<
" != getNumEntriesInLocalRow(localRow=" <<
2376 localRow <<
") = " << getNumEntriesInLocalRow(localRow) <<
2377 ". Please report this bug to the Tpetra developers.");
2382 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2386 const GlobalOrdinal globalRow,
2387 global_inds_host_view_type &indices)
const
2389 const char tfecfFuncName[] =
"getGlobalRowView: ";
2391 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2392 (isLocallyIndexed (), std::runtime_error,
"The graph's indices are "
2393 "currently stored as local indices, so we cannot return a view with "
2394 "global column indices. Use getGlobalRowCopy() instead.");
2398 const RowInfo rowInfo = getRowInfoFromGlobalRowIndex (globalRow);
2399 if (rowInfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid () &&
2400 rowInfo.numEntries > 0) {
2401 indices = gblInds_wdv.getHostSubview(rowInfo.offset1D,
2406 indices =
typename global_inds_dualv_type::t_host::const_type();
2409 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2410 (static_cast<size_t> (indices.size ()) !=
2411 getNumEntriesInGlobalRow (globalRow),
2412 std::logic_error,
"indices.size() = " << indices.extent(0)
2413 <<
" != getNumEntriesInGlobalRow(globalRow=" << globalRow <<
") = "
2414 << getNumEntriesInGlobalRow (globalRow)
2415 <<
". Please report this bug to the Tpetra developers.");
2420 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2424 const Teuchos::ArrayView<const LocalOrdinal>& indices)
2426 const char tfecfFuncName[] =
"insertLocalIndices: ";
2428 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2429 (! isFillActive (), std::runtime_error,
"Fill must be active.");
2430 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2431 (isGloballyIndexed (), std::runtime_error,
2432 "Graph indices are global; use insertGlobalIndices().");
2433 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2434 (! hasColMap (), std::runtime_error,
2435 "Cannot insert local indices without a column Map.");
2436 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2437 (! rowMap_->isNodeLocalElement (localRow), std::runtime_error,
2438 "Local row index " << localRow <<
" is not in the row Map "
2439 "on the calling process.");
2440 if (! indicesAreAllocated ()) {
2441 allocateIndices (LocalIndices, verbose_);
2450 using Teuchos::Array;
2451 using Teuchos::toString;
2453 typedef typename Teuchos::ArrayView<const LocalOrdinal>::size_type size_type;
2456 Array<LocalOrdinal> badColInds;
2457 bool allInColMap =
true;
2458 for (size_type k = 0; k < indices.size (); ++k) {
2460 allInColMap =
false;
2461 badColInds.push_back (indices[k]);
2464 if (! allInColMap) {
2465 std::ostringstream os;
2466 os <<
"Tpetra::CrsGraph::insertLocalIndices: You attempted to insert "
2467 "entries in owned row " << localRow <<
", at the following column "
2468 "indices: " << toString (indices) <<
"." << endl;
2469 os <<
"Of those, the following indices are not in the column Map on "
2470 "this process: " << toString (badColInds) <<
"." << endl <<
"Since "
2471 "the graph has a column Map already, it is invalid to insert entries "
2472 "at those locations.";
2473 TEUCHOS_TEST_FOR_EXCEPTION(! allInColMap, std::invalid_argument, os.str ());
2478 insertLocalIndicesImpl (localRow, indices);
2481 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2482 (! indicesAreAllocated () || ! isLocallyIndexed (), std::logic_error,
2483 "At the end of insertLocalIndices, ! indicesAreAllocated() || "
2484 "! isLocallyIndexed() is true. Please report this bug to the "
2485 "Tpetra developers.");
2489 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2493 const LocalOrdinal numEnt,
2494 const LocalOrdinal inds[])
2496 Teuchos::ArrayView<const LocalOrdinal> indsT (inds, numEnt);
2497 this->insertLocalIndices (localRow, indsT);
2501 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2505 const LocalOrdinal numInputInds,
2506 const GlobalOrdinal inputGblColInds[])
2508 typedef LocalOrdinal LO;
2509 const char tfecfFuncName[] =
"insertGlobalIndices: ";
2511 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2512 (this->isLocallyIndexed (), std::runtime_error,
2513 "graph indices are local; use insertLocalIndices().");
2518 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2519 (! this->isFillActive (), std::runtime_error,
2520 "You are not allowed to call this method if fill is not active. "
2521 "If fillComplete has been called, you must first call resumeFill "
2522 "before you may insert indices.");
2523 if (! indicesAreAllocated ()) {
2524 allocateIndices (GlobalIndices, verbose_);
2526 const LO lclRow = this->rowMap_->getLocalElement (gblRow);
2527 if (lclRow != Tpetra::Details::OrdinalTraits<LO>::invalid ()) {
2529 if (this->hasColMap ()) {
2531 const map_type& colMap = * (this->colMap_);
2536 std::vector<GlobalOrdinal> badColInds;
2537 bool allInColMap =
true;
2538 for (LO k = 0; k < numInputInds; ++k) {
2540 allInColMap =
false;
2541 badColInds.push_back (inputGblColInds[k]);
2544 if (! allInColMap) {
2545 std::ostringstream os;
2546 os <<
"You attempted to insert entries in owned row " << gblRow
2547 <<
", at the following column indices: [";
2548 for (LO k = 0; k < numInputInds; ++k) {
2549 os << inputGblColInds[k];
2550 if (k + static_cast<LO> (1) < numInputInds) {
2554 os <<
"]." << endl <<
"Of those, the following indices are not in "
2555 "the column Map on this process: [";
2556 for (
size_t k = 0; k < badColInds.size (); ++k) {
2557 os << badColInds[k];
2558 if (k +
size_t (1) < badColInds.size ()) {
2562 os <<
"]." << endl <<
"Since the matrix has a column Map already, "
2563 "it is invalid to insert entries at those locations.";
2564 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2565 (
true, std::invalid_argument, os.str ());
2569 this->insertGlobalIndicesImpl (lclRow, inputGblColInds, numInputInds);
2572 this->insertGlobalIndicesIntoNonownedRows (gblRow, inputGblColInds,
2578 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2582 const Teuchos::ArrayView<const GlobalOrdinal>& inputGblColInds)
2584 this->insertGlobalIndices (gblRow, inputGblColInds.size (),
2585 inputGblColInds.getRawPtr ());
2589 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2593 const GlobalOrdinal gblColInds[],
2594 const LocalOrdinal numGblColInds)
2596 typedef LocalOrdinal LO;
2597 typedef GlobalOrdinal GO;
2598 const char tfecfFuncName[] =
"insertGlobalIndicesFiltered: ";
2600 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2601 (this->isLocallyIndexed (), std::runtime_error,
2602 "Graph indices are local; use insertLocalIndices().");
2607 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2608 (! this->isFillActive (), std::runtime_error,
2609 "You are not allowed to call this method if fill is not active. "
2610 "If fillComplete has been called, you must first call resumeFill "
2611 "before you may insert indices.");
2612 if (! indicesAreAllocated ()) {
2613 allocateIndices (GlobalIndices, verbose_);
2616 Teuchos::ArrayView<const GO> gblColInds_av (gblColInds, numGblColInds);
2618 if (! colMap_.is_null ()) {
2619 const map_type& colMap = * (this->colMap_);
2622 while (curOffset < numGblColInds) {
2626 LO endOffset = curOffset;
2627 for ( ; endOffset < numGblColInds; ++endOffset) {
2629 if (lclCol == Tpetra::Details::OrdinalTraits<LO>::invalid ()) {
2636 const LO numIndInSeq = (endOffset - curOffset);
2637 if (numIndInSeq != 0) {
2638 this->insertGlobalIndicesImpl (lclRow, gblColInds + curOffset,
2644 curOffset = endOffset + 1;
2648 this->insertGlobalIndicesImpl (lclRow, gblColInds_av.getRawPtr (),
2649 gblColInds_av.size ());
2653 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2657 const GlobalOrdinal gblColInds[],
2658 const LocalOrdinal numGblColInds)
2663 std::vector<GlobalOrdinal>& nonlocalRow = this->nonlocals_[gblRow];
2664 for (LocalOrdinal k = 0; k < numGblColInds; ++k) {
2668 nonlocalRow.push_back (gblColInds[k]);
2672 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2677 const char tfecfFuncName[] =
"removeLocalIndices: ";
2678 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2679 ! isFillActive (), std::runtime_error,
"requires that fill is active.");
2680 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2681 isStorageOptimized (), std::runtime_error,
2682 "cannot remove indices after optimizeStorage() has been called.");
2683 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2684 isGloballyIndexed (), std::runtime_error,
"graph indices are global.");
2685 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2686 ! rowMap_->isNodeLocalElement (lrow), std::runtime_error,
2687 "Local row " << lrow <<
" is not in the row Map on the calling process.");
2688 if (! indicesAreAllocated ()) {
2689 allocateIndices (LocalIndices, verbose_);
2692 if (k_numRowEntries_.extent (0) != 0) {
2693 this->k_numRowEntries_(lrow) = 0;
2697 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2698 (getNumEntriesInLocalRow (lrow) != 0 ||
2699 ! indicesAreAllocated () ||
2700 ! isLocallyIndexed (), std::logic_error,
2701 "Violated stated post-conditions. Please contact Tpetra team.");
2706 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2709 setAllIndices (
const typename local_graph_device_type::row_map_type& rowPointers,
2710 const typename local_graph_device_type::entries_type::non_const_type& columnIndices)
2713 ProfilingRegion region (
"Tpetra::CrsGraph::setAllIndices");
2714 const char tfecfFuncName[] =
"setAllIndices: ";
2715 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2716 ! hasColMap () || getColMap ().is_null (), std::runtime_error,
2717 "The graph must have a column Map before you may call this method.");
2718 LocalOrdinal numLocalRows = this->getLocalNumRows ();
2720 LocalOrdinal rowPtrLen = rowPointers.size();
2721 if(numLocalRows == 0) {
2722 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2723 rowPtrLen != 0 && rowPtrLen != 1,
2724 std::runtime_error,
"Have 0 local rows, but rowPointers.size() is neither 0 nor 1.");
2727 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2728 rowPtrLen != numLocalRows + 1,
2729 std::runtime_error,
"rowPointers.size() = " << rowPtrLen <<
2730 " != this->getLocalNumRows()+1 = " << (numLocalRows + 1) <<
".");
2735 using exec_space =
typename local_graph_device_type::execution_space;
2736 int columnsOutOfBounds = 0;
2738 Kokkos::parallel_reduce(Kokkos::RangePolicy<exec_space>(0, columnIndices.extent(0)),
2739 KOKKOS_LAMBDA (
const LocalOrdinal i,
int& lOutOfBounds)
2741 if(columnIndices(i) < 0 || columnIndices(i) >= numLocalCols)
2743 }, columnsOutOfBounds);
2744 int globalColsOutOfBounds= 0;
2745 auto comm = this->getComm();
2746 Teuchos::reduceAll<int, int> (*comm, Teuchos::REDUCE_MAX, columnsOutOfBounds,
2747 Teuchos::outArg (globalColsOutOfBounds));
2748 if (globalColsOutOfBounds)
2750 std::string message;
2751 if (columnsOutOfBounds)
2754 message = std::string(
"ERROR, rank ") + std::to_string(comm->getRank()) +
", CrsGraph::setAllIndices(): provided columnIndices are not all within range [0, getLocalNumCols())!\n";
2757 throw std::invalid_argument(
"CrsGraph::setAllIndices(): columnIndices are out of the valid range on at least one process.");
2761 if (debug_ && this->isSorted()) {
2764 using exec_space =
typename local_graph_device_type::execution_space;
2765 using size_type =
typename local_graph_device_type::size_type;
2766 Kokkos::parallel_reduce(Kokkos::RangePolicy<exec_space>(0, numLocalRows),
2767 KOKKOS_LAMBDA (
const LocalOrdinal i,
int& lNotSorted)
2769 size_type rowBegin = rowPointers(i);
2770 size_type rowEnd = rowPointers(i + 1);
2771 for(size_type j = rowBegin + 1; j < rowEnd; j++)
2773 if(columnIndices(j - 1) > columnIndices(j))
2780 int globalNotSorted = 0;
2781 auto comm = this->getComm();
2782 Teuchos::reduceAll<int, int> (*comm, Teuchos::REDUCE_MAX, notSorted,
2783 Teuchos::outArg (globalNotSorted));
2784 if (globalNotSorted)
2786 std::string message;
2790 message = std::string(
"ERROR, rank ") + std::to_string(comm->getRank()) +
", CrsGraph::setAllIndices(): provided columnIndices are not sorted!\n";
2793 throw std::invalid_argument(
"CrsGraph::setAllIndices(): provided columnIndices are not sorted within rows on at least one process.");
2797 indicesAreAllocated_ =
true;
2798 indicesAreLocal_ =
true;
2799 indicesAreSorted_ =
true;
2800 noRedundancies_ =
true;
2802 lclIndsUnpacked_wdv = lclIndsPacked_wdv;
2803 setRowPtrs(rowPointers);
2805 set_need_sync_host_uvm_access();
2809 storageStatus_ = Details::STORAGE_1D_PACKED;
2814 numAllocForAllRows_ = 0;
2815 k_numAllocPerRow_ = decltype (k_numAllocPerRow_) ();
2817 checkInternalState ();
2821 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2825 const Teuchos::ArrayRCP<LocalOrdinal>& columnIndices)
2828 typedef typename local_graph_device_type::row_map_type row_map_type;
2829 typedef typename row_map_type::array_layout layout_type;
2830 typedef typename row_map_type::non_const_value_type row_offset_type;
2831 typedef View<
size_t*, layout_type , Kokkos::HostSpace,
2832 Kokkos::MemoryUnmanaged> input_view_type;
2833 typedef typename row_map_type::non_const_type nc_row_map_type;
2835 const size_t size =
static_cast<size_t> (rowPointers.size ());
2836 constexpr
bool same = std::is_same<size_t, row_offset_type>::value;
2837 input_view_type ptr_in (rowPointers.getRawPtr (), size);
2839 nc_row_map_type ptr_rot (
"Tpetra::CrsGraph::ptr", size);
2842 #ifdef KOKKOS_ENABLE_CXX17
2843 if constexpr (same) {
2844 using lexecution_space =
typename device_type::execution_space;
2854 input_view_type ptr_decoy (rowPointers.getRawPtr (), size);
2857 Kokkos::Impl::if_c<same,
2859 input_view_type>::select (ptr_rot, ptr_decoy),
2865 constexpr
bool inHostMemory =
2866 std::is_same<
typename row_map_type::memory_space,
2867 Kokkos::HostSpace>::value;
2878 View<size_t*, layout_type, device_type> ptr_st (
"Tpetra::CrsGraph::ptr", size);
2890 Kokkos::View<LocalOrdinal*, layout_type, device_type> k_ind =
2891 Kokkos::Compat::getKokkosViewDeepCopy<device_type> (columnIndices ());
2892 setAllIndices (ptr_rot, k_ind);
2896 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2901 using Teuchos::Comm;
2902 using Teuchos::outArg;
2905 using Teuchos::REDUCE_MAX;
2906 using Teuchos::REDUCE_MIN;
2907 using Teuchos::reduceAll;
2912 using size_type =
typename Teuchos::Array<GO>::size_type;
2913 const char tfecfFuncName[] =
"globalAssemble: ";
2915 std::unique_ptr<std::string> prefix;
2917 prefix = this->createPrefix(
"CrsGraph",
"globalAssemble");
2918 std::ostringstream os;
2919 os << *prefix <<
"Start" << endl;
2920 std::cerr << os.str();
2922 RCP<const Comm<int> > comm = getComm ();
2924 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2925 (! isFillActive (), std::runtime_error,
"Fill must be active before "
2926 "you may call this method.");
2928 const size_t myNumNonlocalRows = this->nonlocals_.size ();
2935 const int iHaveNonlocalRows = (myNumNonlocalRows == 0) ? 0 : 1;
2936 int someoneHasNonlocalRows = 0;
2937 reduceAll<int, int> (*comm, REDUCE_MAX, iHaveNonlocalRows,
2938 outArg (someoneHasNonlocalRows));
2939 if (someoneHasNonlocalRows == 0) {
2941 std::ostringstream os;
2942 os << *prefix <<
"Done: No nonlocal rows" << endl;
2943 std::cerr << os.str();
2947 else if (verbose_) {
2948 std::ostringstream os;
2949 os << *prefix <<
"At least 1 process has nonlocal rows"
2951 std::cerr << os.str();
2964 RCP<const map_type> nonlocalRowMap;
2966 Teuchos::Array<size_t> numEntPerNonlocalRow (myNumNonlocalRows);
2968 Teuchos::Array<GO> myNonlocalGblRows (myNumNonlocalRows);
2969 size_type curPos = 0;
2970 for (
auto mapIter = this->nonlocals_.begin ();
2971 mapIter != this->nonlocals_.end ();
2972 ++mapIter, ++curPos) {
2973 myNonlocalGblRows[curPos] = mapIter->first;
2974 std::vector<GO>& gblCols = mapIter->second;
2975 std::sort (gblCols.begin (), gblCols.end ());
2976 auto vecLast = std::unique (gblCols.begin (), gblCols.end ());
2977 gblCols.erase (vecLast, gblCols.end ());
2978 numEntPerNonlocalRow[curPos] = gblCols.size ();
2989 GO myMinNonlocalGblRow = std::numeric_limits<GO>::max ();
2991 auto iter = std::min_element (myNonlocalGblRows.begin (),
2992 myNonlocalGblRows.end ());
2993 if (iter != myNonlocalGblRows.end ()) {
2994 myMinNonlocalGblRow = *iter;
2997 GO gblMinNonlocalGblRow = 0;
2998 reduceAll<int, GO> (*comm, REDUCE_MIN, myMinNonlocalGblRow,
2999 outArg (gblMinNonlocalGblRow));
3000 const GO indexBase = gblMinNonlocalGblRow;
3001 const global_size_t INV = Teuchos::OrdinalTraits<global_size_t>::invalid ();
3002 nonlocalRowMap = rcp (
new map_type (INV, myNonlocalGblRows (), indexBase, comm));
3006 std::ostringstream os;
3007 os << *prefix <<
"nonlocalRowMap->getIndexBase()="
3008 << nonlocalRowMap->getIndexBase() << endl;
3009 std::cerr << os.str();
3017 RCP<crs_graph_type> nonlocalGraph =
3018 rcp(
new crs_graph_type(nonlocalRowMap, numEntPerNonlocalRow()));
3020 size_type curPos = 0;
3021 for (
auto mapIter = this->nonlocals_.begin ();
3022 mapIter != this->nonlocals_.end ();
3023 ++mapIter, ++curPos) {
3024 const GO gblRow = mapIter->first;
3025 std::vector<GO>& gblCols = mapIter->second;
3026 const LO numEnt =
static_cast<LO
> (numEntPerNonlocalRow[curPos]);
3027 nonlocalGraph->insertGlobalIndices (gblRow, numEnt, gblCols.data ());
3031 std::ostringstream os;
3032 os << *prefix <<
"Built nonlocal graph" << endl;
3033 std::cerr << os.str();
3044 auto origRowMap = this->getRowMap ();
3045 const bool origRowMapIsOneToOne = origRowMap->isOneToOne ();
3047 if (origRowMapIsOneToOne) {
3049 std::ostringstream os;
3050 os << *prefix <<
"Original row Map is 1-to-1" << endl;
3051 std::cerr << os.str();
3053 export_type exportToOrig (nonlocalRowMap, origRowMap);
3059 std::ostringstream os;
3060 os << *prefix <<
"Original row Map is NOT 1-to-1" << endl;
3061 std::cerr << os.str();
3068 export_type exportToOneToOne (nonlocalRowMap, oneToOneRowMap);
3074 crs_graph_type oneToOneGraph (oneToOneRowMap, 0);
3078 std::ostringstream os;
3079 os << *prefix <<
"Export nonlocal graph" << endl;
3080 std::cerr << os.str();
3082 oneToOneGraph.doExport (*nonlocalGraph, exportToOneToOne,
Tpetra::INSERT);
3086 nonlocalGraph = Teuchos::null;
3089 import_type importToOrig (oneToOneRowMap, origRowMap);
3091 std::ostringstream os;
3092 os << *prefix <<
"Import nonlocal graph" << endl;
3093 std::cerr << os.str();
3102 decltype (this->nonlocals_) newNonlocals;
3103 std::swap (this->nonlocals_, newNonlocals);
3105 checkInternalState ();
3107 std::ostringstream os;
3108 os << *prefix <<
"Done" << endl;
3109 std::cerr << os.str();
3114 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3119 clearGlobalConstants();
3120 if (params != Teuchos::null) this->setParameterList (params);
3122 indicesAreSorted_ =
true;
3123 noRedundancies_ =
true;
3124 fillComplete_ =
false;
3128 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3143 Teuchos::RCP<const map_type> domMap = this->getDomainMap ();
3144 if (domMap.is_null ()) {
3145 domMap = this->getRowMap ();
3147 Teuchos::RCP<const map_type> ranMap = this->getRangeMap ();
3148 if (ranMap.is_null ()) {
3149 ranMap = this->getRowMap ();
3151 this->fillComplete (domMap, ranMap, params);
3155 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3159 const Teuchos::RCP<const map_type>& rangeMap,
3160 const Teuchos::RCP<Teuchos::ParameterList>& params)
3163 const char tfecfFuncName[] =
"fillComplete: ";
3164 const bool verbose = verbose_;
3166 std::unique_ptr<std::string> prefix;
3168 prefix = this->createPrefix(
"CrsGraph",
"fillComplete");
3169 std::ostringstream os;
3170 os << *prefix <<
"Start" << endl;
3171 std::cerr << os.str();
3174 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3175 (! isFillActive () || isFillComplete (), std::runtime_error,
3176 "Graph fill state must be active (isFillActive() "
3177 "must be true) before calling fillComplete().");
3179 const int numProcs = getComm ()->getSize ();
3187 if (! params.is_null ()) {
3188 if (params->isParameter (
"sort column map ghost gids")) {
3189 sortGhostsAssociatedWithEachProcessor_ =
3190 params->get<
bool> (
"sort column map ghost gids",
3191 sortGhostsAssociatedWithEachProcessor_);
3193 else if (params->isParameter (
"Sort column Map ghost GIDs")) {
3194 sortGhostsAssociatedWithEachProcessor_ =
3195 params->get<
bool> (
"Sort column Map ghost GIDs",
3196 sortGhostsAssociatedWithEachProcessor_);
3202 bool assertNoNonlocalInserts =
false;
3203 if (! params.is_null ()) {
3204 assertNoNonlocalInserts =
3205 params->get<
bool> (
"No Nonlocal Changes", assertNoNonlocalInserts);
3211 if (! indicesAreAllocated ()) {
3214 allocateIndices (LocalIndices, verbose);
3217 allocateIndices (GlobalIndices, verbose);
3225 const bool mayNeedGlobalAssemble = ! assertNoNonlocalInserts && numProcs > 1;
3226 if (mayNeedGlobalAssemble) {
3232 const size_t numNonlocals = nonlocals_.size();
3234 std::ostringstream os;
3235 os << *prefix <<
"Do not need to call globalAssemble; "
3236 "assertNoNonlocalInserts="
3237 << (assertNoNonlocalInserts ?
"true" :
"false")
3238 <<
"numProcs=" << numProcs
3239 <<
", nonlocals_.size()=" << numNonlocals << endl;
3240 std::cerr << os.str();
3242 const int lclNeededGlobalAssemble =
3243 (numProcs > 1 && numNonlocals != 0) ? 1 : 0;
3244 if (lclNeededGlobalAssemble != 0 && verbose) {
3245 std::ostringstream os;
3247 Details::Impl::verbosePrintMap(
3248 os, nonlocals_.begin(), nonlocals_.end(),
3249 nonlocals_.size(),
"nonlocals_");
3250 std::cerr << os.str() << endl;
3254 auto map = this->getMap();
3255 auto comm = map.is_null() ? Teuchos::null : map->getComm();
3256 int gblNeededGlobalAssemble = lclNeededGlobalAssemble;
3257 if (! comm.is_null()) {
3258 using Teuchos::REDUCE_MAX;
3259 using Teuchos::reduceAll;
3260 reduceAll(*comm, REDUCE_MAX, lclNeededGlobalAssemble,
3261 Teuchos::outArg(gblNeededGlobalAssemble));
3263 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3264 (gblNeededGlobalAssemble != 0, std::runtime_error,
3265 "nonlocals_.size()=" << numNonlocals <<
" != 0 on at "
3266 "least one process in the CrsGraph's communicator. This "
3267 "means either that you incorrectly set the "
3268 "\"No Nonlocal Changes\" fillComplete parameter to true, "
3269 "or that you inserted invalid entries. "
3270 "Rerun with the environment variable TPETRA_VERBOSE="
3271 "CrsGraph set to see the entries of nonlocals_ on every "
3272 "MPI process (WARNING: lots of output).");
3275 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3276 (lclNeededGlobalAssemble != 0, std::runtime_error,
3277 "nonlocals_.size()=" << numNonlocals <<
" != 0 on the "
3278 "calling process. This means either that you incorrectly "
3279 "set the \"No Nonlocal Changes\" fillComplete parameter "
3280 "to true, or that you inserted invalid entries. "
3281 "Rerun with the environment "
3282 "variable TPETRA_VERBOSE=CrsGraph set to see the entries "
3283 "of nonlocals_ on every MPI process (WARNING: lots of "
3290 setDomainRangeMaps (domainMap, rangeMap);
3296 Teuchos::Array<int> remotePIDs (0);
3297 const bool mustBuildColMap = ! this->hasColMap ();
3298 if (mustBuildColMap) {
3304 const std::pair<size_t, std::string> makeIndicesLocalResult =
3305 this->makeIndicesLocal(verbose);
3310 using Teuchos::REDUCE_MIN;
3311 using Teuchos::reduceAll;
3312 using Teuchos::outArg;
3314 RCP<const map_type> map = this->getMap ();
3315 RCP<const Teuchos::Comm<int> > comm;
3316 if (! map.is_null ()) {
3317 comm = map->getComm ();
3319 if (comm.is_null ()) {
3320 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3321 (makeIndicesLocalResult.first != 0, std::runtime_error,
3322 makeIndicesLocalResult.second);
3325 const int lclSuccess = (makeIndicesLocalResult.first == 0);
3327 reduceAll (*comm, REDUCE_MIN, lclSuccess, outArg (gblSuccess));
3328 if (gblSuccess != 1) {
3329 std::ostringstream os;
3330 gathervPrint (os, makeIndicesLocalResult.second, *comm);
3331 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3332 (
true, std::runtime_error, os.str ());
3341 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3342 (makeIndicesLocalResult.first != 0, std::runtime_error,
3343 makeIndicesLocalResult.second);
3349 this->sortAndMergeAllIndices (this->isSorted (), this->isMerged ());
3354 this->makeImportExport (remotePIDs, mustBuildColMap);
3357 this->fillLocalGraph (params);
3359 const bool callComputeGlobalConstants = params.get () ==
nullptr ||
3360 params->get (
"compute global constants",
true);
3361 if (callComputeGlobalConstants) {
3362 this->computeGlobalConstants ();
3365 this->computeLocalConstants ();
3367 this->fillComplete_ =
true;
3368 this->checkInternalState ();
3371 std::ostringstream os;
3372 os << *prefix <<
"Done" << endl;
3373 std::cerr << os.str();
3378 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3382 const Teuchos::RCP<const map_type>& rangeMap,
3383 const Teuchos::RCP<const import_type>& importer,
3384 const Teuchos::RCP<const export_type>& exporter,
3385 const Teuchos::RCP<Teuchos::ParameterList>& params)
3387 const char tfecfFuncName[] =
"expertStaticFillComplete: ";
3388 #ifdef HAVE_TPETRA_MMM_TIMINGS
3390 if(!params.is_null())
3391 label = params->get(
"Timer Label",label);
3392 std::string prefix = std::string(
"Tpetra ")+ label + std::string(
": ");
3393 using Teuchos::TimeMonitor;
3394 Teuchos::RCP<Teuchos::TimeMonitor> MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-Setup"))));
3398 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3399 domainMap.is_null () || rangeMap.is_null (),
3400 std::runtime_error,
"The input domain Map and range Map must be nonnull.");
3401 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3402 isFillComplete () || ! hasColMap (), std::runtime_error,
"You may not "
3403 "call this method unless the graph has a column Map.");
3404 auto rowPtrsUnpackedLength = this->getRowPtrsUnpackedHost().extent (0);
3405 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3406 getLocalNumRows () > 0 && rowPtrsUnpackedLength == 0,
3407 std::runtime_error,
"The calling process has getLocalNumRows() = "
3408 << getLocalNumRows () <<
" > 0 rows, but the row offsets array has not "
3410 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3411 static_cast<size_t> (rowPtrsUnpackedLength) != getLocalNumRows () + 1,
3412 std::runtime_error,
"The row offsets array has length " <<
3413 rowPtrsUnpackedLength <<
" != getLocalNumRows()+1 = " <<
3414 (getLocalNumRows () + 1) <<
".");
3429 numAllocForAllRows_ = 0;
3430 k_numAllocPerRow_ = decltype (k_numAllocPerRow_) ();
3431 indicesAreAllocated_ =
true;
3436 indicesAreLocal_ =
true;
3437 indicesAreGlobal_ =
false;
3440 #ifdef HAVE_TPETRA_MMM_TIMINGS
3442 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-Maps"))));
3444 setDomainRangeMaps (domainMap, rangeMap);
3447 indicesAreSorted_ =
true;
3448 noRedundancies_ =
true;
3451 #ifdef HAVE_TPETRA_MMM_TIMINGS
3453 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-mIXcheckI"))));
3456 importer_ = Teuchos::null;
3457 exporter_ = Teuchos::null;
3458 if (importer != Teuchos::null) {
3459 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3460 ! importer->getSourceMap ()->isSameAs (*getDomainMap ()) ||
3461 ! importer->getTargetMap ()->isSameAs (*getColMap ()),
3462 std::invalid_argument,
": importer does not match matrix maps.");
3463 importer_ = importer;
3467 #ifdef HAVE_TPETRA_MMM_TIMINGS
3469 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-mIXcheckE"))));
3472 if (exporter != Teuchos::null) {
3473 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3474 ! exporter->getSourceMap ()->isSameAs (*getRowMap ()) ||
3475 ! exporter->getTargetMap ()->isSameAs (*getRangeMap ()),
3476 std::invalid_argument,
": exporter does not match matrix maps.");
3477 exporter_ = exporter;
3480 #ifdef HAVE_TPETRA_MMM_TIMINGS
3482 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-mIXmake"))));
3484 Teuchos::Array<int> remotePIDs (0);
3485 this->makeImportExport (remotePIDs,
false);
3487 #ifdef HAVE_TPETRA_MMM_TIMINGS
3489 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-fLG"))));
3491 this->fillLocalGraph (params);
3493 const bool callComputeGlobalConstants = params.get () ==
nullptr ||
3494 params->get (
"compute global constants",
true);
3496 if (callComputeGlobalConstants) {
3497 #ifdef HAVE_TPETRA_MMM_TIMINGS
3499 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-cGC (const)"))));
3500 #endif // HAVE_TPETRA_MMM_TIMINGS
3501 this->computeGlobalConstants ();
3504 #ifdef HAVE_TPETRA_MMM_TIMINGS
3506 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-cGC (noconst)"))));
3507 #endif // HAVE_TPETRA_MMM_TIMINGS
3508 this->computeLocalConstants ();
3511 fillComplete_ =
true;
3513 #ifdef HAVE_TPETRA_MMM_TIMINGS
3515 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-cIS"))));
3517 checkInternalState ();
3521 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3524 fillLocalGraph (
const Teuchos::RCP<Teuchos::ParameterList>& params)
3527 typedef typename local_graph_device_type::row_map_type row_map_type;
3528 typedef typename row_map_type::non_const_type non_const_row_map_type;
3529 typedef typename local_graph_device_type::entries_type::non_const_type lclinds_1d_type;
3530 const char tfecfFuncName[] =
"fillLocalGraph (called from fillComplete or "
3531 "expertStaticFillComplete): ";
3532 const size_t lclNumRows = this->getLocalNumRows ();
3537 bool requestOptimizedStorage =
true;
3538 if (! params.is_null () && ! params->get (
"Optimize Storage",
true)) {
3539 requestOptimizedStorage =
false;
3547 auto rowPtrsUnpacked = this->getRowPtrsUnpackedHost();
3549 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3550 (rowPtrsUnpacked.extent (0) == 0, std::logic_error,
3551 "rowPtrsUnpacked_host_ has size zero, but shouldn't");
3552 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3553 (rowPtrsUnpacked.extent (0) != lclNumRows + 1, std::logic_error,
3554 "rowPtrsUnpacked_host_.extent(0) = "
3555 << rowPtrsUnpacked.extent (0) <<
" != (lclNumRows + 1) = "
3556 << (lclNumRows + 1) <<
".");
3557 const size_t numOffsets = rowPtrsUnpacked.extent (0);
3558 const auto valToCheck = rowPtrsUnpacked(numOffsets-1);
3559 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3561 lclIndsUnpacked_wdv.extent (0) != valToCheck,
3562 std::logic_error,
"numOffsets=" << numOffsets <<
" != 0 "
3563 " and lclIndsUnpacked_wdv.extent(0)=" << lclIndsUnpacked_wdv.extent(0)
3564 <<
" != rowPtrsUnpacked_host_(" << numOffsets <<
")=" << valToCheck
3568 size_t allocSize = 0;
3570 allocSize = this->getLocalAllocationSize ();
3572 catch (std::logic_error& e) {
3573 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3574 (
true, std::logic_error,
"getLocalAllocationSize threw "
3575 "std::logic_error: " << e.what ());
3577 catch (std::runtime_error& e) {
3578 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3579 (
true, std::runtime_error,
"getLocalAllocationSize threw "
3580 "std::runtime_error: " << e.what ());
3582 catch (std::exception& e) {
3583 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3584 (
true, std::runtime_error,
"getLocalAllocationSize threw "
3585 "std::exception: " << e.what ());
3588 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3589 (
true, std::runtime_error,
"getLocalAllocationSize threw "
3590 "an exception not a subclass of std::exception.");
3593 if (this->getLocalNumEntries () != allocSize) {
3596 non_const_row_map_type ptr_d;
3597 row_map_type ptr_d_const;
3606 auto rowPtrsUnpacked = this->getRowPtrsUnpackedHost();
3607 if (rowPtrsUnpacked.extent (0) != 0) {
3608 const size_t numOffsets =
3609 static_cast<size_t> (rowPtrsUnpacked.extent (0));
3610 const auto valToCheck = rowPtrsUnpacked(numOffsets - 1);
3611 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3612 (valToCheck !=
size_t(lclIndsUnpacked_wdv.extent(0)),
3613 std::logic_error,
"(Unpacked branch) Before allocating "
3614 "or packing, k_rowPtrs_(" << (numOffsets-1) <<
")="
3615 << valToCheck <<
" != lclIndsUnpacked_wdv.extent(0)="
3616 << lclIndsUnpacked_wdv.extent (0) <<
".");
3626 size_t lclTotalNumEntries = 0;
3630 non_const_row_map_type (
"Tpetra::CrsGraph::ptr", lclNumRows + 1);
3631 ptr_d_const = ptr_d;
3635 typename num_row_entries_type::const_type numRowEnt_h = k_numRowEntries_;
3637 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3638 (
size_t(numRowEnt_h.extent (0)) != lclNumRows,
3639 std::logic_error,
"(Unpacked branch) "
3640 "numRowEnt_h.extent(0)=" << numRowEnt_h.extent(0)
3641 <<
" != getLocalNumRows()=" << lclNumRows <<
"");
3647 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3648 (static_cast<size_t> (ptr_d.extent (0)) != lclNumRows + 1,
3649 std::logic_error,
"(Unpacked branch) After allocating "
3650 "ptr_d, ptr_d.extent(0) = " << ptr_d.extent(0)
3651 <<
" != lclNumRows+1 = " << (lclNumRows+1) <<
".");
3652 const auto valToCheck =
3653 ::Tpetra::Details::getEntryOnHost (ptr_d, lclNumRows);
3654 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3655 (valToCheck != lclTotalNumEntries, std::logic_error,
3656 "Tpetra::CrsGraph::fillLocalGraph: In unpacked branch, "
3657 "after filling ptr_d, ptr_d(lclNumRows=" << lclNumRows
3658 <<
") = " << valToCheck <<
" != total number of entries "
3659 "on the calling process = " << lclTotalNumEntries
3665 lclinds_1d_type ind_d =
3666 lclinds_1d_type (
"Tpetra::CrsGraph::lclInd", lclTotalNumEntries);
3678 typedef pack_functor<
3679 typename local_graph_device_type::entries_type::non_const_type,
3680 typename local_inds_dualv_type::t_dev::const_type,
3682 typename local_graph_device_type::row_map_type> inds_packer_type;
3683 inds_packer_type f (ind_d,
3684 lclIndsUnpacked_wdv.getDeviceView(Access::ReadOnly),
3685 ptr_d, this->getRowPtrsUnpackedDevice());
3687 typedef typename decltype (ind_d)::execution_space exec_space;
3688 typedef Kokkos::RangePolicy<exec_space, LocalOrdinal> range_type;
3689 Kokkos::parallel_for (range_type (0, lclNumRows), f);
3693 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3694 (ptr_d.extent (0) == 0, std::logic_error,
3695 "(\"Optimize Storage\"=true branch) After packing, "
3696 "ptr_d.extent(0)=0.");
3697 if (ptr_d.extent (0) != 0) {
3698 const size_t numOffsets =
static_cast<size_t> (ptr_d.extent (0));
3699 const auto valToCheck =
3700 ::Tpetra::Details::getEntryOnHost (ptr_d, numOffsets - 1);
3701 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3702 (static_cast<size_t> (valToCheck) != ind_d.extent (0),
3703 std::logic_error,
"(\"Optimize Storage\"=true branch) "
3704 "After packing, ptr_d(" << (numOffsets-1) <<
")="
3705 << valToCheck <<
" != ind_d.extent(0)="
3706 << ind_d.extent(0) <<
".");
3710 if (requestOptimizedStorage)
3711 setRowPtrs(ptr_d_const);
3713 setRowPtrsPacked(ptr_d_const);
3714 lclIndsPacked_wdv = local_inds_wdv_type(ind_d);
3718 this->setRowPtrs(rowPtrsUnpacked_dev_);
3719 lclIndsPacked_wdv = lclIndsUnpacked_wdv;
3722 auto rowPtrsPacked_dev = this->getRowPtrsPackedDevice();
3723 auto rowPtrsPacked_host = this->getRowPtrsPackedHost();
3724 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3725 (rowPtrsPacked_dev.extent (0) == 0, std::logic_error,
3726 "(\"Optimize Storage\"=false branch) "
3727 "rowPtrsPacked_dev_.extent(0) = 0.");
3728 if (rowPtrsPacked_dev.extent (0) != 0) {
3729 const size_t numOffsets =
3730 static_cast<size_t> (rowPtrsPacked_dev.extent (0));
3731 const size_t valToCheck =
3732 rowPtrsPacked_host(numOffsets - 1);
3733 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3734 (valToCheck !=
size_t(lclIndsPacked_wdv.extent (0)),
3735 std::logic_error,
"(\"Optimize Storage\"=false branch) "
3736 "rowPtrsPacked_dev_(" << (numOffsets-1) <<
")="
3738 <<
" != lclIndsPacked_wdv.extent(0)="
3739 << lclIndsPacked_wdv.extent (0) <<
".");
3745 auto rowPtrsPacked_dev = this->getRowPtrsPackedDevice();
3746 auto rowPtrsPacked_host = this->getRowPtrsPackedHost();
3747 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3748 (static_cast<size_t> (rowPtrsPacked_dev.extent (0)) != lclNumRows + 1,
3749 std::logic_error,
"After packing, rowPtrsPacked_dev_.extent(0) = " <<
3750 rowPtrsPacked_dev.extent (0) <<
" != lclNumRows+1 = " << (lclNumRows+1)
3752 if (rowPtrsPacked_dev.extent (0) != 0) {
3753 const size_t numOffsets =
static_cast<size_t> (rowPtrsPacked_dev.extent (0));
3754 const auto valToCheck = rowPtrsPacked_host(numOffsets - 1);
3755 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3756 (static_cast<size_t> (valToCheck) != lclIndsPacked_wdv.extent (0),
3757 std::logic_error,
"After packing, rowPtrsPacked_dev_(" << (numOffsets-1)
3758 <<
") = " << valToCheck <<
" != lclIndsPacked_wdv.extent(0) = "
3759 << lclIndsPacked_wdv.extent (0) <<
".");
3763 if (requestOptimizedStorage) {
3769 k_numRowEntries_ = num_row_entries_type ();
3772 lclIndsUnpacked_wdv = lclIndsPacked_wdv;
3774 storageStatus_ = Details::STORAGE_1D_PACKED;
3777 set_need_sync_host_uvm_access();
3780 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3792 const char tfecfFuncName[] =
"replaceColMap: ";
3793 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3794 isLocallyIndexed () || isGloballyIndexed (), std::runtime_error,
3795 "Requires matching maps and non-static graph.");
3796 colMap_ = newColMap;
3799 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3803 const Teuchos::RCP<const import_type>& newImport,
3804 const bool sortIndicesInEachRow)
3806 using Teuchos::REDUCE_MIN;
3807 using Teuchos::reduceAll;
3809 typedef GlobalOrdinal GO;
3810 typedef LocalOrdinal LO;
3811 typedef typename local_inds_dualv_type::t_host col_inds_type;
3812 const char tfecfFuncName[] =
"reindexColumns: ";
3814 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3815 isFillComplete (), std::runtime_error,
"The graph is fill complete "
3816 "(isFillComplete() returns true). You must call resumeFill() before "
3817 "you may call this method.");
3835 const LO lclNumRows =
static_cast<LO
> (this->getLocalNumRows ());
3850 bool allCurColIndsValid =
true;
3855 bool localSuffices =
true;
3863 col_inds_type newLclInds1D;
3864 auto oldLclInds1D = lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
3869 if (indicesAreAllocated ()) {
3870 if (isLocallyIndexed ()) {
3872 const map_type& oldColMap = * (getColMap ());
3874 const size_t allocSize = this->getLocalAllocationSize ();
3875 newLclInds1D = col_inds_type(
"Tpetra::CrsGraph::lclIndsReindexedHost",
3878 for (LO lclRow = 0; lclRow < lclNumRows; ++lclRow) {
3879 const RowInfo rowInfo = this->getRowInfo (lclRow);
3880 const size_t beg = rowInfo.offset1D;
3881 const size_t end = beg + rowInfo.numEntries;
3882 for (
size_t k = beg; k < end; ++k) {
3883 const LO oldLclCol = oldLclInds1D(k);
3884 if (oldLclCol == Teuchos::OrdinalTraits<LO>::invalid ()) {
3885 allCurColIndsValid =
false;
3893 if (gblCol == Teuchos::OrdinalTraits<GO>::invalid ()) {
3894 allCurColIndsValid =
false;
3898 const LO newLclCol = newColMap->getLocalElement (gblCol);
3899 if (newLclCol == Teuchos::OrdinalTraits<LO>::invalid ()) {
3900 localSuffices =
false;
3903 newLclInds1D(k) = newLclCol;
3914 allCurColIndsValid =
false;
3931 for (LO lclRow = 0; lclRow < lclNumRows; ++lclRow) {
3932 const RowInfo rowInfo = this->getRowInfo (lclRow);
3933 auto oldGblRowView = this->getGlobalIndsViewHost (rowInfo);
3934 for (
size_t k = 0; k < rowInfo.numEntries; ++k) {
3935 const GO gblCol = oldGblRowView(k);
3936 if (! newColMap->isNodeGlobalElement (gblCol)) {
3937 localSuffices =
false;
3947 lclSuccess[0] = allCurColIndsValid ? 1 : 0;
3948 lclSuccess[1] = localSuffices ? 1 : 0;
3952 RCP<const Teuchos::Comm<int> > comm =
3953 getRowMap ().is_null () ? Teuchos::null : getRowMap ()->getComm ();
3954 if (! comm.is_null ()) {
3955 reduceAll<int, int> (*comm, REDUCE_MIN, 2, lclSuccess, gblSuccess);
3958 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3959 gblSuccess[0] == 0, std::runtime_error,
"It is not possible to continue."
3960 " The most likely reason is that the graph is locally indexed, but the "
3961 "column Map is missing (null) on some processes, due to a previous call "
3962 "to replaceColMap().");
3964 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3965 gblSuccess[1] == 0, std::runtime_error,
"On some process, the graph "
3966 "contains column indices that are in the old column Map, but not in the "
3967 "new column Map (on that process). This method does NOT redistribute "
3968 "data; it does not claim to do the work of an Import or Export operation."
3969 " This means that for all processess, the calling process MUST own all "
3970 "column indices, in both the old column Map and the new column Map. In "
3971 "this case, you will need to do an Import or Export operation to "
3972 "redistribute data.");
3975 if (isLocallyIndexed ()) {
3977 typename local_inds_dualv_type::t_dev newLclInds1D_dev(
3978 Kokkos::view_alloc(
"Tpetra::CrsGraph::lclIndReindexed",
3979 Kokkos::WithoutInitializing),
3980 newLclInds1D.extent(0));
3991 indicesAreSorted_ =
false;
3992 if (sortIndicesInEachRow) {
3999 const bool sorted =
false;
4000 const bool merged =
true;
4001 this->sortAndMergeAllIndices (sorted, merged);
4004 colMap_ = newColMap;
4006 if (newImport.is_null ()) {
4014 if (! domainMap_.is_null ()) {
4015 if (! domainMap_->isSameAs (* newColMap)) {
4016 importer_ = Teuchos::rcp (
new import_type (domainMap_, newColMap));
4018 importer_ = Teuchos::null;
4023 importer_ = newImport;
4027 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4032 const char prefix[] =
"Tpetra::CrsGraph::replaceDomainMap: ";
4033 TEUCHOS_TEST_FOR_EXCEPTION(
4034 colMap_.is_null (), std::invalid_argument, prefix <<
"You may not call "
4035 "this method unless the graph already has a column Map.");
4036 TEUCHOS_TEST_FOR_EXCEPTION(
4037 newDomainMap.is_null (), std::invalid_argument,
4038 prefix <<
"The new domain Map must be nonnull.");
4041 Teuchos::RCP<const import_type> newImporter = Teuchos::null;
4042 if (newDomainMap != colMap_ && (! newDomainMap->isSameAs (*colMap_))) {
4043 newImporter = rcp(
new import_type(newDomainMap, colMap_));
4045 this->replaceDomainMapAndImporter(newDomainMap, newImporter);
4048 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4052 const Teuchos::RCP<const import_type>& newImporter)
4054 const char prefix[] =
"Tpetra::CrsGraph::replaceDomainMapAndImporter: ";
4055 TEUCHOS_TEST_FOR_EXCEPTION(
4056 colMap_.is_null (), std::invalid_argument, prefix <<
"You may not call "
4057 "this method unless the graph already has a column Map.");
4058 TEUCHOS_TEST_FOR_EXCEPTION(
4059 newDomainMap.is_null (), std::invalid_argument,
4060 prefix <<
"The new domain Map must be nonnull.");
4063 if (newImporter.is_null ()) {
4068 const bool colSameAsDom = colMap_->isSameAs (*newDomainMap);
4069 TEUCHOS_TEST_FOR_EXCEPTION
4070 (!colSameAsDom, std::invalid_argument,
"If the new Import is null, "
4071 "then the new domain Map must be the same as the current column Map.");
4074 const bool colSameAsTgt =
4075 colMap_->isSameAs (* (newImporter->getTargetMap ()));
4076 const bool newDomSameAsSrc =
4077 newDomainMap->isSameAs (* (newImporter->getSourceMap ()));
4078 TEUCHOS_TEST_FOR_EXCEPTION
4079 (! colSameAsTgt || ! newDomSameAsSrc, std::invalid_argument,
"If the "
4080 "new Import is nonnull, then the current column Map must be the same "
4081 "as the new Import's target Map, and the new domain Map must be the "
4082 "same as the new Import's source Map.");
4086 domainMap_ = newDomainMap;
4087 importer_ = Teuchos::rcp_const_cast<
import_type> (newImporter);
4090 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4095 const char prefix[] =
"Tpetra::CrsGraph::replaceRangeMap: ";
4096 TEUCHOS_TEST_FOR_EXCEPTION(
4097 rowMap_.is_null (), std::invalid_argument, prefix <<
"You may not call "
4098 "this method unless the graph already has a row Map.");
4099 TEUCHOS_TEST_FOR_EXCEPTION(
4100 newRangeMap.is_null (), std::invalid_argument,
4101 prefix <<
"The new range Map must be nonnull.");
4104 Teuchos::RCP<const export_type> newExporter = Teuchos::null;
4105 if (newRangeMap != rowMap_ && (! newRangeMap->isSameAs (*rowMap_))) {
4106 newExporter = rcp(
new export_type(rowMap_, newRangeMap));
4108 this->replaceRangeMapAndExporter(newRangeMap, newExporter);
4111 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4115 const Teuchos::RCP<const export_type>& newExporter)
4117 const char prefix[] =
"Tpetra::CrsGraph::replaceRangeMapAndExporter: ";
4118 TEUCHOS_TEST_FOR_EXCEPTION(
4119 rowMap_.is_null (), std::invalid_argument, prefix <<
"You may not call "
4120 "this method unless the graph already has a column Map.");
4121 TEUCHOS_TEST_FOR_EXCEPTION(
4122 newRangeMap.is_null (), std::invalid_argument,
4123 prefix <<
"The new domain Map must be nonnull.");
4126 if (newExporter.is_null ()) {
4131 const bool rowSameAsRange = rowMap_->isSameAs (*newRangeMap);
4132 TEUCHOS_TEST_FOR_EXCEPTION
4133 (!rowSameAsRange, std::invalid_argument,
"If the new Export is null, "
4134 "then the new range Map must be the same as the current row Map.");
4137 const bool newRangeSameAsTgt =
4138 newRangeMap->isSameAs (* (newExporter->getTargetMap ()));
4139 const bool rowSameAsSrc =
4140 rowMap_->isSameAs (* (newExporter->getSourceMap ()));
4141 TEUCHOS_TEST_FOR_EXCEPTION
4142 (! rowSameAsSrc || ! newRangeSameAsTgt, std::invalid_argument,
"If the "
4143 "new Export is nonnull, then the current row Map must be the same "
4144 "as the new Export's source Map, and the new range Map must be the "
4145 "same as the new Export's target Map.");
4149 rangeMap_ = newRangeMap;
4150 exporter_ = Teuchos::rcp_const_cast<
export_type> (newExporter);
4154 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4160 lclIndsPacked_wdv.getDeviceView(Access::ReadWrite),
4161 this->getRowPtrsPackedDevice());
4164 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4169 return local_graph_host_type(
4170 lclIndsPacked_wdv.getHostView(Access::ReadWrite),
4171 this->getRowPtrsPackedHost());
4174 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4179 using ::Tpetra::Details::ProfilingRegion;
4180 using Teuchos::ArrayView;
4181 using Teuchos::outArg;
4182 using Teuchos::reduceAll;
4185 ProfilingRegion regionCGC (
"Tpetra::CrsGraph::computeGlobalConstants");
4187 this->computeLocalConstants ();
4192 if (! this->haveGlobalConstants_) {
4193 const Teuchos::Comm<int>& comm = * (this->getComm ());
4207 lcl =
static_cast<GST
> (this->getLocalNumEntries ());
4209 reduceAll<int,GST> (comm, Teuchos::REDUCE_SUM, 1, &lcl, &gbl);
4210 this->globalNumEntries_ = gbl;
4212 const GST lclMaxNumRowEnt =
static_cast<GST
> (this->nodeMaxNumRowEntries_);
4213 reduceAll<int, GST> (comm, Teuchos::REDUCE_MAX, lclMaxNumRowEnt,
4214 outArg (this->globalMaxNumRowEntries_));
4215 this->haveGlobalConstants_ =
true;
4220 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4225 using ::Tpetra::Details::ProfilingRegion;
4227 ProfilingRegion regionCLC (
"Tpetra::CrsGraph::computeLocalConstants");
4228 if (this->haveLocalConstants_) {
4233 this->nodeMaxNumRowEntries_ =
4234 Teuchos::OrdinalTraits<size_t>::invalid();
4238 auto ptr = this->getRowPtrsPackedDevice();
4239 const LO lclNumRows = ptr.extent(0) == 0 ?
4240 static_cast<LO
> (0) :
4241 (static_cast<LO> (ptr.extent(0)) - static_cast<LO> (1));
4243 const LO lclMaxNumRowEnt =
4244 ::Tpetra::Details::maxDifference (
"Tpetra::CrsGraph: nodeMaxNumRowEntries",
4246 this->nodeMaxNumRowEntries_ =
static_cast<size_t> (lclMaxNumRowEnt);
4247 this->haveLocalConstants_ =
true;
4251 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4252 std::pair<size_t, std::string>
4257 using Teuchos::arcp;
4258 using Teuchos::Array;
4260 typedef LocalOrdinal LO;
4261 typedef GlobalOrdinal GO;
4263 typedef typename local_graph_device_type::row_map_type::non_const_value_type offset_type;
4264 typedef typename num_row_entries_type::non_const_value_type num_ent_type;
4265 const char tfecfFuncName[] =
"makeIndicesLocal: ";
4266 ProfilingRegion regionMakeIndicesLocal (
"Tpetra::CrsGraph::makeIndicesLocal");
4268 std::unique_ptr<std::string> prefix;
4270 prefix = this->createPrefix(
"CrsGraph",
"makeIndicesLocal");
4271 std::ostringstream os;
4272 os << *prefix <<
"lclNumRows: " << getLocalNumRows() << endl;
4273 std::cerr << os.str();
4278 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4279 (! this->hasColMap (), std::logic_error,
"The graph does not have a "
4280 "column Map yet. This method should never be called in that case. "
4281 "Please report this bug to the Tpetra developers.");
4282 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4283 (this->getColMap ().is_null (), std::logic_error,
"The graph claims "
4284 "that it has a column Map, because hasColMap() returns true. However, "
4285 "the result of getColMap() is null. This should never happen. Please "
4286 "report this bug to the Tpetra developers.");
4291 size_t lclNumErrs = 0;
4292 std::ostringstream errStrm;
4294 const LO lclNumRows =
static_cast<LO
> (this->getLocalNumRows ());
4295 const map_type& colMap = * (this->getColMap ());
4297 if (this->isGloballyIndexed () && lclNumRows != 0) {
4299 typename num_row_entries_type::const_type h_numRowEnt =
4300 this->k_numRowEntries_;
4302 auto rowPtrsUnpacked_host = this->getRowPtrsUnpackedHost();
4305 if (rowPtrsUnpacked_host.extent(0) == 0) {
4306 errStrm <<
"Unpacked row pointers (rowPtrsUnpacked_dev_) has length 0. This should never "
4307 "happen here. Please report this bug to the Tpetra developers."
4310 return std::make_pair(Tpetra::Details::OrdinalTraits<size_t>::invalid (),
4313 const auto numEnt = rowPtrsUnpacked_host(lclNumRows);
4322 using Kokkos::view_alloc;
4323 using Kokkos::WithoutInitializing;
4333 const std::string label (
"Tpetra::CrsGraph::lclInd");
4335 std::ostringstream os;
4336 os << *prefix <<
"(Re)allocate lclInd_wdv: old="
4337 << lclIndsUnpacked_wdv.extent(0) <<
", new=" << numEnt << endl;
4338 std::cerr << os.str();
4341 local_inds_dualv_type lclInds_dualv =
4342 local_inds_dualv_type(view_alloc(label, WithoutInitializing),
4355 std::ostringstream os;
4356 os << *prefix <<
"Allocate device mirror k_numRowEnt: "
4357 << h_numRowEnt.extent(0) << endl;
4358 std::cerr << os.str();
4361 Kokkos::create_mirror_view_and_copy (
device_type (), h_numRowEnt);
4365 convertColumnIndicesFromGlobalToLocal<LO, GO, DT, offset_type, num_ent_type> (
4366 lclIndsUnpacked_wdv.getDeviceView(Access::OverwriteAll),
4367 gblInds_wdv.getDeviceView(Access::ReadOnly),
4368 this->getRowPtrsUnpackedDevice(),
4371 if (lclNumErrs != 0) {
4372 const int myRank = [
this] () {
4373 auto map = this->getMap ();
4374 if (map.is_null ()) {
4378 auto comm = map->getComm ();
4379 return comm.is_null () ? 0 : comm->getRank ();
4382 const bool pluralNumErrs = (lclNumErrs !=
static_cast<size_t> (1));
4383 errStrm <<
"(Process " << myRank <<
") When converting column "
4384 "indices from global to local, we encountered " << lclNumErrs
4385 <<
" ind" << (pluralNumErrs ?
"ices" :
"ex")
4386 <<
" that do" << (pluralNumErrs ?
"es" :
"")
4387 <<
" not live in the column Map on this process." << endl;
4394 std::ostringstream os;
4395 os << *prefix <<
"Free gblInds_wdv: "
4396 << gblInds_wdv.extent(0) << endl;
4397 std::cerr << os.str();
4402 this->indicesAreLocal_ =
true;
4403 this->indicesAreGlobal_ =
false;
4404 this->checkInternalState ();
4406 return std::make_pair (lclNumErrs, errStrm.str ());
4409 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4416 const char tfecfFuncName[] =
"makeColMap";
4418 ProfilingRegion regionSortAndMerge (
"Tpetra::CrsGraph::makeColMap");
4419 std::unique_ptr<std::string> prefix;
4421 prefix = this->createPrefix(
"CrsGraph", tfecfFuncName);
4422 std::ostringstream os;
4423 os << *prefix <<
"Start" << endl;
4424 std::cerr << os.str();
4432 Teuchos::RCP<const map_type> colMap = this->colMap_;
4433 const bool sortEachProcsGids =
4434 this->sortGhostsAssociatedWithEachProcessor_;
4444 using Teuchos::outArg;
4445 using Teuchos::REDUCE_MIN;
4446 using Teuchos::reduceAll;
4448 std::ostringstream errStrm;
4449 const int lclErrCode =
4451 getDomainMap (), *
this, sortEachProcsGids, &errStrm);
4452 auto comm = this->getComm ();
4453 if (! comm.is_null ()) {
4454 const int lclSuccess = (lclErrCode == 0) ? 1 : 0;
4456 reduceAll<int, int> (*comm, REDUCE_MIN, lclSuccess,
4457 outArg (gblSuccess));
4458 if (gblSuccess != 1) {
4459 std::ostringstream os;
4461 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4462 (
true, std::runtime_error,
": An error happened on at "
4463 "least one process in the CrsGraph's communicator. "
4464 "Here are all processes' error messages:" << std::endl
4471 getDomainMap (), *
this, sortEachProcsGids,
nullptr);
4476 this->colMap_ = colMap;
4478 checkInternalState ();
4480 std::ostringstream os;
4481 os << *prefix <<
"Done" << endl;
4482 std::cerr << os.str();
4487 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4493 using LO = LocalOrdinal;
4494 using host_execution_space =
4495 typename Kokkos::View<LO*, device_type>::HostMirror::
4497 using range_type = Kokkos::RangePolicy<host_execution_space, LO>;
4498 const char tfecfFuncName[] =
"sortAndMergeAllIndices";
4500 (
"Tpetra::CrsGraph::sortAndMergeAllIndices");
4502 std::unique_ptr<std::string> prefix;
4504 prefix = this->createPrefix(
"CrsGraph", tfecfFuncName);
4505 std::ostringstream os;
4506 os << *prefix <<
"Start: "
4507 <<
"sorted=" << (sorted ?
"true" :
"false")
4508 <<
", merged=" << (merged ?
"true" :
"false") << endl;
4509 std::cerr << os.str();
4511 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4512 (this->isGloballyIndexed(), std::logic_error,
4513 "This method may only be called after makeIndicesLocal." );
4514 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4515 (! merged && this->isStorageOptimized(), std::logic_error,
4516 "The graph is already storage optimized, so we shouldn't be "
4517 "merging any indices. "
4518 "Please report this bug to the Tpetra developers.");
4520 if (! sorted || ! merged) {
4521 const LO lclNumRows(this->getLocalNumRows());
4522 auto range = range_type(0, lclNumRows);
4525 size_t totalNumDups = 0;
4527 lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
4529 Kokkos::parallel_reduce(range,
4530 [
this, sorted, merged] (
const LO lclRow,
size_t& numDups)
4532 const RowInfo rowInfo = this->getRowInfo(lclRow);
4533 numDups += this->sortAndMergeRowIndices(rowInfo, sorted, merged);
4537 std::ostringstream os;
4538 os << *prefix <<
"totalNumDups=" << totalNumDups << endl;
4539 std::cerr << os.str();
4543 lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
4545 Kokkos::parallel_for(range,
4546 [
this, sorted, merged] (
const LO lclRow)
4548 const RowInfo rowInfo = this->getRowInfo(lclRow);
4549 this->sortAndMergeRowIndices(rowInfo, sorted, merged);
4553 this->indicesAreSorted_ =
true;
4554 this->noRedundancies_ =
true;
4558 std::ostringstream os;
4559 os << *prefix <<
"Done" << endl;
4560 std::cerr << os.str();
4564 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4568 const bool useRemotePIDs)
4570 using ::Tpetra::Details::ProfilingRegion;
4571 using Teuchos::ParameterList;
4574 const char tfecfFuncName[] =
"makeImportExport: ";
4575 ProfilingRegion regionMIE (
"Tpetra::CrsGraph::makeImportExport");
4577 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4578 (! this->hasColMap (), std::logic_error,
4579 "This method may not be called unless the graph has a column Map.");
4580 RCP<ParameterList> params = this->getNonconstParameterList ();
4589 if (importer_.is_null ()) {
4591 if (domainMap_ != colMap_ && (! domainMap_->isSameAs (*colMap_))) {
4592 if (params.is_null () || ! params->isSublist (
"Import")) {
4593 if (useRemotePIDs) {
4594 importer_ = rcp (
new import_type (domainMap_, colMap_, remotePIDs));
4597 importer_ = rcp (
new import_type (domainMap_, colMap_));
4601 RCP<ParameterList> importSublist = sublist (params,
"Import",
true);
4602 if (useRemotePIDs) {
4603 RCP<import_type> newImp =
4604 rcp (
new import_type (domainMap_, colMap_, remotePIDs,
4609 importer_ = rcp (
new import_type (domainMap_, colMap_, importSublist));
4617 if (exporter_.is_null ()) {
4619 if (rangeMap_ != rowMap_ && ! rangeMap_->isSameAs (*rowMap_)) {
4620 if (params.is_null () || ! params->isSublist (
"Export")) {
4621 exporter_ = rcp (
new export_type (rowMap_, rangeMap_));
4624 RCP<ParameterList> exportSublist = sublist (params,
"Export",
true);
4625 exporter_ = rcp (
new export_type (rowMap_, rangeMap_, exportSublist));
4632 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4637 std::ostringstream oss;
4638 oss << dist_object_type::description ();
4639 if (isFillComplete ()) {
4640 oss <<
"{status = fill complete"
4641 <<
", global rows = " << getGlobalNumRows()
4642 <<
", global cols = " << getGlobalNumCols()
4643 <<
", global num entries = " << getGlobalNumEntries()
4647 oss <<
"{status = fill not complete"
4648 <<
", global rows = " << getGlobalNumRows()
4655 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4659 const Teuchos::EVerbosityLevel verbLevel)
const
4661 using Teuchos::ArrayView;
4662 using Teuchos::Comm;
4664 using Teuchos::VERB_DEFAULT;
4665 using Teuchos::VERB_NONE;
4666 using Teuchos::VERB_LOW;
4667 using Teuchos::VERB_MEDIUM;
4668 using Teuchos::VERB_HIGH;
4669 using Teuchos::VERB_EXTREME;
4673 Teuchos::EVerbosityLevel vl = verbLevel;
4674 if (vl == VERB_DEFAULT) vl = VERB_LOW;
4675 RCP<const Comm<int> > comm = this->getComm();
4676 const int myImageID = comm->getRank(),
4677 numImages = comm->getSize();
4679 for (
size_t dec=10; dec<getGlobalNumRows(); dec *= 10) {
4682 width = std::max<size_t> (width,
static_cast<size_t> (11)) + 2;
4683 Teuchos::OSTab tab (out);
4691 if (vl != VERB_NONE) {
4692 if (myImageID == 0) out << this->description() << std::endl;
4694 if (isFillComplete() && myImageID == 0) {
4695 out <<
"Global max number of row entries = " << globalMaxNumRowEntries_ << std::endl;
4698 if (vl == VERB_MEDIUM || vl == VERB_HIGH || vl == VERB_EXTREME) {
4699 if (myImageID == 0) out <<
"\nRow map: " << std::endl;
4700 rowMap_->describe(out,vl);
4701 if (colMap_ != Teuchos::null) {
4702 if (myImageID == 0) out <<
"\nColumn map: " << std::endl;
4703 colMap_->describe(out,vl);
4705 if (domainMap_ != Teuchos::null) {
4706 if (myImageID == 0) out <<
"\nDomain map: " << std::endl;
4707 domainMap_->describe(out,vl);
4709 if (rangeMap_ != Teuchos::null) {
4710 if (myImageID == 0) out <<
"\nRange map: " << std::endl;
4711 rangeMap_->describe(out,vl);
4715 if (vl == VERB_MEDIUM || vl == VERB_HIGH || vl == VERB_EXTREME) {
4716 for (
int imageCtr = 0; imageCtr < numImages; ++imageCtr) {
4717 if (myImageID == imageCtr) {
4718 out <<
"Node ID = " << imageCtr << std::endl
4719 <<
"Node number of entries = " << this->getLocalNumEntries () << std::endl
4720 <<
"Node max number of entries = " << nodeMaxNumRowEntries_ << std::endl;
4721 if (! indicesAreAllocated ()) {
4722 out <<
"Indices are not allocated." << std::endl;
4731 if (vl == VERB_HIGH || vl == VERB_EXTREME) {
4732 for (
int imageCtr = 0; imageCtr < numImages; ++imageCtr) {
4733 if (myImageID == imageCtr) {
4734 out << std::setw(width) <<
"Node ID"
4735 << std::setw(width) <<
"Global Row"
4736 << std::setw(width) <<
"Num Entries";
4737 if (vl == VERB_EXTREME) {
4741 const LocalOrdinal lclNumRows =
4742 static_cast<LocalOrdinal
> (this->getLocalNumRows ());
4743 for (LocalOrdinal r=0; r < lclNumRows; ++r) {
4744 const RowInfo rowinfo = this->getRowInfo (r);
4745 GlobalOrdinal gid = rowMap_->getGlobalElement(r);
4746 out << std::setw(width) << myImageID
4747 << std::setw(width) << gid
4748 << std::setw(width) << rowinfo.numEntries;
4749 if (vl == VERB_EXTREME) {
4751 if (isGloballyIndexed()) {
4752 auto rowview = gblInds_wdv.getHostView(Access::ReadOnly);
4753 for (
size_t j=0; j < rowinfo.numEntries; ++j){
4754 GlobalOrdinal colgid = rowview[j + rowinfo.offset1D];
4755 out << colgid <<
" ";
4758 else if (isLocallyIndexed()) {
4759 auto rowview = lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
4760 for (
size_t j=0; j < rowinfo.numEntries; ++j) {
4761 LocalOrdinal collid = rowview[j + rowinfo.offset1D];
4762 out << colMap_->getGlobalElement(collid) <<
" ";
4778 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4789 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4794 const size_t numSameIDs,
4805 const char tfecfFuncName[] =
"copyAndPermute: ";
4806 const bool verbose = verbose_;
4808 std::unique_ptr<std::string> prefix;
4810 prefix = this->createPrefix(
"CrsGraph",
"copyAndPermute");
4811 std::ostringstream os;
4812 os << *prefix << endl;
4813 std::cerr << os.str ();
4816 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4817 (permuteToLIDs.extent (0) != permuteFromLIDs.extent (0),
4818 std::runtime_error,
"permuteToLIDs.extent(0) = "
4819 << permuteToLIDs.extent (0) <<
" != permuteFromLIDs.extent(0) = "
4820 << permuteFromLIDs.extent (0) <<
".");
4828 std::ostringstream os;
4829 os << *prefix <<
"Compute padding" << endl;
4830 std::cerr << os.str ();
4832 auto padding = computeCrsPadding(srcRowGraph, numSameIDs,
4833 permuteToLIDs, permuteFromLIDs, verbose);
4834 applyCrsPadding(*padding, verbose);
4839 const this_CRS_type* srcCrsGraph =
4840 dynamic_cast<const this_CRS_type*
> (&source);
4842 const map_type& srcRowMap = *(srcRowGraph.getRowMap());
4843 const map_type& tgtRowMap = *(getRowMap());
4844 const bool src_filled = srcRowGraph.isFillComplete();
4845 nonconst_global_inds_host_view_type row_copy;
4851 if (src_filled || srcCrsGraph ==
nullptr) {
4853 std::ostringstream os;
4854 os << *prefix <<
"src_filled || srcCrsGraph == nullptr" << endl;
4855 std::cerr << os.str ();
4862 for (
size_t i = 0; i < numSameIDs; ++i, ++myid) {
4864 size_t row_length = srcRowGraph.getNumEntriesInGlobalRow (gid);
4865 Kokkos::resize(row_copy,row_length);
4866 size_t check_row_length = 0;
4867 srcRowGraph.getGlobalRowCopy (gid, row_copy, check_row_length);
4868 this->insertGlobalIndices (gid, row_length, row_copy.data());
4872 std::ostringstream os;
4873 os << *prefix <<
"! src_filled && srcCrsGraph != nullptr" << endl;
4874 std::cerr << os.str ();
4876 for (
size_t i = 0; i < numSameIDs; ++i, ++myid) {
4878 global_inds_host_view_type row;
4879 srcCrsGraph->getGlobalRowView (gid, row);
4880 this->insertGlobalIndices (gid, row.extent(0), row.data());
4887 auto permuteToLIDs_h = permuteToLIDs.view_host ();
4888 auto permuteFromLIDs_h = permuteFromLIDs.view_host ();
4890 if (src_filled || srcCrsGraph ==
nullptr) {
4891 for (LO i = 0; i < static_cast<LO> (permuteToLIDs_h.extent (0)); ++i) {
4894 size_t row_length = srcRowGraph.getNumEntriesInGlobalRow (srcgid);
4895 Kokkos::resize(row_copy,row_length);
4896 size_t check_row_length = 0;
4897 srcRowGraph.getGlobalRowCopy (srcgid, row_copy, check_row_length);
4898 this->insertGlobalIndices (mygid, row_length, row_copy.data());
4901 for (LO i = 0; i < static_cast<LO> (permuteToLIDs_h.extent (0)); ++i) {
4904 global_inds_host_view_type row;
4905 srcCrsGraph->getGlobalRowView (srcgid, row);
4906 this->insertGlobalIndices (mygid, row.extent(0), row.data());
4911 std::ostringstream os;
4912 os << *prefix <<
"Done" << endl;
4913 std::cerr << os.str ();
4917 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4926 using LO = local_ordinal_type;
4927 using row_ptrs_type =
4928 typename local_graph_device_type::row_map_type::non_const_type;
4929 using range_policy =
4930 Kokkos::RangePolicy<execution_space, Kokkos::IndexType<LO>>;
4931 const char tfecfFuncName[] =
"applyCrsPadding";
4932 ProfilingRegion regionCAP(
"Tpetra::CrsGraph::applyCrsPadding");
4934 std::unique_ptr<std::string> prefix;
4936 prefix = this->createPrefix(
"CrsGraph", tfecfFuncName);
4937 std::ostringstream os;
4938 os << *prefix <<
"padding: ";
4941 std::cerr << os.str();
4943 const int myRank = ! verbose ? -1 : [&] () {
4944 auto map = this->getMap();
4945 if (map.is_null()) {
4948 auto comm = map->getComm();
4949 if (comm.is_null()) {
4952 return comm->getRank();
4961 if (! indicesAreAllocated()) {
4963 std::ostringstream os;
4964 os << *prefix <<
"Call allocateIndices" << endl;
4965 std::cerr << os.str();
4967 allocateIndices(GlobalIndices, verbose);
4969 TEUCHOS_ASSERT( indicesAreAllocated() );
4974 auto rowPtrsUnpacked_dev = this->getRowPtrsUnpackedDevice();
4976 std::ostringstream os;
4977 os << *prefix <<
"Allocate row_ptrs_beg: "
4978 << rowPtrsUnpacked_dev.extent(0) << endl;
4979 std::cerr << os.str();
4981 using Kokkos::view_alloc;
4982 using Kokkos::WithoutInitializing;
4983 row_ptrs_type row_ptrs_beg(
4984 view_alloc(
"row_ptrs_beg", WithoutInitializing),
4985 rowPtrsUnpacked_dev.extent(0));
4989 const size_t N = row_ptrs_beg.extent(0) == 0 ? size_t(0) :
4990 size_t(row_ptrs_beg.extent(0) - 1);
4992 std::ostringstream os;
4993 os << *prefix <<
"Allocate row_ptrs_end: " << N << endl;
4994 std::cerr << os.str();
4996 row_ptrs_type row_ptrs_end(
4997 view_alloc(
"row_ptrs_end", WithoutInitializing), N);
4998 row_ptrs_type num_row_entries;
5000 const bool refill_num_row_entries = k_numRowEntries_.extent(0) != 0;
5002 execution_space().fence();
5004 if (refill_num_row_entries) {
5008 row_ptrs_type(view_alloc(
"num_row_entries", WithoutInitializing), N);
5010 Kokkos::parallel_for
5011 (
"Fill end row pointers", range_policy(0, N),
5012 KOKKOS_LAMBDA (
const size_t i) {
5013 row_ptrs_end(i) = row_ptrs_beg(i) + num_row_entries(i);
5020 Kokkos::parallel_for
5021 (
"Fill end row pointers", range_policy(0, N),
5022 KOKKOS_LAMBDA (
const size_t i) {
5023 row_ptrs_end(i) = row_ptrs_beg(i+1);
5027 if (isGloballyIndexed()) {
5029 padding, myRank, verbose);
5032 padCrsArrays(row_ptrs_beg, row_ptrs_end, lclIndsUnpacked_wdv,
5033 padding, myRank, verbose);
5036 if (refill_num_row_entries) {
5037 Kokkos::parallel_for
5038 (
"Fill num entries", range_policy(0, N),
5039 KOKKOS_LAMBDA (
const size_t i) {
5040 num_row_entries(i) = row_ptrs_end(i) - row_ptrs_beg(i);
5045 std::ostringstream os;
5046 os << *prefix <<
"Reassign k_rowPtrs_; old size: "
5047 << rowPtrsUnpacked_dev.extent(0) <<
", new size: "
5048 << row_ptrs_beg.extent(0) << endl;
5049 std::cerr << os.str();
5050 TEUCHOS_ASSERT( rowPtrsUnpacked_dev.extent(0) == row_ptrs_beg.extent(0) );
5053 setRowPtrsUnpacked(row_ptrs_beg);
5056 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5058 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type
5060 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5062 const RowGraph<LocalOrdinal,GlobalOrdinal,Node>& source,
5063 const size_t numSameIDs,
5064 const Kokkos::DualView<
const local_ordinal_type*,
5065 buffer_device_type>& permuteToLIDs,
5066 const Kokkos::DualView<
const local_ordinal_type*,
5067 buffer_device_type>& permuteFromLIDs,
5068 const bool verbose)
const
5073 std::unique_ptr<std::string> prefix;
5076 "computeCrsPadding(same & permute)");
5077 std::ostringstream os;
5078 os << *prefix <<
"{numSameIDs: " << numSameIDs
5079 <<
", numPermutes: " << permuteFromLIDs.extent(0) <<
"}"
5081 std::cerr << os.str();
5084 const int myRank = [&] () {
5085 auto comm = rowMap_.is_null() ? Teuchos::null :
5087 return comm.is_null() ? -1 : comm->getRank();
5089 std::unique_ptr<padding_type> padding(
5090 new padding_type(myRank, numSameIDs,
5091 permuteFromLIDs.extent(0)));
5093 computeCrsPaddingForSameIDs(*padding, source,
5094 static_cast<LO>(numSameIDs));
5095 computeCrsPaddingForPermutedIDs(*padding, source, permuteToLIDs,
5100 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5102 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5103 computeCrsPaddingForSameIDs(
5104 padding_type& padding,
5105 const RowGraph<local_ordinal_type, global_ordinal_type,
5107 const local_ordinal_type numSameIDs)
const
5110 using GO = global_ordinal_type;
5111 using Details::Impl::getRowGraphGlobalRow;
5113 const char tfecfFuncName[] =
"computeCrsPaddingForSameIds";
5115 std::unique_ptr<std::string> prefix;
5116 const bool verbose = verbose_;
5118 prefix = this->
createPrefix(
"CrsGraph", tfecfFuncName);
5119 std::ostringstream os;
5120 os << *prefix <<
"numSameIDs: " << numSameIDs << endl;
5121 std::cerr << os.str();
5124 if (numSameIDs == 0) {
5128 const map_type& srcRowMap = *(source.getRowMap());
5129 const map_type& tgtRowMap = *rowMap_;
5130 using this_CRS_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
5131 const this_CRS_type* srcCrs =
dynamic_cast<const this_CRS_type*
>(&source);
5132 const bool src_is_unique =
5133 srcCrs ==
nullptr ?
false : srcCrs->isMerged();
5134 const bool tgt_is_unique = this->isMerged();
5136 std::vector<GO> srcGblColIndsScratch;
5137 std::vector<GO> tgtGblColIndsScratch;
5139 execute_sync_host_uvm_access();
5140 for (LO lclRowInd = 0; lclRowInd < numSameIDs; ++lclRowInd) {
5141 const GO srcGblRowInd = srcRowMap.getGlobalElement(lclRowInd);
5142 const GO tgtGblRowInd = tgtRowMap.getGlobalElement(lclRowInd);
5143 auto srcGblColInds = getRowGraphGlobalRow(
5144 srcGblColIndsScratch, source, srcGblRowInd);
5145 auto tgtGblColInds = getRowGraphGlobalRow(
5146 tgtGblColIndsScratch, *
this, tgtGblRowInd);
5147 padding.update_same(lclRowInd, tgtGblColInds.getRawPtr(),
5148 tgtGblColInds.size(), tgt_is_unique,
5149 srcGblColInds.getRawPtr(),
5150 srcGblColInds.size(), src_is_unique);
5153 std::ostringstream os;
5154 os << *prefix <<
"Done" << endl;
5155 std::cerr << os.str();
5159 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5161 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5162 computeCrsPaddingForPermutedIDs(
5163 padding_type& padding,
5164 const RowGraph<local_ordinal_type, global_ordinal_type,
5166 const Kokkos::DualView<
const local_ordinal_type*,
5167 buffer_device_type>& permuteToLIDs,
5168 const Kokkos::DualView<
const local_ordinal_type*,
5169 buffer_device_type>& permuteFromLIDs)
const
5172 using GO = global_ordinal_type;
5173 using Details::Impl::getRowGraphGlobalRow;
5175 const char tfecfFuncName[] =
"computeCrsPaddingForPermutedIds";
5177 std::unique_ptr<std::string> prefix;
5178 const bool verbose = verbose_;
5180 prefix = this->
createPrefix(
"CrsGraph", tfecfFuncName);
5181 std::ostringstream os;
5182 os << *prefix <<
"permuteToLIDs.extent(0): "
5183 << permuteToLIDs.extent(0)
5184 <<
", permuteFromLIDs.extent(0): "
5185 << permuteFromLIDs.extent(0) << endl;
5186 std::cerr << os.str();
5189 if (permuteToLIDs.extent(0) == 0) {
5193 const map_type& srcRowMap = *(source.getRowMap());
5194 const map_type& tgtRowMap = *rowMap_;
5195 using this_CRS_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
5196 const this_CRS_type* srcCrs =
dynamic_cast<const this_CRS_type*
>(&source);
5197 const bool src_is_unique =
5198 srcCrs ==
nullptr ?
false : srcCrs->isMerged();
5199 const bool tgt_is_unique = this->isMerged();
5201 TEUCHOS_ASSERT( ! permuteToLIDs.need_sync_host() );
5202 auto permuteToLIDs_h = permuteToLIDs.view_host();
5203 TEUCHOS_ASSERT( ! permuteFromLIDs.need_sync_host() );
5204 auto permuteFromLIDs_h = permuteFromLIDs.view_host();
5206 std::vector<GO> srcGblColIndsScratch;
5207 std::vector<GO> tgtGblColIndsScratch;
5208 const LO numPermutes =
static_cast<LO
>(permuteToLIDs_h.extent(0));
5210 execute_sync_host_uvm_access();
5211 for (LO whichPermute = 0; whichPermute < numPermutes; ++whichPermute) {
5212 const LO srcLclRowInd = permuteFromLIDs_h[whichPermute];
5213 const GO srcGblRowInd = srcRowMap.getGlobalElement(srcLclRowInd);
5214 auto srcGblColInds = getRowGraphGlobalRow(
5215 srcGblColIndsScratch, source, srcGblRowInd);
5216 const LO tgtLclRowInd = permuteToLIDs_h[whichPermute];
5217 const GO tgtGblRowInd = tgtRowMap.getGlobalElement(tgtLclRowInd);
5218 auto tgtGblColInds = getRowGraphGlobalRow(
5219 tgtGblColIndsScratch, *
this, tgtGblRowInd);
5220 padding.update_permute(whichPermute, tgtLclRowInd,
5221 tgtGblColInds.getRawPtr(),
5222 tgtGblColInds.size(), tgt_is_unique,
5223 srcGblColInds.getRawPtr(),
5224 srcGblColInds.size(), src_is_unique);
5228 std::ostringstream os;
5229 os << *prefix <<
"Done" << endl;
5230 std::cerr << os.str();
5234 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5236 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type
5238 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5239 computeCrsPaddingForImports(
5240 const Kokkos::DualView<
const local_ordinal_type*,
5241 buffer_device_type>& importLIDs,
5242 Kokkos::DualView<packet_type*, buffer_device_type> imports,
5243 Kokkos::DualView<size_t*, buffer_device_type> numPacketsPerLID,
5244 const bool verbose)
const
5246 using Details::Impl::getRowGraphGlobalRow;
5249 using GO = global_ordinal_type;
5250 const char tfecfFuncName[] =
"computeCrsPaddingForImports";
5252 std::unique_ptr<std::string> prefix;
5254 prefix = this->
createPrefix(
"CrsGraph", tfecfFuncName);
5255 std::ostringstream os;
5256 os << *prefix <<
"importLIDs.extent(0): "
5257 << importLIDs.extent(0)
5258 <<
", imports.extent(0): "
5259 << imports.extent(0)
5260 <<
", numPacketsPerLID.extent(0): "
5261 << numPacketsPerLID.extent(0) << endl;
5262 std::cerr << os.str();
5265 const LO numImports =
static_cast<LO
>(importLIDs.extent(0));
5266 const int myRank = [&] () {
5267 auto comm = rowMap_.is_null() ? Teuchos::null :
5269 return comm.is_null() ? -1 : comm->getRank();
5271 std::unique_ptr<padding_type> padding(
5272 new padding_type(myRank, numImports));
5274 if (imports.need_sync_host()) {
5275 imports.sync_host();
5277 auto imports_h = imports.view_host();
5278 if (numPacketsPerLID.need_sync_host ()) {
5279 numPacketsPerLID.sync_host();
5281 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
5283 TEUCHOS_ASSERT( ! importLIDs.need_sync_host() );
5284 auto importLIDs_h = importLIDs.view_host();
5286 const map_type& tgtRowMap = *rowMap_;
5290 constexpr
bool src_is_unique =
false;
5291 const bool tgt_is_unique = isMerged();
5293 std::vector<GO> tgtGblColIndsScratch;
5295 execute_sync_host_uvm_access();
5296 for (LO whichImport = 0; whichImport < numImports; ++whichImport) {
5301 const LO origSrcNumEnt =
5302 static_cast<LO
>(numPacketsPerLID_h[whichImport]);
5303 GO*
const srcGblColInds = imports_h.data() + offset;
5305 const LO tgtLclRowInd = importLIDs_h[whichImport];
5306 const GO tgtGblRowInd =
5307 tgtRowMap.getGlobalElement(tgtLclRowInd);
5308 auto tgtGblColInds = getRowGraphGlobalRow(
5309 tgtGblColIndsScratch, *
this, tgtGblRowInd);
5310 const size_t origTgtNumEnt(tgtGblColInds.size());
5312 padding->update_import(whichImport, tgtLclRowInd,
5313 tgtGblColInds.getRawPtr(),
5314 origTgtNumEnt, tgt_is_unique,
5316 origSrcNumEnt, src_is_unique);
5317 offset += origSrcNumEnt;
5321 std::ostringstream os;
5322 os << *prefix <<
"Done" << endl;
5323 std::cerr << os.str();
5328 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5330 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type
5332 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5333 computePaddingForCrsMatrixUnpack(
5334 const Kokkos::DualView<
const local_ordinal_type*,
5335 buffer_device_type>& importLIDs,
5336 Kokkos::DualView<char*, buffer_device_type> imports,
5337 Kokkos::DualView<size_t*, buffer_device_type> numPacketsPerLID,
5338 const bool verbose)
const
5340 using Details::Impl::getRowGraphGlobalRow;
5341 using Details::PackTraits;
5344 using GO = global_ordinal_type;
5345 const char tfecfFuncName[] =
"computePaddingForCrsMatrixUnpack";
5347 std::unique_ptr<std::string> prefix;
5349 prefix = this->
createPrefix(
"CrsGraph", tfecfFuncName);
5350 std::ostringstream os;
5351 os << *prefix <<
"importLIDs.extent(0): "
5352 << importLIDs.extent(0)
5353 <<
", imports.extent(0): "
5354 << imports.extent(0)
5355 <<
", numPacketsPerLID.extent(0): "
5356 << numPacketsPerLID.extent(0) << endl;
5357 std::cerr << os.str();
5359 const bool extraVerbose =
5362 const LO numImports =
static_cast<LO
>(importLIDs.extent(0));
5363 TEUCHOS_ASSERT( LO(numPacketsPerLID.extent(0)) >= numImports );
5364 const int myRank = [&] () {
5365 auto comm = rowMap_.is_null() ? Teuchos::null :
5367 return comm.is_null() ? -1 : comm->getRank();
5369 std::unique_ptr<padding_type> padding(
5370 new padding_type(myRank, numImports));
5372 if (imports.need_sync_host()) {
5373 imports.sync_host();
5375 auto imports_h = imports.view_host();
5376 if (numPacketsPerLID.need_sync_host ()) {
5377 numPacketsPerLID.sync_host();
5379 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
5381 TEUCHOS_ASSERT( ! importLIDs.need_sync_host() );
5382 auto importLIDs_h = importLIDs.view_host();
5384 const map_type& tgtRowMap = *rowMap_;
5388 constexpr
bool src_is_unique =
false;
5389 const bool tgt_is_unique = isMerged();
5391 std::vector<GO> srcGblColIndsScratch;
5392 std::vector<GO> tgtGblColIndsScratch;
5394 execute_sync_host_uvm_access();
5395 for (LO whichImport = 0; whichImport < numImports; ++whichImport) {
5400 const size_t numBytes = numPacketsPerLID_h[whichImport];
5402 std::ostringstream os;
5403 os << *prefix <<
"whichImport=" << whichImport
5404 <<
", numImports=" << numImports
5405 <<
", numBytes=" << numBytes << endl;
5406 std::cerr << os.str();
5408 if (numBytes == 0) {
5411 LO origSrcNumEnt = 0;
5412 const size_t numEntBeg = offset;
5413 const size_t numEntLen =
5415 TEUCHOS_ASSERT( numBytes >= numEntLen );
5416 TEUCHOS_ASSERT( imports_h.extent(0) >= numEntBeg + numEntLen );
5418 imports_h.data() + numEntBeg);
5420 std::ostringstream os;
5421 os << *prefix <<
"whichImport=" << whichImport
5422 <<
", numImports=" << numImports
5423 <<
", origSrcNumEnt=" << origSrcNumEnt << endl;
5424 std::cerr << os.str();
5426 TEUCHOS_ASSERT( origSrcNumEnt >= LO(0) );
5427 TEUCHOS_ASSERT( numBytes >=
size_t(numEntLen + origSrcNumEnt *
sizeof(GO)) );
5428 const size_t gidsBeg = numEntBeg + numEntLen;
5429 if (srcGblColIndsScratch.size() < size_t(origSrcNumEnt)) {
5430 srcGblColIndsScratch.resize(origSrcNumEnt);
5432 GO*
const srcGblColInds = srcGblColIndsScratch.data();
5433 PackTraits<GO>::unpackArray(srcGblColInds,
5434 imports_h.data() + gidsBeg,
5436 const LO tgtLclRowInd = importLIDs_h[whichImport];
5437 const GO tgtGblRowInd =
5438 tgtRowMap.getGlobalElement(tgtLclRowInd);
5439 auto tgtGblColInds = getRowGraphGlobalRow(
5440 tgtGblColIndsScratch, *
this, tgtGblRowInd);
5441 const size_t origNumTgtEnt(tgtGblColInds.size());
5444 std::ostringstream os;
5445 os << *prefix <<
"whichImport=" << whichImport
5446 <<
", numImports=" << numImports
5447 <<
": Call padding->update_import" << endl;
5448 std::cerr << os.str();
5450 padding->update_import(whichImport, tgtLclRowInd,
5451 tgtGblColInds.getRawPtr(),
5452 origNumTgtEnt, tgt_is_unique,
5454 origSrcNumEnt, src_is_unique);
5459 std::ostringstream os;
5460 os << *prefix <<
"Done" << endl;
5461 std::cerr << os.str();
5466 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5468 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5470 (
const SrcDistObject& source,
5471 const Kokkos::DualView<
const local_ordinal_type*,
5472 buffer_device_type>& exportLIDs,
5473 Kokkos::DualView<packet_type*,
5474 buffer_device_type>& exports,
5475 Kokkos::DualView<
size_t*,
5476 buffer_device_type> numPacketsPerLID,
5477 size_t& constantNumPackets)
5480 using GO = global_ordinal_type;
5482 using crs_graph_type =
5483 CrsGraph<local_ordinal_type, global_ordinal_type, node_type>;
5484 const char tfecfFuncName[] =
"packAndPrepare: ";
5485 ProfilingRegion region_papn (
"Tpetra::CrsGraph::packAndPrepare");
5487 const bool verbose = verbose_;
5488 std::unique_ptr<std::string> prefix;
5490 prefix = this->
createPrefix(
"CrsGraph",
"packAndPrepare");
5491 std::ostringstream os;
5492 os << *prefix <<
"Start" << endl;
5493 std::cerr << os.str();
5496 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5497 (exportLIDs.extent (0) != numPacketsPerLID.extent (0),
5499 "exportLIDs.extent(0) = " << exportLIDs.extent (0)
5500 <<
" != numPacketsPerLID.extent(0) = " << numPacketsPerLID.extent (0)
5502 const row_graph_type* srcRowGraphPtr =
5503 dynamic_cast<const row_graph_type*
> (&source);
5504 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5505 (srcRowGraphPtr ==
nullptr, std::invalid_argument,
"Source of an Export "
5506 "or Import operation to a CrsGraph must be a RowGraph with the same "
5507 "template parameters.");
5511 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5512 (this->isFillComplete (), std::runtime_error,
5513 "The target graph of an Import or Export must not be fill complete.");
5515 const crs_graph_type* srcCrsGraphPtr =
5516 dynamic_cast<const crs_graph_type*
> (&source);
5518 if (srcCrsGraphPtr ==
nullptr) {
5519 using Teuchos::ArrayView;
5523 std::ostringstream os;
5524 os << *prefix <<
"Source is a RowGraph but not a CrsGraph"
5526 std::cerr << os.str();
5533 TEUCHOS_ASSERT( ! exportLIDs.need_sync_host () );
5534 auto exportLIDs_h = exportLIDs.view_host ();
5535 ArrayView<const LO> exportLIDs_av (exportLIDs_h.data (),
5536 exportLIDs_h.extent (0));
5537 Teuchos::Array<GO> exports_a;
5539 numPacketsPerLID.clear_sync_state ();
5540 numPacketsPerLID.modify_host ();
5541 auto numPacketsPerLID_h = numPacketsPerLID.view_host ();
5542 ArrayView<size_t> numPacketsPerLID_av (numPacketsPerLID_h.data (),
5543 numPacketsPerLID_h.extent (0));
5544 srcRowGraphPtr->pack (exportLIDs_av, exports_a, numPacketsPerLID_av,
5545 constantNumPackets);
5546 const size_t newSize =
static_cast<size_t> (exports_a.size ());
5547 if (static_cast<size_t> (exports.extent (0)) != newSize) {
5548 using exports_dv_type = Kokkos::DualView<packet_type*, buffer_device_type>;
5549 exports = exports_dv_type (
"exports", newSize);
5551 Kokkos::View<
const packet_type*, Kokkos::HostSpace,
5552 Kokkos::MemoryUnmanaged> exports_a_h (exports_a.getRawPtr (), newSize);
5553 exports.clear_sync_state ();
5554 exports.modify_host ();
5559 else if (! getColMap ().is_null () &&
5560 (this->getRowPtrsPackedDevice().extent (0) != 0 ||
5561 getRowMap ()->getLocalNumElements () == 0)) {
5563 std::ostringstream os;
5564 os << *prefix <<
"packCrsGraphNew path" << endl;
5565 std::cerr << os.str();
5567 using export_pids_type =
5568 Kokkos::DualView<const int*, buffer_device_type>;
5569 export_pids_type exportPIDs;
5571 using NT = node_type;
5573 packCrsGraphNew<LO,GO,NT> (*srcCrsGraphPtr, exportLIDs, exportPIDs,
5574 exports, numPacketsPerLID,
5575 constantNumPackets,
false);
5578 srcCrsGraphPtr->packFillActiveNew (exportLIDs, exports, numPacketsPerLID,
5579 constantNumPackets);
5583 std::ostringstream os;
5584 os << *prefix <<
"Done" << endl;
5585 std::cerr << os.str();
5589 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5592 pack (
const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
5593 Teuchos::Array<GlobalOrdinal>& exports,
5594 const Teuchos::ArrayView<size_t>& numPacketsPerLID,
5595 size_t& constantNumPackets)
const
5597 auto col_map = this->getColMap();
5599 if( !col_map.is_null() && (this->getRowPtrsPackedDevice().extent(0) != 0 || getRowMap()->getLocalNumElements() ==0)) {
5601 packCrsGraph<LocalOrdinal,GlobalOrdinal,Node>(*
this, exports, numPacketsPerLID,
5602 exportLIDs, constantNumPackets);
5605 this->packFillActive(exportLIDs, exports, numPacketsPerLID,
5606 constantNumPackets);
5610 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5613 packFillActive (
const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
5614 Teuchos::Array<GlobalOrdinal>& exports,
5615 const Teuchos::ArrayView<size_t>& numPacketsPerLID,
5616 size_t& constantNumPackets)
const
5619 using LO = LocalOrdinal;
5620 using GO = GlobalOrdinal;
5621 using host_execution_space =
5622 typename Kokkos::View<size_t*, device_type>::
5623 HostMirror::execution_space;
5624 const char tfecfFuncName[] =
"packFillActive: ";
5625 const bool verbose = verbose_;
5627 const auto numExportLIDs = exportLIDs.size ();
5628 std::unique_ptr<std::string> prefix;
5630 prefix = this->createPrefix(
"CrsGraph",
"allocateIndices");
5631 std::ostringstream os;
5632 os << *prefix <<
"numExportLIDs=" << numExportLIDs << endl;
5633 std::cerr << os.str();
5635 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5636 (numExportLIDs != numPacketsPerLID.size (), std::runtime_error,
5637 "exportLIDs.size() = " << numExportLIDs <<
" != numPacketsPerLID.size()"
5638 " = " << numPacketsPerLID.size () <<
".");
5640 const map_type& rowMap = * (this->getRowMap ());
5641 const map_type*
const colMapPtr = this->colMap_.getRawPtr ();
5642 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5643 (this->isLocallyIndexed () && colMapPtr ==
nullptr, std::logic_error,
5644 "This graph claims to be locally indexed, but its column Map is nullptr. "
5645 "This should never happen. Please report this bug to the Tpetra "
5649 constantNumPackets = 0;
5653 size_t*
const numPacketsPerLID_raw = numPacketsPerLID.getRawPtr ();
5654 const LO*
const exportLIDs_raw = exportLIDs.getRawPtr ();
5661 Kokkos::RangePolicy<host_execution_space, LO> inputRange (0, numExportLIDs);
5662 size_t totalNumPackets = 0;
5663 size_t errCount = 0;
5666 typedef Kokkos::Device<host_execution_space, Kokkos::HostSpace>
5668 Kokkos::View<size_t, host_device_type> errCountView (&errCount);
5669 constexpr
size_t ONE = 1;
5671 execute_sync_host_uvm_access();
5672 Kokkos::parallel_reduce (
"Tpetra::CrsGraph::pack: totalNumPackets",
5674 [=] (
const LO& i,
size_t& curTotalNumPackets) {
5675 const GO gblRow = rowMap.getGlobalElement (exportLIDs_raw[i]);
5676 if (gblRow == Tpetra::Details::OrdinalTraits<GO>::invalid ()) {
5677 Kokkos::atomic_add (&errCountView(), ONE);
5678 numPacketsPerLID_raw[i] = 0;
5681 const size_t numEnt = this->getNumEntriesInGlobalRow (gblRow);
5682 numPacketsPerLID_raw[i] = numEnt;
5683 curTotalNumPackets += numEnt;
5689 std::ostringstream os;
5690 os << *prefix <<
"totalNumPackets=" << totalNumPackets << endl;
5691 std::cerr << os.str();
5693 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5694 (errCount != 0, std::logic_error,
"totalNumPackets count encountered "
5695 "one or more errors! errCount = " << errCount
5696 <<
", totalNumPackets = " << totalNumPackets <<
".");
5700 exports.resize (totalNumPackets);
5702 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5703 (! this->supportsRowViews (), std::logic_error,
5704 "this->supportsRowViews() returns false; this should never happen. "
5705 "Please report this bug to the Tpetra developers.");
5711 std::ostringstream os;
5712 os << *prefix <<
"Pack into exports" << endl;
5713 std::cerr << os.str();
5718 GO*
const exports_raw = exports.getRawPtr ();
5720 Kokkos::parallel_scan (
"Tpetra::CrsGraph::pack: pack from views",
5721 inputRange, [=, &prefix]
5722 (
const LO i,
size_t& exportsOffset,
const bool final) {
5723 const size_t curOffset = exportsOffset;
5724 const GO gblRow = rowMap.getGlobalElement (exportLIDs_raw[i]);
5725 const RowInfo rowInfo =
5726 this->getRowInfoFromGlobalRowIndex (gblRow);
5728 using TDO = Tpetra::Details::OrdinalTraits<size_t>;
5729 if (rowInfo.localRow == TDO::invalid ()) {
5731 std::ostringstream os;
5732 os << *prefix <<
": INVALID rowInfo: i=" << i
5733 <<
", lclRow=" << exportLIDs_raw[i] << endl;
5734 std::cerr << os.str();
5736 Kokkos::atomic_add (&errCountView(), ONE);
5738 else if (curOffset + rowInfo.numEntries > totalNumPackets) {
5740 std::ostringstream os;
5741 os << *prefix <<
": UH OH! For i=" << i <<
", lclRow="
5742 << exportLIDs_raw[i] <<
", gblRow=" << gblRow <<
", curOffset "
5743 "(= " << curOffset <<
") + numEnt (= " << rowInfo.numEntries
5744 <<
") > totalNumPackets (= " << totalNumPackets <<
")."
5746 std::cerr << os.str();
5748 Kokkos::atomic_add (&errCountView(), ONE);
5751 const LO numEnt =
static_cast<LO
> (rowInfo.numEntries);
5752 if (this->isLocallyIndexed ()) {
5753 auto lclColInds = getLocalIndsViewHost (rowInfo);
5755 for (LO k = 0; k < numEnt; ++k) {
5756 const LO lclColInd = lclColInds(k);
5757 const GO gblColInd = colMapPtr->getGlobalElement (lclColInd);
5761 exports_raw[curOffset + k] = gblColInd;
5764 exportsOffset = curOffset + numEnt;
5766 else if (this->isGloballyIndexed ()) {
5767 auto gblColInds = getGlobalIndsViewHost (rowInfo);
5769 for (LO k = 0; k < numEnt; ++k) {
5770 const GO gblColInd = gblColInds(k);
5774 exports_raw[curOffset + k] = gblColInd;
5777 exportsOffset = curOffset + numEnt;
5785 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5786 (errCount != 0, std::logic_error,
"Packing encountered "
5787 "one or more errors! errCount = " << errCount
5788 <<
", totalNumPackets = " << totalNumPackets <<
".");
5791 std::ostringstream os;
5792 os << *prefix <<
"Done" << endl;
5793 std::cerr << os.str();
5797 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5799 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5800 packFillActiveNew (
const Kokkos::DualView<
const local_ordinal_type*,
5801 buffer_device_type>& exportLIDs,
5802 Kokkos::DualView<packet_type*,
5803 buffer_device_type>& exports,
5804 Kokkos::DualView<
size_t*,
5805 buffer_device_type> numPacketsPerLID,
5806 size_t& constantNumPackets)
const
5810 using GO = global_ordinal_type;
5811 using host_execution_space =
typename Kokkos::View<
size_t*,
5812 device_type>::HostMirror::execution_space;
5813 using host_device_type =
5814 Kokkos::Device<host_execution_space, Kokkos::HostSpace>;
5815 using exports_dv_type =
5816 Kokkos::DualView<packet_type*, buffer_device_type>;
5817 const char tfecfFuncName[] =
"packFillActiveNew: ";
5818 const bool verbose = verbose_;
5820 const auto numExportLIDs = exportLIDs.extent (0);
5821 std::unique_ptr<std::string> prefix;
5823 prefix = this->
createPrefix(
"CrsGraph",
"packFillActiveNew");
5824 std::ostringstream os;
5825 os << *prefix <<
"numExportLIDs: " << numExportLIDs
5826 <<
", numPacketsPerLID.extent(0): "
5827 << numPacketsPerLID.extent(0) << endl;
5828 std::cerr << os.str();
5830 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5831 (numExportLIDs != numPacketsPerLID.extent (0), std::runtime_error,
5832 "exportLIDs.extent(0) = " << numExportLIDs
5833 <<
" != numPacketsPerLID.extent(0) = "
5834 << numPacketsPerLID.extent (0) <<
".");
5835 TEUCHOS_ASSERT( ! exportLIDs.need_sync_host () );
5836 auto exportLIDs_h = exportLIDs.view_host ();
5838 const map_type& rowMap = * (this->getRowMap ());
5839 const map_type*
const colMapPtr = this->colMap_.getRawPtr ();
5840 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5841 (this->isLocallyIndexed () && colMapPtr ==
nullptr, std::logic_error,
5842 "This graph claims to be locally indexed, but its column Map is nullptr. "
5843 "This should never happen. Please report this bug to the Tpetra "
5847 constantNumPackets = 0;
5849 numPacketsPerLID.clear_sync_state ();
5850 numPacketsPerLID.modify_host ();
5851 auto numPacketsPerLID_h = numPacketsPerLID.view_host ();
5858 using range_type = Kokkos::RangePolicy<host_execution_space, LO>;
5859 range_type inputRange (0, numExportLIDs);
5860 size_t totalNumPackets = 0;
5861 size_t errCount = 0;
5864 Kokkos::View<size_t, host_device_type> errCountView (&errCount);
5865 constexpr
size_t ONE = 1;
5868 std::ostringstream os;
5869 os << *prefix <<
"Compute totalNumPackets" << endl;
5870 std::cerr << os.str ();
5873 execute_sync_host_uvm_access();
5874 totalNumPackets = 0;
5875 for (
size_t i=0; i<numExportLIDs; ++i) {
5876 const LO lclRow = exportLIDs_h[i];
5877 const GO gblRow = rowMap.getGlobalElement (lclRow);
5878 if (gblRow == Tpetra::Details::OrdinalTraits<GO>::invalid ()) {
5880 std::ostringstream os;
5881 os << *prefix <<
"For i=" << i <<
", lclRow=" << lclRow
5882 <<
" not in row Map on this process" << endl;
5883 std::cerr << os.str();
5885 Kokkos::atomic_add (&errCountView(), ONE);
5886 numPacketsPerLID_h(i) = 0;
5889 const size_t numEnt = this->getNumEntriesInGlobalRow (gblRow);
5890 numPacketsPerLID_h(i) = numEnt;
5891 totalNumPackets += numEnt;
5896 std::ostringstream os;
5897 os << *prefix <<
"totalNumPackets: " << totalNumPackets
5898 <<
", errCount: " << errCount << endl;
5899 std::cerr << os.str ();
5901 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5902 (errCount != 0, std::logic_error,
"totalNumPackets count encountered "
5903 "one or more errors! totalNumPackets: " << totalNumPackets
5904 <<
", errCount: " << errCount <<
".");
5907 if (
size_t(exports.extent (0)) < totalNumPackets) {
5909 exports = exports_dv_type (
"exports", totalNumPackets);
5912 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5913 (! this->supportsRowViews (), std::logic_error,
5914 "this->supportsRowViews() returns false; this should never happen. "
5915 "Please report this bug to the Tpetra developers.");
5921 std::ostringstream os;
5922 os << *prefix <<
"Pack into exports buffer" << endl;
5923 std::cerr << os.str();
5926 exports.clear_sync_state ();
5927 exports.modify_host ();
5928 auto exports_h = exports.view_host ();
5934 if(isLocallyIndexed())
5935 lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
5936 else if(isGloballyIndexed())
5937 gblInds_wdv.getHostView(Access::ReadOnly);
5940 Kokkos::parallel_scan
5941 (
"Tpetra::CrsGraph::packFillActiveNew: Pack exports",
5942 inputRange, [=, &prefix]
5943 (
const LO i,
size_t& exportsOffset,
const bool final) {
5944 const size_t curOffset = exportsOffset;
5945 const LO lclRow = exportLIDs_h(i);
5946 const GO gblRow = rowMap.getGlobalElement (lclRow);
5947 if (gblRow == Details::OrdinalTraits<GO>::invalid ()) {
5949 std::ostringstream os;
5950 os << *prefix <<
"For i=" << i <<
", lclRow=" << lclRow
5951 <<
" not in row Map on this process" << endl;
5952 std::cerr << os.str();
5954 Kokkos::atomic_add (&errCountView(), ONE);
5958 const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex (gblRow);
5959 if (rowInfo.localRow == Details::OrdinalTraits<size_t>::invalid ()) {
5961 std::ostringstream os;
5962 os << *prefix <<
"For i=" << i <<
", lclRow=" << lclRow
5963 <<
", gblRow=" << gblRow <<
": invalid rowInfo"
5965 std::cerr << os.str();
5967 Kokkos::atomic_add (&errCountView(), ONE);
5971 if (curOffset + rowInfo.numEntries > totalNumPackets) {
5973 std::ostringstream os;
5974 os << *prefix <<
"For i=" << i <<
", lclRow=" << lclRow
5975 <<
", gblRow=" << gblRow <<
", curOffset (= "
5976 << curOffset <<
") + numEnt (= " << rowInfo.numEntries
5977 <<
") > totalNumPackets (= " << totalNumPackets
5979 std::cerr << os.str();
5981 Kokkos::atomic_add (&errCountView(), ONE);
5985 const LO numEnt =
static_cast<LO
> (rowInfo.numEntries);
5986 if (this->isLocallyIndexed ()) {
5987 auto lclColInds = getLocalIndsViewHost(rowInfo);
5989 for (LO k = 0; k < numEnt; ++k) {
5990 const LO lclColInd = lclColInds(k);
5991 const GO gblColInd = colMapPtr->getGlobalElement (lclColInd);
5995 exports_h(curOffset + k) = gblColInd;
5998 exportsOffset = curOffset + numEnt;
6000 else if (this->isGloballyIndexed ()) {
6001 auto gblColInds = getGlobalIndsViewHost(rowInfo);
6003 for (LO k = 0; k < numEnt; ++k) {
6004 const GO gblColInd = gblColInds(k);
6008 exports_h(curOffset + k) = gblColInd;
6011 exportsOffset = curOffset + numEnt;
6025 std::ostringstream os;
6026 os << *prefix <<
"errCount=" << errCount <<
"; Done" << endl;
6027 std::cerr << os.str();
6031 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
6037 Kokkos::DualView<packet_type*,
6039 Kokkos::DualView<
size_t*,
6048 const char tfecfFuncName[] =
"unpackAndCombine";
6050 ProfilingRegion regionCGC(
"Tpetra::CrsGraph::unpackAndCombine");
6051 const bool verbose = verbose_;
6053 std::unique_ptr<std::string> prefix;
6055 prefix = this->createPrefix(
"CrsGraph", tfecfFuncName);
6056 std::ostringstream os;
6057 os << *prefix <<
"Start" << endl;
6058 std::cerr << os.str ();
6061 auto padding = computeCrsPaddingForImports(
6062 importLIDs, imports, numPacketsPerLID, verbose);
6063 applyCrsPadding(*padding, verbose);
6065 std::ostringstream os;
6066 os << *prefix <<
"Done computing & applying padding" << endl;
6067 std::cerr << os.str ();
6088 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6089 (importLIDs.extent (0) != numPacketsPerLID.extent (0),
6090 std::runtime_error,
": importLIDs.extent(0) = "
6091 << importLIDs.extent (0) <<
" != numPacketsPerLID.extent(0) = "
6092 << numPacketsPerLID.extent (0) <<
".");
6093 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6094 (isFillComplete (), std::runtime_error,
6095 ": Import or Export operations are not allowed on a target "
6096 "CrsGraph that is fillComplete.");
6098 const size_t numImportLIDs(importLIDs.extent(0));
6099 if (numPacketsPerLID.need_sync_host()) {
6100 numPacketsPerLID.sync_host();
6102 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
6103 if (imports.need_sync_host()) {
6104 imports.sync_host();
6106 auto imports_h = imports.view_host();
6107 TEUCHOS_ASSERT( ! importLIDs.need_sync_host() );
6108 auto importLIDs_h = importLIDs.view_host();
6111 Teuchos::Array<LO> lclColInds;
6112 if (isLocallyIndexed()) {
6114 std::ostringstream os;
6115 os << *prefix <<
"Preallocate local indices scratch" << endl;
6116 std::cerr << os.str();
6118 size_t maxNumInserts = 0;
6119 for (
size_t i = 0; i < numImportLIDs; ++i) {
6120 maxNumInserts = std::max (maxNumInserts, numPacketsPerLID_h[i]);
6123 std::ostringstream os;
6124 os << *prefix <<
"Local indices scratch size: "
6125 << maxNumInserts << endl;
6126 std::cerr << os.str();
6128 lclColInds.resize (maxNumInserts);
6132 std::ostringstream os;
6134 if (isGloballyIndexed()) {
6135 os <<
"Graph is globally indexed";
6138 os <<
"Graph is neither locally nor globally indexed";
6141 std::cerr << os.str();
6145 TEUCHOS_ASSERT( ! rowMap_.is_null() );
6149 size_t importsOffset = 0;
6150 for (
size_t i = 0; i < numImportLIDs; ++i) {
6152 std::ostringstream os;
6153 os << *prefix <<
"i=" << i <<
", numImportLIDs="
6154 << numImportLIDs << endl;
6155 std::cerr << os.str();
6159 const LO lclRow = importLIDs_h[i];
6161 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6162 (gblRow == Teuchos::OrdinalTraits<GO>::invalid(),
6163 std::logic_error,
"importLIDs[i=" << i <<
"]="
6164 << lclRow <<
" is not in the row Map on the calling "
6166 const LO numEnt = numPacketsPerLID_h[i];
6167 const GO*
const gblColInds = (numEnt == 0) ?
nullptr :
6168 imports_h.data() + importsOffset;
6169 if (! isLocallyIndexed()) {
6170 insertGlobalIndicesFiltered(lclRow, gblColInds, numEnt);
6175 for (LO j = 0; j < numEnt; j++) {
6176 lclColInds[j] = colMap_->getLocalElement(gblColInds[j]);
6178 insertLocalIndices(lclRow, numEnt, lclColInds.data());
6180 importsOffset += numEnt;
6183 catch (std::exception& e) {
6184 TEUCHOS_TEST_FOR_EXCEPTION
6185 (
true, std::runtime_error,
6186 "Tpetra::CrsGraph::unpackAndCombine: Insert loop threw an "
6187 "exception: " << endl << e.what());
6191 std::ostringstream os;
6192 os << *prefix <<
"Done" << endl;
6193 std::cerr << os.str();
6197 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
6202 using Teuchos::Comm;
6203 using Teuchos::null;
6204 using Teuchos::ParameterList;
6210 RCP<const map_type> rowMap, domainMap, rangeMap, colMap;
6211 RCP<import_type> importer;
6212 RCP<export_type> exporter;
6215 RCP<const Comm<int> > newComm =
6216 (newMap.is_null ()) ? null : newMap->getComm ();
6218 if (! domainMap_.is_null ()) {
6219 if (domainMap_.getRawPtr () == rowMap_.getRawPtr ()) {
6226 domainMap = domainMap_->replaceCommWithSubset (newComm);
6229 if (! rangeMap_.is_null ()) {
6230 if (rangeMap_.getRawPtr () == rowMap_.getRawPtr ()) {
6237 rangeMap = rangeMap_->replaceCommWithSubset (newComm);
6240 if (! colMap_.is_null ()) {
6241 colMap = colMap_->replaceCommWithSubset (newComm);
6245 if (! newComm.is_null ()) {
6246 RCP<ParameterList> params = this->getNonconstParameterList ();
6254 if (! rangeMap_.is_null () &&
6255 rangeMap != rowMap &&
6256 ! rangeMap->isSameAs (*rowMap)) {
6257 if (params.is_null () || ! params->isSublist (
"Export")) {
6258 exporter = rcp (
new export_type (rowMap, rangeMap));
6261 RCP<ParameterList> exportSublist = sublist (params,
"Export",
true);
6262 exporter = rcp (
new export_type (rowMap, rangeMap, exportSublist));
6266 if (! domainMap_.is_null () &&
6267 domainMap != colMap &&
6268 ! domainMap->isSameAs (*colMap)) {
6269 if (params.is_null () || ! params->isSublist (
"Import")) {
6270 importer = rcp (
new import_type (domainMap, colMap));
6272 RCP<ParameterList> importSublist = sublist (params,
"Import",
true);
6273 importer = rcp (
new import_type (domainMap, colMap, importSublist));
6281 exporter_ = exporter;
6282 importer_ = importer;
6289 this->map_ = rowMap;
6290 domainMap_ = domainMap;
6291 rangeMap_ = rangeMap;
6295 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
6301 using LO = LocalOrdinal;
6302 using GO = GlobalOrdinal;
6303 const char tfecfFuncName[] =
"getLocalDiagOffsets: ";
6304 const bool verbose = verbose_;
6306 std::unique_ptr<std::string> prefix;
6308 prefix = this->createPrefix(
"CrsGraph",
"getLocalDiagOffsets");
6309 std::ostringstream os;
6310 os << *prefix <<
"offsets.extent(0)=" << offsets.extent(0)
6312 std::cerr << os.str();
6315 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6316 (! hasColMap (), std::runtime_error,
"The graph must have a column Map.");
6317 const LO lclNumRows =
static_cast<LO
> (this->getLocalNumRows ());
6318 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6319 (static_cast<LO> (offsets.extent (0)) < lclNumRows,
6320 std::invalid_argument,
"offsets.extent(0) = " <<
6321 offsets.extent (0) <<
" < getLocalNumRows() = " << lclNumRows <<
".");
6323 const map_type& rowMap = * (this->getRowMap ());
6324 const map_type& colMap = * (this->getColMap ());
6330 bool allRowMapDiagEntriesInColMap =
true;
6331 bool allDiagEntriesFound =
true;
6332 bool allOffsetsCorrect =
true;
6333 bool noOtherWeirdness =
true;
6334 using wrong_offsets_type = std::vector<std::pair<LO, size_t> >;
6335 wrong_offsets_type wrongOffsets(0);
6339 auto lclRowMap = rowMap.getLocalMap ();
6346 const bool sorted = this->isSorted ();
6347 if (isFillComplete ()) {
6348 auto lclGraph = this->getLocalGraphDevice ();
6349 ::Tpetra::Details::getGraphDiagOffsets (offsets, lclRowMap, lclColMap,
6351 lclGraph.entries, sorted);
6357 auto offsets_h = Kokkos::create_mirror_view (offsets);
6359 for (LO lclRowInd = 0; lclRowInd < lclNumRows; ++lclRowInd) {
6363 const GO gblRowInd = lclRowMap.getGlobalElement (lclRowInd);
6364 const GO gblColInd = gblRowInd;
6365 const LO lclColInd = lclColMap.getLocalElement (gblColInd);
6367 if (lclColInd == Tpetra::Details::OrdinalTraits<LO>::invalid ()) {
6368 allRowMapDiagEntriesInColMap =
false;
6369 offsets_h(lclRowInd) = Tpetra::Details::OrdinalTraits<size_t>::invalid ();
6372 const RowInfo rowInfo = this->getRowInfo (lclRowInd);
6373 if (static_cast<LO> (rowInfo.localRow) == lclRowInd &&
6374 rowInfo.numEntries > 0) {
6376 auto colInds = this->getLocalIndsViewHost (rowInfo);
6377 const size_t hint = 0;
6378 const size_t offset =
6379 KokkosSparse::findRelOffset (colInds, rowInfo.numEntries,
6380 lclColInd, hint, sorted);
6381 offsets_h(lclRowInd) = offset;
6388 typename local_inds_dualv_type::t_host::const_type lclColInds;
6390 lclColInds = this->getLocalIndsViewHost (rowInfo);
6393 noOtherWeirdness =
false;
6396 if (noOtherWeirdness) {
6397 const size_t numEnt = lclColInds.extent (0);
6398 if (offset >= numEnt) {
6401 allOffsetsCorrect =
false;
6402 wrongOffsets.push_back (std::make_pair (lclRowInd, offset));
6404 const LO actualLclColInd = lclColInds(offset);
6405 const GO actualGblColInd = lclColMap.getGlobalElement (actualLclColInd);
6406 if (actualGblColInd != gblColInd) {
6407 allOffsetsCorrect =
false;
6408 wrongOffsets.push_back (std::make_pair (lclRowInd, offset));
6415 offsets_h(lclRowInd) = Tpetra::Details::OrdinalTraits<size_t>::invalid ();
6416 allDiagEntriesFound =
false;
6424 if (verbose && wrongOffsets.size () != 0) {
6425 std::ostringstream os;
6426 os << *prefix <<
"Wrong offsets: [";
6427 for (
size_t k = 0; k < wrongOffsets.size (); ++k) {
6428 os <<
"(" << wrongOffsets[k].first <<
","
6429 << wrongOffsets[k].second <<
")";
6430 if (k + 1 < wrongOffsets.size ()) {
6435 std::cerr << os.str();
6439 using Teuchos::reduceAll;
6441 Teuchos::RCP<const Teuchos::Comm<int> > comm = this->getComm ();
6442 const bool localSuccess =
6443 allRowMapDiagEntriesInColMap && allDiagEntriesFound && allOffsetsCorrect;
6444 const int numResults = 5;
6446 lclResults[0] = allRowMapDiagEntriesInColMap ? 1 : 0;
6447 lclResults[1] = allDiagEntriesFound ? 1 : 0;
6448 lclResults[2] = allOffsetsCorrect ? 1 : 0;
6449 lclResults[3] = noOtherWeirdness ? 1 : 0;
6452 lclResults[4] = ! localSuccess ? comm->getRank () : comm->getSize ();
6460 reduceAll<int, int> (*comm, Teuchos::REDUCE_MIN,
6461 numResults, lclResults, gblResults);
6463 if (gblResults[0] != 1 || gblResults[1] != 1 || gblResults[2] != 1
6464 || gblResults[3] != 1) {
6465 std::ostringstream os;
6466 os <<
"Issue(s) that we noticed (on Process " << gblResults[4] <<
", "
6467 "possibly among others): " << endl;
6468 if (gblResults[0] == 0) {
6469 os <<
" - The column Map does not contain at least one diagonal entry "
6470 "of the graph." << endl;
6472 if (gblResults[1] == 0) {
6473 os <<
" - On one or more processes, some row does not contain a "
6474 "diagonal entry." << endl;
6476 if (gblResults[2] == 0) {
6477 os <<
" - On one or more processes, some offsets are incorrect."
6480 if (gblResults[3] == 0) {
6481 os <<
" - One or more processes had some other error."
6484 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
true, std::runtime_error, os.str());
6489 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
6495 const char tfecfFuncName[] =
"getLocalOffRankOffsets: ";
6496 const bool verbose = verbose_;
6498 std::unique_ptr<std::string> prefix;
6500 prefix = this->createPrefix(
"CrsGraph",
"getLocalOffRankOffsets");
6501 std::ostringstream os;
6502 os << *prefix <<
"offsets.extent(0)=" << offsets.extent(0)
6504 std::cerr << os.str();
6507 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6508 (! hasColMap (), std::runtime_error,
"The graph must have a column Map.");
6511 const size_t lclNumRows = this->getLocalNumRows ();
6513 if (haveLocalOffRankOffsets_ && k_offRankOffsets_.extent(0) == lclNumRows+1) {
6514 offsets = k_offRankOffsets_;
6517 haveLocalOffRankOffsets_ =
false;
6519 const map_type& colMap = * (this->getColMap ());
6520 const map_type& domMap = * (this->getDomainMap ());
6531 TEUCHOS_ASSERT(this->isSorted ());
6532 if (isFillComplete ()) {
6533 k_offRankOffsets_ = offset_device_view_type(Kokkos::ViewAllocateWithoutInitializing(
"offRankOffset"), lclNumRows+1);
6534 auto lclGraph = this->getLocalGraphDevice ();
6535 ::Tpetra::Details::getGraphOffRankOffsets (k_offRankOffsets_,
6536 lclColMap, lclDomMap,
6538 offsets = k_offRankOffsets_;
6539 haveLocalOffRankOffsets_ =
true;
6541 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6542 (
true, std::logic_error,
"Can't get off-rank offsets for non-fill-complete graph");
6564 template<
class DeviceType,
6565 const bool memSpaceIsHostSpace =
6566 std::is_same<
typename DeviceType::memory_space,
6567 Kokkos::HostSpace>::value>
6568 struct HelpGetLocalDiagOffsets {};
6570 template<
class DeviceType>
6571 struct HelpGetLocalDiagOffsets<DeviceType, true> {
6572 typedef DeviceType device_type;
6573 typedef Kokkos::View<
size_t*, Kokkos::HostSpace,
6574 Kokkos::MemoryUnmanaged> device_offsets_type;
6575 typedef Kokkos::View<
size_t*, Kokkos::HostSpace,
6576 Kokkos::MemoryUnmanaged> host_offsets_type;
6578 static device_offsets_type
6579 getDeviceOffsets (
const host_offsets_type& hostOffsets)
6587 copyBackIfNeeded (
const host_offsets_type& ,
6588 const device_offsets_type& )
6592 template<
class DeviceType>
6593 struct HelpGetLocalDiagOffsets<DeviceType, false> {
6594 typedef DeviceType device_type;
6598 typedef Kokkos::View<size_t*, device_type> device_offsets_type;
6599 typedef Kokkos::View<
size_t*, Kokkos::HostSpace,
6600 Kokkos::MemoryUnmanaged> host_offsets_type;
6602 static device_offsets_type
6603 getDeviceOffsets (
const host_offsets_type& hostOffsets)
6607 return device_offsets_type (
"offsets", hostOffsets.extent (0));
6611 copyBackIfNeeded (
const host_offsets_type& hostOffsets,
6612 const device_offsets_type& deviceOffsets)