40 #ifndef TPETRA_CRSGRAPH_DEF_HPP
41 #define TPETRA_CRSGRAPH_DEF_HPP
50 #include "Tpetra_Details_getGraphDiagOffsets.hpp"
51 #include "Tpetra_Details_getGraphOffRankOffsets.hpp"
52 #include "Tpetra_Details_makeColMap.hpp"
56 #include "Tpetra_Distributor.hpp"
57 #include "Teuchos_SerialDenseMatrix.hpp"
58 #include "Tpetra_Vector.hpp"
61 #include "Tpetra_Details_packCrsGraph.hpp"
62 #include "Tpetra_Details_unpackCrsGraphAndCombine.hpp"
63 #include "Tpetra_Details_CrsPadding.hpp"
70 #include <type_traits>
78 template<
class MapIter>
80 verbosePrintMap(std::ostream& out,
86 using ::Tpetra::Details::Behavior;
89 out << mapName <<
": {";
90 const size_t maxNumToPrint =
92 if (maxNumToPrint == 0) {
98 const size_t numToPrint = numEnt > maxNumToPrint ?
99 maxNumToPrint : numEnt;
101 for (MapIter it = beg; it != end; ++it) {
102 out <<
"(" << (*it).first <<
", ";
106 if (count +
size_t(1) < numToPrint) {
111 if (count < numEnt) {
118 template<
class LO,
class GO,
class Node>
119 Teuchos::ArrayView<GO>
120 getRowGraphGlobalRow(
121 std::vector<GO>& gblColIndsStorage,
122 const RowGraph<LO, GO, Node>& graph,
125 size_t origNumEnt = graph.getNumEntriesInGlobalRow(gblRowInd);
126 if (gblColIndsStorage.size() < origNumEnt) {
127 gblColIndsStorage.resize(origNumEnt);
129 typename CrsGraph<LO,GO,Node>::nonconst_global_inds_host_view_type gblColInds(gblColIndsStorage.data(),
131 graph.getGlobalRowCopy(gblRowInd, gblColInds, origNumEnt);
132 Teuchos::ArrayView<GO> retval(gblColIndsStorage.data(),origNumEnt);
136 template<
class LO,
class GO,
class DT,
class OffsetType,
class NumEntType>
137 class ConvertColumnIndicesFromGlobalToLocal {
139 ConvertColumnIndicesFromGlobalToLocal (const ::Kokkos::View<LO*, DT>& lclColInds,
140 const ::Kokkos::View<const GO*, DT>& gblColInds,
141 const ::Kokkos::View<const OffsetType*, DT>& ptr,
142 const ::Tpetra::Details::LocalMap<LO, GO, DT>& lclColMap,
143 const ::Kokkos::View<const NumEntType*, DT>& numRowEnt) :
144 lclColInds_ (lclColInds),
145 gblColInds_ (gblColInds),
147 lclColMap_ (lclColMap),
148 numRowEnt_ (numRowEnt)
152 operator () (
const LO& lclRow, OffsetType& curNumBad)
const
154 const OffsetType offset = ptr_(lclRow);
158 const LO numEnt =
static_cast<LO
> (numRowEnt_(lclRow));
159 for (LO j = 0; j < numEnt; ++j) {
160 const GO gid = gblColInds_(offset + j);
161 const LO lid = lclColMap_.getLocalElement (gid);
162 lclColInds_(offset + j) = lid;
163 if (lid == ::Tpetra::Details::OrdinalTraits<LO>::invalid ()) {
170 run (const ::Kokkos::View<LO*, DT>& lclColInds,
171 const ::Kokkos::View<const GO*, DT>& gblColInds,
172 const ::Kokkos::View<const OffsetType*, DT>& ptr,
173 const ::Tpetra::Details::LocalMap<LO, GO, DT>& lclColMap,
174 const ::Kokkos::View<const NumEntType*, DT>& numRowEnt)
176 typedef ::Kokkos::RangePolicy<typename DT::execution_space, LO> range_type;
177 typedef ConvertColumnIndicesFromGlobalToLocal<LO, GO, DT, OffsetType, NumEntType> functor_type;
179 const LO lclNumRows = ptr.extent (0) == 0 ?
180 static_cast<LO
> (0) : static_cast<LO> (ptr.extent (0) - 1);
181 OffsetType numBad = 0;
183 ::Kokkos::parallel_reduce (range_type (0, lclNumRows),
184 functor_type (lclColInds, gblColInds, ptr,
185 lclColMap, numRowEnt),
191 ::Kokkos::View<LO*, DT> lclColInds_;
192 ::Kokkos::View<const GO*, DT> gblColInds_;
193 ::Kokkos::View<const OffsetType*, DT> ptr_;
195 ::Kokkos::View<const NumEntType*, DT> numRowEnt_;
214 template<
class LO,
class GO,
class DT,
class OffsetType,
class NumEntType>
217 const Kokkos::View<const GO*, DT>& gblColInds,
218 const Kokkos::View<const OffsetType*, DT>& ptr,
220 const Kokkos::View<const NumEntType*, DT>& numRowEnt)
222 using Impl::ConvertColumnIndicesFromGlobalToLocal;
223 typedef ConvertColumnIndicesFromGlobalToLocal<LO, GO, DT, OffsetType, NumEntType> impl_type;
224 return impl_type::run (lclColInds, gblColInds, ptr, lclColMap, numRowEnt);
227 template<
class ViewType,
class LO>
228 class MaxDifference {
230 MaxDifference (
const ViewType& ptr) : ptr_ (ptr) {}
232 KOKKOS_INLINE_FUNCTION
void init (LO& dst)
const {
236 KOKKOS_INLINE_FUNCTION
void
237 join (LO& dst,
const LO& src)
const
239 dst = (src > dst) ? src : dst;
242 KOKKOS_INLINE_FUNCTION
void
243 operator () (
const LO lclRow, LO& maxNumEnt)
const
245 const LO numEnt =
static_cast<LO
> (ptr_(lclRow+1) - ptr_(lclRow));
246 maxNumEnt = (numEnt > maxNumEnt) ? numEnt : maxNumEnt;
249 typename ViewType::const_type ptr_;
252 template<
class ViewType,
class LO>
253 typename ViewType::non_const_value_type
254 maxDifference (
const char kernelLabel[],
258 if (lclNumRows == 0) {
261 return static_cast<LO
> (0);
264 using execution_space =
typename ViewType::execution_space;
265 using range_type = Kokkos::RangePolicy<execution_space, LO>;
267 Kokkos::parallel_reduce (kernelLabel,
268 range_type (0, lclNumRows),
269 MaxDifference<ViewType, LO> (ptr),
277 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
279 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
284 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
286 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
291 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
292 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
293 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
294 const size_t maxNumEntriesPerRow,
295 const Teuchos::RCP<Teuchos::ParameterList>& params) :
298 , numAllocForAllRows_ (maxNumEntriesPerRow)
300 const char tfecfFuncName[] =
301 "CrsGraph(rowMap,maxNumEntriesPerRow,params): ";
303 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
304 (maxNumEntriesPerRow == Teuchos::OrdinalTraits<size_t>::invalid (),
305 std::invalid_argument,
"The allocation hint maxNumEntriesPerRow must be "
306 "a valid size_t value, which in this case means it must not be "
307 "Teuchos::OrdinalTraits<size_t>::invalid().");
312 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
314 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
315 const Teuchos::RCP<const map_type>& colMap,
316 const size_t maxNumEntriesPerRow,
317 const Teuchos::RCP<Teuchos::ParameterList>& params) :
321 , numAllocForAllRows_ (maxNumEntriesPerRow)
323 const char tfecfFuncName[] =
324 "CrsGraph(rowMap,colMap,maxNumEntriesPerRow,params): ";
326 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
327 maxNumEntriesPerRow == Teuchos::OrdinalTraits<size_t>::invalid (),
328 std::invalid_argument,
"The allocation hint maxNumEntriesPerRow must be "
329 "a valid size_t value, which in this case means it must not be "
330 "Teuchos::OrdinalTraits<size_t>::invalid().");
336 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
338 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
339 const Teuchos::ArrayView<const size_t>& numEntPerRow,
340 const Teuchos::RCP<Teuchos::ParameterList>& params) :
343 , numAllocForAllRows_ (0)
345 const char tfecfFuncName[] =
346 "CrsGraph(rowMap,numEntPerRow,params): ";
349 const size_t lclNumRows = rowMap.is_null () ?
350 static_cast<size_t> (0) : rowMap->getLocalNumElements ();
351 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
352 static_cast<size_t> (numEntPerRow.size ()) != lclNumRows,
353 std::invalid_argument,
"numEntPerRow has length " << numEntPerRow.size ()
354 <<
" != the local number of rows " << lclNumRows <<
" as specified by "
355 "the input row Map.");
358 for (
size_t r = 0; r < lclNumRows; ++r) {
359 const size_t curRowCount = numEntPerRow[r];
360 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
361 (curRowCount == Teuchos::OrdinalTraits<size_t>::invalid (),
362 std::invalid_argument,
"numEntPerRow(" << r <<
") "
363 "specifies an invalid number of entries "
364 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
372 typedef typename out_view_type::non_const_type nc_view_type;
373 typedef Kokkos::View<
const size_t*,
374 typename nc_view_type::array_layout,
376 Kokkos::MemoryUnmanaged> in_view_type;
377 in_view_type numAllocPerRowIn (numEntPerRow.getRawPtr (), lclNumRows);
378 nc_view_type numAllocPerRowOut (
"Tpetra::CrsGraph::numAllocPerRow",
381 using exec_space =
typename nc_view_type::execution_space;
391 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
393 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
394 const Kokkos::DualView<const size_t*, device_type>& numEntPerRow,
395 const Teuchos::RCP<Teuchos::ParameterList>& params) :
398 , k_numAllocPerRow_ (numEntPerRow.h_view)
399 , numAllocForAllRows_ (0)
401 const char tfecfFuncName[] =
402 "CrsGraph(rowMap,numEntPerRow,params): ";
405 const size_t lclNumRows = rowMap.is_null () ?
406 static_cast<size_t> (0) : rowMap->getLocalNumElements ();
407 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
408 static_cast<size_t> (numEntPerRow.extent (0)) != lclNumRows,
409 std::invalid_argument,
"numEntPerRow has length " <<
410 numEntPerRow.extent (0) <<
" != the local number of rows " <<
411 lclNumRows <<
" as specified by " "the input row Map.");
414 for (
size_t r = 0; r < lclNumRows; ++r) {
415 const size_t curRowCount = numEntPerRow.h_view(r);
416 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
417 (curRowCount == Teuchos::OrdinalTraits<size_t>::invalid (),
418 std::invalid_argument,
"numEntPerRow(" << r <<
") "
419 "specifies an invalid number of entries "
420 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
429 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
431 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
432 const Teuchos::RCP<const map_type>& colMap,
433 const Kokkos::DualView<const size_t*, device_type>& numEntPerRow,
434 const Teuchos::RCP<Teuchos::ParameterList>& params) :
438 , k_numAllocPerRow_ (numEntPerRow.h_view)
439 , numAllocForAllRows_ (0)
441 const char tfecfFuncName[] =
442 "CrsGraph(rowMap,colMap,numEntPerRow,params): ";
445 const size_t lclNumRows = rowMap.is_null () ?
446 static_cast<size_t> (0) : rowMap->getLocalNumElements ();
447 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
448 static_cast<size_t> (numEntPerRow.extent (0)) != lclNumRows,
449 std::invalid_argument,
"numEntPerRow has length " <<
450 numEntPerRow.extent (0) <<
" != the local number of rows " <<
451 lclNumRows <<
" as specified by " "the input row Map.");
454 for (
size_t r = 0; r < lclNumRows; ++r) {
455 const size_t curRowCount = numEntPerRow.h_view(r);
456 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
457 (curRowCount == Teuchos::OrdinalTraits<size_t>::invalid (),
458 std::invalid_argument,
"numEntPerRow(" << r <<
") "
459 "specifies an invalid number of entries "
460 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
469 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
471 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
472 const Teuchos::RCP<const map_type>& colMap,
473 const Teuchos::ArrayView<const size_t>& numEntPerRow,
474 const Teuchos::RCP<Teuchos::ParameterList>& params) :
478 , numAllocForAllRows_ (0)
480 const char tfecfFuncName[] =
481 "CrsGraph(rowMap,colMap,numEntPerRow,params): ";
484 const size_t lclNumRows = rowMap.is_null () ?
485 static_cast<size_t> (0) : rowMap->getLocalNumElements ();
486 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
487 static_cast<size_t> (numEntPerRow.size ()) != lclNumRows,
488 std::invalid_argument,
"numEntPerRow has length " << numEntPerRow.size ()
489 <<
" != the local number of rows " << lclNumRows <<
" as specified by "
490 "the input row Map.");
493 for (
size_t r = 0; r < lclNumRows; ++r) {
494 const size_t curRowCount = numEntPerRow[r];
495 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
496 (curRowCount == Teuchos::OrdinalTraits<size_t>::invalid (),
497 std::invalid_argument,
"numEntPerRow(" << r <<
") "
498 "specifies an invalid number of entries "
499 "(Teuchos::OrdinalTraits<size_t>::invalid()).");
507 typedef typename out_view_type::non_const_type nc_view_type;
508 typedef Kokkos::View<
const size_t*,
509 typename nc_view_type::array_layout,
511 Kokkos::MemoryUnmanaged> in_view_type;
512 in_view_type numAllocPerRowIn (numEntPerRow.getRawPtr (), lclNumRows);
513 nc_view_type numAllocPerRowOut (
"Tpetra::CrsGraph::numAllocPerRow",
516 using exec_space =
typename nc_view_type::execution_space;
525 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
528 const Teuchos::RCP<const map_type>& rowMap,
529 const Teuchos::RCP<Teuchos::ParameterList>& params) :
532 , colMap_(originalGraph.colMap_)
533 , numAllocForAllRows_(originalGraph.numAllocForAllRows_)
534 , storageStatus_(originalGraph.storageStatus_)
535 , indicesAreAllocated_(originalGraph.indicesAreAllocated_)
536 , indicesAreLocal_(originalGraph.indicesAreLocal_)
537 , indicesAreSorted_(originalGraph.indicesAreSorted_)
541 int numRows = rowMap->getLocalNumElements();
542 size_t numNonZeros = originalGraph.rowPtrsPacked_host_(numRows);
543 auto rowsToUse = Kokkos::pair<size_t, size_t>(0, numRows+1);
545 rowPtrsUnpacked_dev_ = Kokkos::subview(originalGraph.rowPtrsUnpacked_dev_, rowsToUse);
546 rowPtrsUnpacked_host_ = Kokkos::subview(originalGraph.rowPtrsUnpacked_host_, rowsToUse);
548 rowPtrsPacked_dev_ = Kokkos::subview(originalGraph.rowPtrsPacked_dev_, rowsToUse);
549 rowPtrsPacked_host_ = Kokkos::subview(originalGraph.rowPtrsPacked_host_, rowsToUse);
551 if (indicesAreLocal_) {
562 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
564 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
565 const Teuchos::RCP<const map_type>& colMap,
566 const typename local_graph_device_type::row_map_type& rowPointers,
567 const typename local_graph_device_type::entries_type::non_const_type& columnIndices,
568 const Teuchos::RCP<Teuchos::ParameterList>& params) :
572 , numAllocForAllRows_(0)
573 , storageStatus_(Details::STORAGE_1D_PACKED)
574 , indicesAreAllocated_(true)
575 , indicesAreLocal_(true)
578 if (! params.is_null() && params->isParameter(
"sorted") &&
579 ! params->get<
bool>(
"sorted")) {
589 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
591 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
592 const Teuchos::RCP<const map_type>& colMap,
593 const Teuchos::ArrayRCP<size_t>& rowPointers,
594 const Teuchos::ArrayRCP<LocalOrdinal> & columnIndices,
595 const Teuchos::RCP<Teuchos::ParameterList>& params) :
599 , numAllocForAllRows_ (0)
600 , storageStatus_ (Details::STORAGE_1D_PACKED)
601 , indicesAreAllocated_ (true)
602 , indicesAreLocal_ (true)
605 if (! params.is_null() && params->isParameter(
"sorted") &&
606 ! params->get<
bool>(
"sorted")) {
616 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
618 CrsGraph (
const Teuchos::RCP<const map_type>& rowMap,
619 const Teuchos::RCP<const map_type>& colMap,
621 const Teuchos::RCP<Teuchos::ParameterList>& params)
630 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
633 const Teuchos::RCP<const map_type>& rowMap,
634 const Teuchos::RCP<const map_type>& colMap,
635 const Teuchos::RCP<const map_type>& domainMap,
636 const Teuchos::RCP<const map_type>& rangeMap,
637 const Teuchos::RCP<Teuchos::ParameterList>& params)
641 , numAllocForAllRows_ (0)
642 , storageStatus_ (Details::STORAGE_1D_PACKED)
643 , indicesAreAllocated_ (true)
644 , indicesAreLocal_ (true)
647 const char tfecfFuncName[] =
"CrsGraph(Kokkos::LocalStaticCrsGraph,Map,Map,Map,Map)";
649 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
650 colMap.is_null (), std::runtime_error,
651 ": The input column Map must be nonnull.");
652 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
653 k_local_graph_.numRows () != rowMap->getLocalNumElements (),
655 ": The input row Map and the input local graph need to have the same "
656 "number of rows. The row Map claims " << rowMap->getLocalNumElements ()
657 <<
" row(s), but the local graph claims " << k_local_graph_.numRows ()
667 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
669 ": cannot have 1D data structures allocated.");
671 if(! params.is_null() && params->isParameter(
"sorted") &&
672 ! params->get<
bool>(
"sorted")) {
680 rangeMap .is_null() ?
rowMap_ : rangeMap);
681 Teuchos::Array<int> remotePIDs (0);
686 this->setRowPtrs(k_local_graph_.row_map);
688 set_need_sync_host_uvm_access();
690 const bool callComputeGlobalConstants = params.get () ==
nullptr ||
691 params->get (
"compute global constants",
true);
693 if (callComputeGlobalConstants) {
696 this->fillComplete_ =
true;
700 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
703 const Teuchos::RCP<const map_type>& rowMap,
704 const Teuchos::RCP<const map_type>& colMap,
705 const Teuchos::RCP<const map_type>& domainMap,
706 const Teuchos::RCP<const map_type>& rangeMap,
707 const Teuchos::RCP<const import_type>& importer,
708 const Teuchos::RCP<const export_type>& exporter,
709 const Teuchos::RCP<Teuchos::ParameterList>& params) :
713 rangeMap_ (rangeMap.is_null () ? rowMap : rangeMap),
714 domainMap_ (domainMap.is_null () ? rowMap : domainMap),
715 importer_ (importer),
716 exporter_ (exporter),
717 numAllocForAllRows_ (0),
718 storageStatus_ (Details::STORAGE_1D_PACKED),
719 indicesAreAllocated_ (true),
720 indicesAreLocal_ (true)
723 const char tfecfFuncName[] =
"Tpetra::CrsGraph(local_graph_device_type,"
724 "Map,Map,Map,Map,Import,Export,params): ";
726 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
727 (colMap.is_null (), std::runtime_error,
728 "The input column Map must be nonnull.");
732 setRowPtrs(lclGraph.row_map);
734 set_need_sync_host_uvm_access();
736 if (! params.is_null() && params->isParameter(
"sorted") &&
737 ! params->get<
bool>(
"sorted")) {
744 const bool callComputeGlobalConstants =
745 params.get () ==
nullptr ||
746 params->get (
"compute global constants",
true);
747 if (callComputeGlobalConstants) {
750 fillComplete_ =
true;
754 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
755 Teuchos::RCP<const Teuchos::ParameterList>
760 using Teuchos::ParameterList;
761 using Teuchos::parameterList;
763 RCP<ParameterList> params = parameterList (
"Tpetra::CrsGraph");
766 RCP<ParameterList> importSublist = parameterList (
"Import");
778 Distributor distributor (rowMap_->getComm (), importSublist);
779 params->set (
"Import", *importSublist,
"How the Import performs communication.");
785 params->set (
"Export", *importSublist,
"How the Export performs communication.");
790 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
795 Teuchos::RCP<const Teuchos::ParameterList> validParams =
796 getValidParameters ();
797 params->validateParametersAndSetDefaults (*validParams);
798 this->setMyParamList (params);
801 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
806 return rowMap_->getGlobalNumElements ();
809 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
814 const char tfecfFuncName[] =
"getGlobalNumCols: ";
815 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
816 ! isFillComplete () || getDomainMap ().is_null (), std::runtime_error,
817 "The graph does not have a domain Map. You may not call this method in "
819 return getDomainMap ()->getGlobalNumElements ();
823 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
828 return this->rowMap_.is_null () ?
829 static_cast<size_t> (0) :
830 this->rowMap_->getLocalNumElements ();
834 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
839 const char tfecfFuncName[] =
"getLocalNumCols: ";
840 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
841 ! hasColMap (), std::runtime_error,
842 "The graph does not have a column Map. You may not call this method "
843 "unless the graph has a column Map. This requires either that a custom "
844 "column Map was given to the constructor, or that fillComplete() has "
846 return colMap_.is_null () ?
static_cast<size_t> (0) :
847 colMap_->getLocalNumElements ();
852 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
853 Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
860 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
861 Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
868 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
869 Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
876 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
877 Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::map_type>
884 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
885 Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::import_type>
892 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
893 Teuchos::RCP<const typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::export_type>
900 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
905 return ! colMap_.is_null ();
908 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
916 const bool isOpt = indicesAreAllocated_ &&
917 k_numRowEntries_.extent (0) == 0 &&
918 getLocalNumRows () > 0;
924 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
929 const char tfecfFuncName[] =
"getGlobalNumEntries: ";
930 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
931 (! this->haveGlobalConstants_, std::logic_error,
932 "The graph does not have global constants computed, "
933 "but the user has requested them.");
935 return globalNumEntries_;
939 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
944 typedef LocalOrdinal LO;
946 if (this->indicesAreAllocated_) {
947 const LO lclNumRows = this->getLocalNumRows ();
948 if (lclNumRows == 0) {
949 return static_cast<size_t> (0);
953 auto numEntPerRow = this->k_numRowEntries_;
954 const LO numNumEntPerRow = numEntPerRow.extent (0);
955 if (numNumEntPerRow == 0) {
956 if (static_cast<LO> (this->rowPtrsPacked_dev_.extent (0)) <
957 static_cast<LO> (lclNumRows + 1)) {
958 return static_cast<size_t> (0);
961 return this->rowPtrsPacked_host_(lclNumRows);
971 typedef typename num_row_entries_type::execution_space
973 typedef Kokkos::RangePolicy<host_exec_space, LO> range_type;
975 const LO upperLoopBound = lclNumRows < numNumEntPerRow ?
978 size_t nodeNumEnt = 0;
979 Kokkos::parallel_reduce (
"Tpetra::CrsGraph::getNumNodeEntries",
980 range_type (0, upperLoopBound),
981 [=] (
const LO& k,
size_t& lclSum) {
982 lclSum += numEntPerRow(k);
989 return static_cast<size_t> (0);
993 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
998 const char tfecfFuncName[] =
"getGlobalMaxNumRowEntries: ";
999 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1000 (! this->haveGlobalConstants_, std::logic_error,
1001 "The graph does not have global constants computed, "
1002 "but the user has requested them.");
1004 return globalMaxNumRowEntries_;
1007 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1012 return nodeMaxNumRowEntries_;
1015 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1020 return fillComplete_;
1023 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1028 return ! fillComplete_;
1032 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1037 return indicesAreLocal_;
1040 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1045 return indicesAreGlobal_;
1048 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1053 typedef LocalOrdinal LO;
1055 if (this->indicesAreAllocated_) {
1056 const LO lclNumRows = this->getLocalNumRows ();
1057 if (lclNumRows == 0) {
1058 return static_cast<size_t> (0);
1060 else if (storageStatus_ == Details::STORAGE_1D_PACKED) {
1061 if (static_cast<LO> (this->rowPtrsPacked_dev_.extent (0)) <
1062 static_cast<LO> (lclNumRows + 1)) {
1063 return static_cast<size_t> (0);
1066 return this->rowPtrsPacked_host_(lclNumRows);
1069 else if (storageStatus_ == Details::STORAGE_1D_UNPACKED) {
1070 if (rowPtrsUnpacked_host_.extent (0) == 0) {
1071 return static_cast<size_t> (0);
1074 return rowPtrsUnpacked_host_(lclNumRows);
1078 return static_cast<size_t> (0);
1082 return Tpetra::Details::OrdinalTraits<size_t>::invalid ();
1086 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1087 Teuchos::RCP<const Teuchos::Comm<int> >
1091 return this->rowMap_.is_null () ? Teuchos::null : this->rowMap_->getComm ();
1094 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1099 return rowMap_->getIndexBase ();
1102 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1107 return indicesAreAllocated_;
1110 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1115 return indicesAreSorted_;
1118 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1123 return noRedundancies_;
1126 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1136 indicesAreSorted_ =
false;
1137 noRedundancies_ =
false;
1141 haveLocalConstants_ =
false;
1144 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1150 using Teuchos::arcp;
1151 using Teuchos::Array;
1152 using Teuchos::ArrayRCP;
1154 typedef Teuchos::ArrayRCP<size_t>::size_type size_type;
1155 typedef typename local_graph_device_type::row_map_type::non_const_type
1156 non_const_row_map_type;
1157 const char tfecfFuncName[] =
"allocateIndices: ";
1158 const char suffix[] =
1159 " Please report this bug to the Tpetra developers.";
1160 ProfilingRegion profRegion(
"Tpetra::CrsGraph::allocateIndices");
1162 std::unique_ptr<std::string> prefix;
1164 prefix = this->createPrefix(
"CrsGraph", tfecfFuncName);
1165 std::ostringstream os;
1166 os << *prefix <<
"Start: lg="
1167 << (lg == GlobalIndices ?
"GlobalIndices" :
"LocalIndices")
1168 <<
", numRows: " << this->getLocalNumRows() << endl;
1169 std::cerr << os.str();
1175 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1176 (isLocallyIndexed () && lg == GlobalIndices, std::logic_error,
1177 ": The graph is locally indexed, but Tpetra code is calling "
1178 "this method with lg=GlobalIndices." << suffix);
1179 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1180 (isGloballyIndexed () && lg == LocalIndices, std::logic_error,
1181 ": The graph is globally indexed, but Tpetra code is calling "
1182 "this method with lg=LocalIndices." << suffix);
1183 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1184 (indicesAreAllocated (), std::logic_error,
": The graph's "
1185 "indices are already allocated, but Tpetra is calling "
1186 "allocateIndices again." << suffix);
1187 const size_t numRows = this->getLocalNumRows ();
1194 std::ostringstream os;
1195 os << *prefix <<
"Allocate k_rowPtrs: " << (numRows+1) << endl;
1196 std::cerr << os.str();
1198 non_const_row_map_type k_rowPtrs (
"Tpetra::CrsGraph::ptr", numRows + 1);
1200 if (this->k_numAllocPerRow_.extent (0) != 0) {
1205 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1206 (this->k_numAllocPerRow_.extent (0) != numRows,
1207 std::invalid_argument,
"k_numAllocPerRow_ is allocated, that is, "
1208 "has nonzero length " << this->k_numAllocPerRow_.extent (0)
1209 <<
", but its length != numRows = " << numRows <<
".");
1227 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1228 (this->numAllocForAllRows_ ==
1229 Tpetra::Details::OrdinalTraits<size_t>::invalid (),
1230 std::invalid_argument,
"numAllocForAllRows_ has an invalid value, "
1231 "namely Tpetra::Details::OrdinalTraits<size_t>::invalid() = " <<
1232 Tpetra::Details::OrdinalTraits<size_t>::invalid () <<
".");
1239 setRowPtrsUnpacked(k_rowPtrs);
1242 const size_type numInds = rowPtrsUnpacked_host_(numRows);
1243 if (lg == LocalIndices) {
1245 std::ostringstream os;
1246 os << *prefix <<
"Allocate local column indices "
1247 "lclIndsUnpacked_wdv: " << numInds << endl;
1248 std::cerr << os.str();
1250 lclIndsUnpacked_wdv = local_inds_wdv_type (
1251 local_inds_dualv_type(
"Tpetra::CrsGraph::lclInd",numInds));
1255 std::ostringstream os;
1256 os << *prefix <<
"Allocate global column indices "
1257 "gblInds_wdv: " << numInds << endl;
1258 std::cerr << os.str();
1260 gblInds_wdv = global_inds_wdv_type (
1261 global_inds_dualv_type(
"Tpetra::CrsGraph::gblInd",numInds));
1263 storageStatus_ = Details::STORAGE_1D_UNPACKED;
1265 this->indicesAreLocal_ = (lg == LocalIndices);
1266 this->indicesAreGlobal_ = (lg == GlobalIndices);
1269 using Kokkos::ViewAllocateWithoutInitializing;
1270 typedef decltype (k_numRowEntries_) row_ent_type;
1271 const
char label[] = "Tpetra::CrsGraph::numRowEntries";
1273 std::ostringstream os;
1274 os << *prefix <<
"Allocate k_numRowEntries_: " << numRows
1276 std::cerr << os.str();
1278 row_ent_type numRowEnt (ViewAllocateWithoutInitializing (label), numRows);
1282 this->k_numRowEntries_ = numRowEnt;
1286 this->numAllocForAllRows_ = 0;
1287 this->k_numAllocPerRow_ = decltype (k_numAllocPerRow_) ();
1288 this->indicesAreAllocated_ =
true;
1291 this->checkInternalState ();
1293 catch (std::logic_error& e) {
1294 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1295 (
true, std::logic_error,
"At end of allocateIndices, "
1296 "checkInternalState threw std::logic_error: "
1299 catch (std::exception& e) {
1300 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1301 (
true, std::runtime_error,
"At end of allocateIndices, "
1302 "checkInternalState threw std::exception: "
1306 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1307 (
true, std::runtime_error,
"At end of allocateIndices, "
1308 "checkInternalState threw an exception "
1309 "not a subclass of std::exception.");
1313 std::ostringstream os;
1314 os << *prefix <<
"Done" << endl;
1315 std::cerr << os.str();
1319 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1320 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
1321 local_inds_dualv_type::t_host::const_type
1325 if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
1326 return typename local_inds_dualv_type::t_host::const_type ();
1328 return lclIndsUnpacked_wdv.getHostSubview(rowinfo.offset1D,
1333 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1335 local_inds_dualv_type::t_host
1339 if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
1340 return typename local_inds_dualv_type::t_host ();
1342 return lclIndsUnpacked_wdv.getHostSubview(rowinfo.offset1D,
1347 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1349 global_inds_dualv_type::t_host::const_type
1353 if (rowinfo.allocSize == 0 || gblInds_wdv.extent(0) == 0)
1354 return typename global_inds_dualv_type::t_host::const_type ();
1356 return gblInds_wdv.getHostSubview(rowinfo.offset1D,
1361 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1363 local_inds_dualv_type::t_dev::const_type
1367 if (rowinfo.allocSize == 0 || lclIndsUnpacked_wdv.extent(0) == 0)
1368 return typename local_inds_dualv_type::t_dev::const_type ();
1370 return lclIndsUnpacked_wdv.getDeviceSubview(rowinfo.offset1D,
1375 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1377 global_inds_dualv_type::t_dev::const_type
1381 if (rowinfo.allocSize == 0 || gblInds_wdv.extent(0) == 0)
1382 return typename global_inds_dualv_type::t_dev::const_type ();
1384 return gblInds_wdv.getDeviceSubview(rowinfo.offset1D,
1390 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1395 const size_t STINV = Teuchos::OrdinalTraits<size_t>::invalid ();
1397 if (this->rowMap_.is_null () || ! this->rowMap_->isNodeLocalElement (myRow)) {
1398 ret.localRow = STINV;
1401 ret.offset1D = STINV;
1405 ret.localRow =
static_cast<size_t> (myRow);
1406 if (this->indicesAreAllocated ()) {
1408 if (rowPtrsUnpacked_host_.extent (0) == 0) {
1413 ret.offset1D = rowPtrsUnpacked_host_(myRow);
1414 ret.allocSize = rowPtrsUnpacked_host_(myRow+1) - rowPtrsUnpacked_host_(myRow);
1417 ret.numEntries = (this->k_numRowEntries_.extent (0) == 0) ?
1419 this->k_numRowEntries_(myRow);
1425 ret.allocSize = (this->k_numAllocPerRow_.extent (0) != 0) ?
1426 this->k_numAllocPerRow_(myRow) :
1427 this->numAllocForAllRows_;
1429 ret.offset1D = STINV;
1436 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1441 const size_t STINV = Teuchos::OrdinalTraits<size_t>::invalid ();
1443 if (this->rowMap_.is_null ()) {
1444 ret.localRow = STINV;
1447 ret.offset1D = STINV;
1450 const LocalOrdinal myRow = this->rowMap_->getLocalElement (gblRow);
1451 if (myRow == Teuchos::OrdinalTraits<LocalOrdinal>::invalid ()) {
1452 ret.localRow = STINV;
1455 ret.offset1D = STINV;
1459 ret.localRow =
static_cast<size_t> (myRow);
1460 if (this->indicesAreAllocated ()) {
1464 if (rowPtrsUnpacked_host_.extent (0) == 0) {
1469 ret.offset1D = rowPtrsUnpacked_host_(myRow);
1470 ret.allocSize = rowPtrsUnpacked_host_(myRow+1) - rowPtrsUnpacked_host_(myRow);
1473 ret.numEntries = (this->k_numRowEntries_.extent (0) == 0) ?
1475 this->k_numRowEntries_(myRow);
1481 ret.allocSize = (this->k_numAllocPerRow_.extent (0) != 0) ?
1482 this->k_numAllocPerRow_(myRow) :
1483 this->numAllocForAllRows_;
1485 ret.offset1D = STINV;
1492 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1497 using Teuchos::OrdinalTraits;
1498 typedef LocalOrdinal LO;
1499 typedef GlobalOrdinal GO;
1505 static_assert (
sizeof (GlobalOrdinal) >=
sizeof (LocalOrdinal),
1506 "Tpetra::CrsGraph: sizeof(GlobalOrdinal) must be >= sizeof(LocalOrdinal).");
1509 static_assert (
sizeof (
size_t) >=
sizeof (LocalOrdinal),
1510 "Tpetra::CrsGraph: sizeof(size_t) must be >= sizeof(LocalOrdinal).");
1511 static_assert (
sizeof(GST) >=
sizeof(
size_t),
1512 "Tpetra::CrsGraph: sizeof(Tpetra::global_size_t) must be >= sizeof(size_t).");
1520 const char msg[] =
"Tpetra::CrsGraph: Object cannot be created with the "
1521 "given template arguments: size assumptions are not valid.";
1522 TEUCHOS_TEST_FOR_EXCEPTION(
1523 static_cast<size_t> (Teuchos::OrdinalTraits<LO>::max ()) > Teuchos::OrdinalTraits<size_t>::max (),
1524 std::runtime_error, msg);
1525 TEUCHOS_TEST_FOR_EXCEPTION(
1526 static_cast<GST> (Teuchos::OrdinalTraits<LO>::max ()) > static_cast<GST> (Teuchos::OrdinalTraits<GO>::max ()),
1527 std::runtime_error, msg);
1528 TEUCHOS_TEST_FOR_EXCEPTION(
1529 static_cast<size_t> (Teuchos::OrdinalTraits<GO>::max ()) > Teuchos::OrdinalTraits<GST>::max(),
1530 std::runtime_error, msg);
1531 TEUCHOS_TEST_FOR_EXCEPTION(
1532 Teuchos::OrdinalTraits<size_t>::max () > Teuchos::OrdinalTraits<GST>::max (),
1533 std::runtime_error, msg);
1537 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1541 const SLocalGlobalViews &newInds,
1542 const ELocalGlobal lg,
1543 const ELocalGlobal I)
1545 using Teuchos::ArrayView;
1546 typedef LocalOrdinal LO;
1547 typedef GlobalOrdinal GO;
1548 const char tfecfFuncName[] =
"insertIndices: ";
1550 size_t oldNumEnt = 0;
1552 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1553 (lg != GlobalIndices && lg != LocalIndices, std::invalid_argument,
1554 "lg must be either GlobalIndices or LocalIndices.");
1555 oldNumEnt = this->getNumEntriesInLocalRow (rowinfo.localRow);
1558 size_t numNewInds = 0;
1559 if (lg == GlobalIndices) {
1560 ArrayView<const GO> new_ginds = newInds.ginds;
1561 numNewInds = new_ginds.size();
1562 if (I == GlobalIndices) {
1563 auto gind_view = gblInds_wdv.getHostView(Access::ReadWrite);
1565 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1566 (static_cast<size_t> (gind_view.size ()) <
1567 rowinfo.numEntries + numNewInds, std::logic_error,
1568 "gind_view.size() = " << gind_view.size ()
1569 <<
" < rowinfo.numEntries (= " << rowinfo.numEntries
1570 <<
") + numNewInds (= " << numNewInds <<
").");
1572 GO*
const gblColInds_out = gind_view.data () + rowinfo.offset1D
1573 + rowinfo.numEntries;
1574 for (
size_t k = 0; k < numNewInds; ++k) {
1575 gblColInds_out[k] = new_ginds[k];
1578 else if (I == LocalIndices) {
1579 auto lind_view = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
1581 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1582 (static_cast<size_t> (lind_view.size ()) <
1583 rowinfo.numEntries + numNewInds, std::logic_error,
1584 "lind_view.size() = " << lind_view.size ()
1585 <<
" < rowinfo.numEntries (= " << rowinfo.numEntries
1586 <<
") + numNewInds (= " << numNewInds <<
").");
1588 LO*
const lclColInds_out = lind_view.data () + rowinfo.offset1D
1589 + rowinfo.numEntries;
1590 for (
size_t k = 0; k < numNewInds; ++k) {
1591 lclColInds_out[k] = colMap_->getLocalElement (new_ginds[k]);
1595 else if (lg == LocalIndices) {
1596 ArrayView<const LO> new_linds = newInds.linds;
1597 numNewInds = new_linds.size();
1598 if (I == LocalIndices) {
1599 auto lind_view = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
1601 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1602 (static_cast<size_t> (lind_view.size ()) <
1603 rowinfo.numEntries + numNewInds, std::logic_error,
1604 "lind_view.size() = " << lind_view.size ()
1605 <<
" < rowinfo.numEntries (= " << rowinfo.numEntries
1606 <<
") + numNewInds (= " << numNewInds <<
").");
1608 LO*
const lclColInds_out = lind_view.data () + rowinfo.offset1D
1609 + rowinfo.numEntries;
1610 for (
size_t k = 0; k < numNewInds; ++k) {
1611 lclColInds_out[k] = new_linds[k];
1614 else if (I == GlobalIndices) {
1615 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1616 (
true, std::logic_error,
"The case where the input indices are local "
1617 "and the indices to write are global (lg=LocalIndices, I="
1618 "GlobalIndices) is not implemented, because it does not make sense."
1619 << std::endl <<
"If you have correct local column indices, that "
1620 "means the graph has a column Map. In that case, you should be "
1621 "storing local indices.");
1625 rowinfo.numEntries += numNewInds;
1626 this->k_numRowEntries_(rowinfo.localRow) += numNewInds;
1627 this->setLocallyModified ();
1630 const size_t chkNewNumEnt =
1631 this->getNumEntriesInLocalRow (rowinfo.localRow);
1632 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1633 (chkNewNumEnt != oldNumEnt + numNewInds, std::logic_error,
1634 "chkNewNumEnt = " << chkNewNumEnt
1635 <<
" != oldNumEnt (= " << oldNumEnt
1636 <<
") + numNewInds (= " << numNewInds <<
").");
1642 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1646 const GlobalOrdinal inputGblColInds[],
1647 const size_t numInputInds)
1649 return this->insertGlobalIndicesImpl (this->getRowInfo (lclRow),
1650 inputGblColInds, numInputInds);
1653 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1657 const GlobalOrdinal inputGblColInds[],
1658 const size_t numInputInds,
1659 std::function<
void(
const size_t,
const size_t,
const size_t)> fun)
1663 using Kokkos::subview;
1664 using Kokkos::MemoryUnmanaged;
1665 using Teuchos::ArrayView;
1666 using LO = LocalOrdinal;
1667 using GO = GlobalOrdinal;
1668 const char tfecfFuncName[] =
"insertGlobalIndicesImpl: ";
1669 const LO lclRow =
static_cast<LO
> (rowInfo.localRow);
1671 auto numEntries = rowInfo.numEntries;
1672 using inp_view_type = View<const GO*, Kokkos::HostSpace, MemoryUnmanaged>;
1673 inp_view_type inputInds(inputGblColInds, numInputInds);
1676 auto gblIndsHostView = this->gblInds_wdv.getHostView(Access::ReadWrite);
1679 numEntries, inputInds, fun);
1682 const bool insertFailed =
1683 numInserted == Teuchos::OrdinalTraits<size_t>::invalid();
1685 constexpr
size_t ONE (1);
1686 const int myRank = this->getComm()->getRank();
1687 std::ostringstream os;
1689 os <<
"Proc " << myRank <<
": Not enough capacity to insert "
1691 <<
" ind" << (numInputInds != ONE ?
"ices" :
"ex")
1692 <<
" into local row " << lclRow <<
", which currently has "
1693 << rowInfo.numEntries
1694 <<
" entr" << (rowInfo.numEntries != ONE ?
"ies" :
"y")
1695 <<
" and total allocation size " << rowInfo.allocSize
1697 const size_t maxNumToPrint =
1699 ArrayView<const GO> inputGblColIndsView(inputGblColInds,
1702 "column indices", maxNumToPrint);
1704 auto curGblColInds = getGlobalIndsViewHost(rowInfo);
1705 ArrayView<const GO> curGblColIndsView(curGblColInds.data(),
1706 rowInfo.numEntries);
1708 "column indices", maxNumToPrint);
1709 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1710 (
true, std::runtime_error, os.str());
1713 this->k_numRowEntries_(lclRow) += numInserted;
1715 this->setLocallyModified();
1720 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1724 const Teuchos::ArrayView<const LocalOrdinal>& indices,
1725 std::function<
void(
const size_t,
const size_t,
const size_t)> fun)
1727 using Kokkos::MemoryUnmanaged;
1728 using Kokkos::subview;
1730 using LO = LocalOrdinal;
1731 const char tfecfFuncName[] =
"insertLocallIndicesImpl: ";
1733 const RowInfo rowInfo = this->getRowInfo(myRow);
1735 size_t numNewInds = 0;
1736 size_t newNumEntries = 0;
1738 auto numEntries = rowInfo.numEntries;
1740 using inp_view_type = View<const LO*, Kokkos::HostSpace, MemoryUnmanaged>;
1741 inp_view_type inputInds(indices.getRawPtr(), indices.size());
1742 size_t numInserted = 0;
1744 auto lclInds = lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
1746 numEntries, inputInds, fun);
1749 const bool insertFailed =
1750 numInserted == Teuchos::OrdinalTraits<size_t>::invalid();
1752 constexpr
size_t ONE (1);
1753 const size_t numInputInds(indices.size());
1754 const int myRank = this->getComm()->getRank();
1755 std::ostringstream os;
1756 os <<
"On MPI Process " << myRank <<
": Not enough capacity to "
1757 "insert " << numInputInds
1758 <<
" ind" << (numInputInds != ONE ?
"ices" :
"ex")
1759 <<
" into local row " << myRow <<
", which currently has "
1760 << rowInfo.numEntries
1761 <<
" entr" << (rowInfo.numEntries != ONE ?
"ies" :
"y")
1762 <<
" and total allocation size " << rowInfo.allocSize <<
".";
1763 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1764 (
true, std::runtime_error, os.str());
1766 numNewInds = numInserted;
1767 newNumEntries = rowInfo.numEntries + numNewInds;
1769 this->k_numRowEntries_(myRow) += numNewInds;
1770 this->setLocallyModified ();
1773 const size_t chkNewNumEntries = this->getNumEntriesInLocalRow (myRow);
1774 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1775 (chkNewNumEntries != newNumEntries, std::logic_error,
1776 "getNumEntriesInLocalRow(" << myRow <<
") = " << chkNewNumEntries
1777 <<
" != newNumEntries = " << newNumEntries
1778 <<
". Please report this bug to the Tpetra developers.");
1782 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1786 const Teuchos::ArrayView<const GlobalOrdinal>& indices,
1787 std::function<
void(
const size_t,
const size_t,
const size_t)> fun)
const
1789 using GO = GlobalOrdinal;
1791 using Kokkos::MemoryUnmanaged;
1792 auto invalidCount = Teuchos::OrdinalTraits<size_t>::invalid();
1794 using inp_view_type = View<const GO*, Kokkos::HostSpace, MemoryUnmanaged>;
1795 inp_view_type inputInds(indices.getRawPtr(), indices.size());
1797 size_t numFound = 0;
1798 LocalOrdinal lclRow = rowInfo.localRow;
1799 if (this->isLocallyIndexed())
1801 if (this->colMap_.is_null())
1802 return invalidCount;
1803 const auto& colMap = *(this->colMap_);
1804 auto map = [&](GO
const gblInd){
return colMap.getLocalElement(gblInd);};
1807 lclIndsUnpacked_wdv.getHostView(Access::ReadOnly), inputInds, map, fun);
1809 else if (this->isGloballyIndexed())
1813 gblInds_wdv.getHostView(Access::ReadOnly), inputInds, fun);
1819 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1826 const size_t origNumEnt = rowInfo.numEntries;
1827 if (origNumEnt != Tpetra::Details::OrdinalTraits<size_t>::invalid () &&
1829 auto lclColInds = this->getLocalIndsViewHostNonConst (rowInfo);
1831 LocalOrdinal*
const lclColIndsRaw = lclColInds.data ();
1833 std::sort (lclColIndsRaw, lclColIndsRaw + origNumEnt);
1837 LocalOrdinal*
const beg = lclColIndsRaw;
1838 LocalOrdinal*
const end = beg + rowInfo.numEntries;
1839 LocalOrdinal*
const newend = std::unique (beg, end);
1840 const size_t newNumEnt = newend - beg;
1843 this->k_numRowEntries_(rowInfo.localRow) = newNumEnt;
1844 return origNumEnt - newNumEnt;
1847 return static_cast<size_t> (0);
1851 return static_cast<size_t> (0);
1856 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1860 const Teuchos::RCP<const map_type>& rangeMap)
1863 if (domainMap_ != domainMap) {
1864 domainMap_ = domainMap;
1865 importer_ = Teuchos::null;
1867 if (rangeMap_ != rangeMap) {
1868 rangeMap_ = rangeMap;
1869 exporter_ = Teuchos::null;
1874 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1879 const auto INV = Teuchos::OrdinalTraits<global_size_t>::invalid();
1881 globalNumEntries_ = INV;
1882 globalMaxNumRowEntries_ = INV;
1883 haveGlobalConstants_ =
false;
1887 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
1894 const char tfecfFuncName[] =
"checkInternalState: ";
1895 const char suffix[] =
" Please report this bug to the Tpetra developers.";
1897 std::unique_ptr<std::string> prefix;
1899 prefix = this->createPrefix(
"CrsGraph",
"checkInternalState");
1900 std::ostringstream os;
1901 os << *prefix <<
"Start" << endl;
1902 std::cerr << os.str();
1905 const global_size_t GSTI = Teuchos::OrdinalTraits<global_size_t>::invalid ();
1911 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1912 (this->rowMap_.is_null (), std::logic_error,
1913 "Row Map is null." << suffix);
1916 const LocalOrdinal lclNumRows =
1917 static_cast<LocalOrdinal
> (this->getLocalNumRows ());
1919 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1920 (this->isFillActive () == this->isFillComplete (), std::logic_error,
1921 "Graph cannot be both fill active and fill complete." << suffix);
1922 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1923 (this->isFillComplete () &&
1924 (this->colMap_.is_null () ||
1925 this->rangeMap_.is_null () ||
1926 this->domainMap_.is_null ()),
1928 "Graph is full complete, but at least one of {column, range, domain} "
1929 "Map is null." << suffix);
1930 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1931 (this->isStorageOptimized () && ! this->indicesAreAllocated (),
1932 std::logic_error,
"Storage is optimized, but indices are not "
1933 "allocated, not even trivially." << suffix);
1935 size_t nodeAllocSize = 0;
1937 nodeAllocSize = this->getLocalAllocationSize ();
1939 catch (std::logic_error& e) {
1940 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1941 (
true, std::runtime_error,
"getLocalAllocationSize threw "
1942 "std::logic_error: " << e.what ());
1944 catch (std::exception& e) {
1945 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1946 (
true, std::runtime_error,
"getLocalAllocationSize threw an "
1947 "std::exception: " << e.what ());
1950 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1951 (
true, std::runtime_error,
"getLocalAllocationSize threw an exception "
1952 "not a subclass of std::exception.");
1955 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1956 (this->isStorageOptimized () &&
1957 nodeAllocSize != this->getLocalNumEntries (),
1958 std::logic_error,
"Storage is optimized, but "
1959 "this->getLocalAllocationSize() = " << nodeAllocSize
1960 <<
" != this->getLocalNumEntries() = " << this->getLocalNumEntries ()
1962 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1963 (! this->haveGlobalConstants_ &&
1964 (this->globalNumEntries_ != GSTI ||
1965 this->globalMaxNumRowEntries_ != GSTI),
1966 std::logic_error,
"Graph claims not to have global constants, but "
1967 "some of the global constants are not marked as invalid." << suffix);
1968 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1969 (this->haveGlobalConstants_ &&
1970 (this->globalNumEntries_ == GSTI ||
1971 this->globalMaxNumRowEntries_ == GSTI),
1972 std::logic_error,
"Graph claims to have global constants, but "
1973 "some of them are marked as invalid." << suffix);
1974 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1975 (this->haveGlobalConstants_ &&
1976 (this->globalNumEntries_ < this->getLocalNumEntries () ||
1977 this->globalMaxNumRowEntries_ < this->nodeMaxNumRowEntries_),
1978 std::logic_error,
"Graph claims to have global constants, and "
1979 "all of the values of the global constants are valid, but "
1980 "some of the local constants are greater than "
1981 "their corresponding global constants." << suffix);
1982 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1983 (this->indicesAreAllocated () &&
1984 (this->numAllocForAllRows_ != 0 ||
1985 this->k_numAllocPerRow_.extent (0) != 0),
1986 std::logic_error,
"The graph claims that its indices are allocated, but "
1987 "either numAllocForAllRows_ (= " << this->numAllocForAllRows_ <<
") is "
1988 "nonzero, or k_numAllocPerRow_ has nonzero dimension. In other words, "
1989 "the graph is supposed to release its \"allocation specifications\" "
1990 "when it allocates its indices." << suffix);
1991 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1992 (rowPtrsUnpacked_host_.extent(0) != rowPtrsUnpacked_dev_.extent(0),
1993 std::logic_error,
"The host and device views of k_rowPtrs_ have "
1994 "different sizes; rowPtrsUnpacked_host_ has size "
1995 << rowPtrsUnpacked_host_.extent(0)
1996 <<
", but rowPtrsUnpacked_dev_ has size "
1997 << rowPtrsUnpacked_dev_.extent(0)
1999 if (isGloballyIndexed() && rowPtrsUnpacked_host_.extent(0) != 0) {
2000 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2001 (
size_t(rowPtrsUnpacked_host_.extent(0)) !=
size_t(lclNumRows + 1),
2002 std::logic_error,
"The graph is globally indexed and "
2003 "k_rowPtrs has nonzero size " << rowPtrsUnpacked_host_.extent(0)
2004 <<
", but that size does not equal lclNumRows+1 = "
2005 << (lclNumRows+1) <<
"." << suffix);
2006 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2007 (rowPtrsUnpacked_host_(lclNumRows) !=
size_t(gblInds_wdv.extent(0)),
2008 std::logic_error,
"The graph is globally indexed and "
2009 "k_rowPtrs_ has nonzero size " << rowPtrsUnpacked_host_.extent(0)
2010 <<
", but k_rowPtrs_(lclNumRows=" << lclNumRows <<
")="
2011 << rowPtrsUnpacked_host_(lclNumRows)
2012 <<
" != gblInds_wdv.extent(0)="
2013 << gblInds_wdv.extent(0) <<
"." << suffix);
2015 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2016 (this->isLocallyIndexed () &&
2017 this->rowPtrsUnpacked_host_.extent (0) != 0 &&
2018 (
static_cast<size_t> (rowPtrsUnpacked_host_.extent (0)) !=
2019 static_cast<size_t> (lclNumRows + 1) ||
2020 this->rowPtrsUnpacked_host_(lclNumRows) !=
2021 static_cast<size_t> (this->lclIndsUnpacked_wdv.extent (0))),
2022 std::logic_error,
"If k_rowPtrs_ has nonzero size and "
2023 "the graph is locally indexed, then "
2024 "k_rowPtrs_ must have N+1 rows, and "
2025 "k_rowPtrs_(N) must equal lclIndsUnpacked_wdv.extent(0)." << suffix);
2027 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2028 (this->indicesAreAllocated () &&
2029 nodeAllocSize > 0 &&
2030 this->lclIndsUnpacked_wdv.extent (0) == 0 &&
2031 this->gblInds_wdv.extent (0) == 0,
2032 std::logic_error,
"Graph is allocated nontrivially, but "
2033 "but 1-D allocations are not present." << suffix);
2035 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2036 (! this->indicesAreAllocated () &&
2037 ((this->rowPtrsUnpacked_host_.extent (0) != 0 ||
2038 this->k_numRowEntries_.extent (0) != 0) ||
2039 this->lclIndsUnpacked_wdv.extent (0) != 0 ||
2040 this->gblInds_wdv.extent (0) != 0),
2041 std::logic_error,
"If indices are not allocated, "
2042 "then none of the buffers should be." << suffix);
2046 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2047 ((this->indicesAreLocal_ || this->indicesAreGlobal_) &&
2048 ! this->indicesAreAllocated_,
2049 std::logic_error,
"Indices may be local or global only if they are "
2050 "allocated." << suffix);
2051 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2052 (this->indicesAreLocal_ && this->indicesAreGlobal_,
2053 std::logic_error,
"Indices may not be both local and global." << suffix);
2054 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2055 (indicesAreLocal_ && gblInds_wdv.extent (0) != 0,
2056 std::logic_error,
"Indices are local, but "
2057 "gblInds_wdv.extent(0) (= " << gblInds_wdv.extent (0)
2058 <<
") != 0. In other words, if indices are local, then "
2059 "allocations of global indices should not be present."
2061 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2062 (indicesAreGlobal_ && lclIndsUnpacked_wdv.extent (0) != 0,
2063 std::logic_error,
"Indices are global, but "
2064 "lclIndsUnpacked_wdv.extent(0) (= " << lclIndsUnpacked_wdv.extent(0)
2065 <<
") != 0. In other words, if indices are global, "
2066 "then allocations for local indices should not be present."
2068 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2069 (indicesAreLocal_ && nodeAllocSize > 0 &&
2070 lclIndsUnpacked_wdv.extent (0) == 0 && getLocalNumRows () > 0,
2071 std::logic_error,
"Indices are local and "
2072 "getLocalAllocationSize() = " << nodeAllocSize <<
" > 0, but "
2073 "lclIndsUnpacked_wdv.extent(0) = 0 and getLocalNumRows() = "
2074 << getLocalNumRows () <<
" > 0." << suffix);
2075 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2076 (indicesAreGlobal_ && nodeAllocSize > 0 &&
2077 gblInds_wdv.extent (0) == 0 && getLocalNumRows () > 0,
2078 std::logic_error,
"Indices are global and "
2079 "getLocalAllocationSize() = " << nodeAllocSize <<
" > 0, but "
2080 "gblInds_wdv.extent(0) = 0 and getLocalNumRows() = "
2081 << getLocalNumRows () <<
" > 0." << suffix);
2083 if (this->indicesAreAllocated () &&
2084 this->rowPtrsUnpacked_host_.extent (0) != 0) {
2085 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2086 (static_cast<size_t> (this->rowPtrsUnpacked_host_.extent (0)) !=
2087 this->getLocalNumRows () + 1,
2088 std::logic_error,
"Indices are allocated and "
2089 "k_rowPtrs_ has nonzero length, but rowPtrsUnpacked_host_.extent(0) = "
2090 << this->rowPtrsUnpacked_host_.extent (0) <<
" != getLocalNumRows()+1 = "
2091 << (this->getLocalNumRows () + 1) <<
"." << suffix);
2092 const size_t actualNumAllocated =
2093 this->rowPtrsUnpacked_host_(this->getLocalNumRows());
2094 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2095 (this->isLocallyIndexed () &&
2096 static_cast<size_t> (this->lclIndsUnpacked_wdv.extent (0)) != actualNumAllocated,
2097 std::logic_error,
"Graph is locally indexed, indices are "
2098 "are allocated, and k_rowPtrs_ has nonzero length, but "
2099 "lclIndsUnpacked_wdv.extent(0) = " << this->lclIndsUnpacked_wdv.extent (0)
2100 <<
" != actualNumAllocated = " << actualNumAllocated << suffix);
2101 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2102 (this->isGloballyIndexed () &&
2103 static_cast<size_t> (this->gblInds_wdv.extent (0)) != actualNumAllocated,
2104 std::logic_error,
"Graph is globally indexed, indices "
2105 "are allocated, and k_rowPtrs_ has nonzero length, but "
2106 "gblInds_wdv.extent(0) = " << this->gblInds_wdv.extent (0)
2107 <<
" != actualNumAllocated = " << actualNumAllocated << suffix);
2111 std::ostringstream os;
2112 os << *prefix <<
"Done" << endl;
2113 std::cerr << os.str();
2119 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2124 const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex (globalRow);
2125 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid ()) {
2126 return Teuchos::OrdinalTraits<size_t>::invalid ();
2129 return rowInfo.numEntries;
2134 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2139 const RowInfo rowInfo = this->getRowInfo (localRow);
2140 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid ()) {
2141 return Teuchos::OrdinalTraits<size_t>::invalid ();
2144 return rowInfo.numEntries;
2149 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2154 const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex (globalRow);
2155 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid ()) {
2156 return Teuchos::OrdinalTraits<size_t>::invalid ();
2159 return rowInfo.allocSize;
2164 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2169 const RowInfo rowInfo = this->getRowInfo (localRow);
2170 if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid ()) {
2171 return Teuchos::OrdinalTraits<size_t>::invalid ();
2174 return rowInfo.allocSize;
2179 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2180 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::row_ptrs_host_view_type
2184 return rowPtrsPacked_host_;
2187 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2188 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::row_ptrs_device_view_type
2192 return rowPtrsPacked_dev_;
2196 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2197 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::local_inds_host_view_type
2201 return lclIndsPacked_wdv.getHostView(Access::ReadOnly);
2204 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2209 return lclIndsPacked_wdv.getDeviceView(Access::ReadOnly);
2212 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2216 nonconst_local_inds_host_view_type & indices,
2217 size_t& numEntries)
const
2219 using Teuchos::ArrayView;
2220 const char tfecfFuncName[] =
"getLocalRowCopy: ";
2222 TEUCHOS_TEST_FOR_EXCEPTION(
2223 isGloballyIndexed () && ! hasColMap (), std::runtime_error,
2224 "Tpetra::CrsGraph::getLocalRowCopy: The graph is globally indexed and "
2225 "does not have a column Map yet. That means we don't have local indices "
2226 "for columns yet, so it doesn't make sense to call this method. If the "
2227 "graph doesn't have a column Map yet, you should call fillComplete on "
2232 const RowInfo rowinfo = this->getRowInfo (localRow);
2234 const size_t theNumEntries = rowinfo.numEntries;
2235 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2236 (static_cast<size_t> (indices.size ()) < theNumEntries,std::runtime_error,
2237 "Specified storage (size==" << indices.size () <<
") does not suffice "
2238 "to hold all " << theNumEntries <<
" entry/ies for this row.");
2239 numEntries = theNumEntries;
2241 if (rowinfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid ()) {
2242 if (isLocallyIndexed ()) {
2243 auto lclInds = getLocalIndsViewHost(rowinfo);
2244 for (
size_t j = 0; j < theNumEntries; ++j) {
2245 indices[j] = lclInds(j);
2248 else if (isGloballyIndexed ()) {
2249 auto gblInds = getGlobalIndsViewHost(rowinfo);
2250 for (
size_t j = 0; j < theNumEntries; ++j) {
2251 indices[j] = colMap_->getLocalElement (gblInds(j));
2258 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2262 nonconst_global_inds_host_view_type &indices,
2263 size_t& numEntries)
const
2265 using Teuchos::ArrayView;
2266 const char tfecfFuncName[] =
"getGlobalRowCopy: ";
2270 const RowInfo rowinfo = getRowInfoFromGlobalRowIndex (globalRow);
2271 const size_t theNumEntries = rowinfo.numEntries;
2272 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2273 static_cast<size_t> (indices.size ()) < theNumEntries, std::runtime_error,
2274 "Specified storage (size==" << indices.size () <<
") does not suffice "
2275 "to hold all " << theNumEntries <<
" entry/ies for this row.");
2276 numEntries = theNumEntries;
2278 if (rowinfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid ()) {
2279 if (isLocallyIndexed ()) {
2280 auto lclInds = getLocalIndsViewHost(rowinfo);
2281 for (
size_t j = 0; j < theNumEntries; ++j) {
2282 indices[j] = colMap_->getGlobalElement (lclInds(j));
2285 else if (isGloballyIndexed ()) {
2286 auto gblInds = getGlobalIndsViewHost(rowinfo);
2287 for (
size_t j = 0; j < theNumEntries; ++j) {
2288 indices[j] = gblInds(j);
2295 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2299 const LocalOrdinal localRow,
2300 local_inds_host_view_type &indices)
const
2302 const char tfecfFuncName[] =
"getLocalRowView: ";
2304 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2305 (isGloballyIndexed (), std::runtime_error,
"The graph's indices are "
2306 "currently stored as global indices, so we cannot return a view with "
2307 "local column indices, whether or not the graph has a column Map. If "
2308 "the graph _does_ have a column Map, use getLocalRowCopy() instead.");
2310 const RowInfo rowInfo = getRowInfo (localRow);
2311 if (rowInfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid () &&
2312 rowInfo.numEntries > 0) {
2313 indices = lclIndsUnpacked_wdv.getHostSubview(rowInfo.offset1D,
2320 indices = local_inds_host_view_type();
2324 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2325 (static_cast<size_t> (indices.size ()) !=
2326 getNumEntriesInLocalRow (localRow), std::logic_error,
"indices.size() "
2327 "= " << indices.extent(0) <<
" != getNumEntriesInLocalRow(localRow=" <<
2328 localRow <<
") = " << getNumEntriesInLocalRow(localRow) <<
2329 ". Please report this bug to the Tpetra developers.");
2334 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2338 const GlobalOrdinal globalRow,
2339 global_inds_host_view_type &indices)
const
2341 const char tfecfFuncName[] =
"getGlobalRowView: ";
2343 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2344 (isLocallyIndexed (), std::runtime_error,
"The graph's indices are "
2345 "currently stored as local indices, so we cannot return a view with "
2346 "global column indices. Use getGlobalRowCopy() instead.");
2350 const RowInfo rowInfo = getRowInfoFromGlobalRowIndex (globalRow);
2351 if (rowInfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid () &&
2352 rowInfo.numEntries > 0) {
2353 indices = gblInds_wdv.getHostSubview(rowInfo.offset1D,
2358 indices =
typename global_inds_dualv_type::t_host::const_type();
2361 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2362 (static_cast<size_t> (indices.size ()) !=
2363 getNumEntriesInGlobalRow (globalRow),
2364 std::logic_error,
"indices.size() = " << indices.extent(0)
2365 <<
" != getNumEntriesInGlobalRow(globalRow=" << globalRow <<
") = "
2366 << getNumEntriesInGlobalRow (globalRow)
2367 <<
". Please report this bug to the Tpetra developers.");
2372 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2376 const Teuchos::ArrayView<const LocalOrdinal>& indices)
2378 const char tfecfFuncName[] =
"insertLocalIndices: ";
2380 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2381 (! isFillActive (), std::runtime_error,
"Fill must be active.");
2382 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2383 (isGloballyIndexed (), std::runtime_error,
2384 "Graph indices are global; use insertGlobalIndices().");
2385 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2386 (! hasColMap (), std::runtime_error,
2387 "Cannot insert local indices without a column Map.");
2388 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2389 (! rowMap_->isNodeLocalElement (localRow), std::runtime_error,
2390 "Local row index " << localRow <<
" is not in the row Map "
2391 "on the calling process.");
2392 if (! indicesAreAllocated ()) {
2393 allocateIndices (LocalIndices, verbose_);
2402 using Teuchos::Array;
2403 using Teuchos::toString;
2405 typedef typename Teuchos::ArrayView<const LocalOrdinal>::size_type size_type;
2408 Array<LocalOrdinal> badColInds;
2409 bool allInColMap =
true;
2410 for (size_type k = 0; k < indices.size (); ++k) {
2412 allInColMap =
false;
2413 badColInds.push_back (indices[k]);
2416 if (! allInColMap) {
2417 std::ostringstream os;
2418 os <<
"Tpetra::CrsGraph::insertLocalIndices: You attempted to insert "
2419 "entries in owned row " << localRow <<
", at the following column "
2420 "indices: " << toString (indices) <<
"." << endl;
2421 os <<
"Of those, the following indices are not in the column Map on "
2422 "this process: " << toString (badColInds) <<
"." << endl <<
"Since "
2423 "the graph has a column Map already, it is invalid to insert entries "
2424 "at those locations.";
2425 TEUCHOS_TEST_FOR_EXCEPTION(! allInColMap, std::invalid_argument, os.str ());
2430 insertLocalIndicesImpl (localRow, indices);
2433 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2434 (! indicesAreAllocated () || ! isLocallyIndexed (), std::logic_error,
2435 "At the end of insertLocalIndices, ! indicesAreAllocated() || "
2436 "! isLocallyIndexed() is true. Please report this bug to the "
2437 "Tpetra developers.");
2441 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2445 const LocalOrdinal numEnt,
2446 const LocalOrdinal inds[])
2448 Teuchos::ArrayView<const LocalOrdinal> indsT (inds, numEnt);
2449 this->insertLocalIndices (localRow, indsT);
2453 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2457 const LocalOrdinal numInputInds,
2458 const GlobalOrdinal inputGblColInds[])
2460 typedef LocalOrdinal LO;
2461 const char tfecfFuncName[] =
"insertGlobalIndices: ";
2463 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2464 (this->isLocallyIndexed (), std::runtime_error,
2465 "graph indices are local; use insertLocalIndices().");
2470 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2471 (! this->isFillActive (), std::runtime_error,
2472 "You are not allowed to call this method if fill is not active. "
2473 "If fillComplete has been called, you must first call resumeFill "
2474 "before you may insert indices.");
2475 if (! indicesAreAllocated ()) {
2476 allocateIndices (GlobalIndices, verbose_);
2478 const LO lclRow = this->rowMap_->getLocalElement (gblRow);
2479 if (lclRow != Tpetra::Details::OrdinalTraits<LO>::invalid ()) {
2481 if (this->hasColMap ()) {
2483 const map_type& colMap = * (this->colMap_);
2488 std::vector<GlobalOrdinal> badColInds;
2489 bool allInColMap =
true;
2490 for (LO k = 0; k < numInputInds; ++k) {
2492 allInColMap =
false;
2493 badColInds.push_back (inputGblColInds[k]);
2496 if (! allInColMap) {
2497 std::ostringstream os;
2498 os <<
"You attempted to insert entries in owned row " << gblRow
2499 <<
", at the following column indices: [";
2500 for (LO k = 0; k < numInputInds; ++k) {
2501 os << inputGblColInds[k];
2502 if (k + static_cast<LO> (1) < numInputInds) {
2506 os <<
"]." << endl <<
"Of those, the following indices are not in "
2507 "the column Map on this process: [";
2508 for (
size_t k = 0; k < badColInds.size (); ++k) {
2509 os << badColInds[k];
2510 if (k +
size_t (1) < badColInds.size ()) {
2514 os <<
"]." << endl <<
"Since the matrix has a column Map already, "
2515 "it is invalid to insert entries at those locations.";
2516 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2517 (
true, std::invalid_argument, os.str ());
2521 this->insertGlobalIndicesImpl (lclRow, inputGblColInds, numInputInds);
2524 this->insertGlobalIndicesIntoNonownedRows (gblRow, inputGblColInds,
2530 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2534 const Teuchos::ArrayView<const GlobalOrdinal>& inputGblColInds)
2536 this->insertGlobalIndices (gblRow, inputGblColInds.size (),
2537 inputGblColInds.getRawPtr ());
2541 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2545 const GlobalOrdinal gblColInds[],
2546 const LocalOrdinal numGblColInds)
2548 typedef LocalOrdinal LO;
2549 typedef GlobalOrdinal GO;
2550 const char tfecfFuncName[] =
"insertGlobalIndicesFiltered: ";
2552 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2553 (this->isLocallyIndexed (), std::runtime_error,
2554 "Graph indices are local; use insertLocalIndices().");
2559 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2560 (! this->isFillActive (), std::runtime_error,
2561 "You are not allowed to call this method if fill is not active. "
2562 "If fillComplete has been called, you must first call resumeFill "
2563 "before you may insert indices.");
2564 if (! indicesAreAllocated ()) {
2565 allocateIndices (GlobalIndices, verbose_);
2568 Teuchos::ArrayView<const GO> gblColInds_av (gblColInds, numGblColInds);
2570 if (! colMap_.is_null ()) {
2571 const map_type& colMap = * (this->colMap_);
2574 while (curOffset < numGblColInds) {
2578 LO endOffset = curOffset;
2579 for ( ; endOffset < numGblColInds; ++endOffset) {
2581 if (lclCol == Tpetra::Details::OrdinalTraits<LO>::invalid ()) {
2588 const LO numIndInSeq = (endOffset - curOffset);
2589 if (numIndInSeq != 0) {
2590 this->insertGlobalIndicesImpl (lclRow, gblColInds + curOffset,
2596 curOffset = endOffset + 1;
2600 this->insertGlobalIndicesImpl (lclRow, gblColInds_av.getRawPtr (),
2601 gblColInds_av.size ());
2605 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2609 const GlobalOrdinal gblColInds[],
2610 const LocalOrdinal numGblColInds)
2615 std::vector<GlobalOrdinal>& nonlocalRow = this->nonlocals_[gblRow];
2616 for (LocalOrdinal k = 0; k < numGblColInds; ++k) {
2620 nonlocalRow.push_back (gblColInds[k]);
2624 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2629 const char tfecfFuncName[] =
"removeLocalIndices: ";
2630 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2631 ! isFillActive (), std::runtime_error,
"requires that fill is active.");
2632 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2633 isStorageOptimized (), std::runtime_error,
2634 "cannot remove indices after optimizeStorage() has been called.");
2635 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2636 isGloballyIndexed (), std::runtime_error,
"graph indices are global.");
2637 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2638 ! rowMap_->isNodeLocalElement (lrow), std::runtime_error,
2639 "Local row " << lrow <<
" is not in the row Map on the calling process.");
2640 if (! indicesAreAllocated ()) {
2641 allocateIndices (LocalIndices, verbose_);
2644 if (k_numRowEntries_.extent (0) != 0) {
2645 this->k_numRowEntries_(lrow) = 0;
2649 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2650 (getNumEntriesInLocalRow (lrow) != 0 ||
2651 ! indicesAreAllocated () ||
2652 ! isLocallyIndexed (), std::logic_error,
2653 "Violated stated post-conditions. Please contact Tpetra team.");
2658 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2661 setAllIndices (
const typename local_graph_device_type::row_map_type& rowPointers,
2662 const typename local_graph_device_type::entries_type::non_const_type& columnIndices)
2665 ProfilingRegion region (
"Tpetra::CrsGraph::setAllIndices");
2666 const char tfecfFuncName[] =
"setAllIndices: ";
2667 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2668 ! hasColMap () || getColMap ().is_null (), std::runtime_error,
2669 "The graph must have a column Map before you may call this method.");
2670 LocalOrdinal numLocalRows = this->getLocalNumRows ();
2672 LocalOrdinal rowPtrLen = rowPointers.size();
2673 if(numLocalRows == 0) {
2674 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2675 rowPtrLen != 0 && rowPtrLen != 1,
2676 std::runtime_error,
"Have 0 local rows, but rowPointers.size() is neither 0 nor 1.");
2679 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
2680 rowPtrLen != numLocalRows + 1,
2681 std::runtime_error,
"rowPointers.size() = " << rowPtrLen <<
2682 " != this->getLocalNumRows()+1 = " << (numLocalRows + 1) <<
".");
2687 using exec_space =
typename local_graph_device_type::execution_space;
2688 int columnsOutOfBounds = 0;
2690 Kokkos::parallel_reduce(Kokkos::RangePolicy<exec_space>(0, columnIndices.extent(0)),
2691 KOKKOS_LAMBDA (
const LocalOrdinal i,
int& lOutOfBounds)
2693 if(columnIndices(i) < 0 || columnIndices(i) >= numLocalCols)
2695 }, columnsOutOfBounds);
2696 int globalColsOutOfBounds= 0;
2697 auto comm = this->getComm();
2698 Teuchos::reduceAll<int, int> (*comm, Teuchos::REDUCE_MAX, columnsOutOfBounds,
2699 Teuchos::outArg (globalColsOutOfBounds));
2700 if (globalColsOutOfBounds)
2702 std::string message;
2703 if (columnsOutOfBounds)
2706 message = std::string(
"ERROR, rank ") + std::to_string(comm->getRank()) +
", CrsGraph::setAllIndices(): provided columnIndices are not all within range [0, getLocalNumCols())!\n";
2709 throw std::invalid_argument(
"CrsGraph::setAllIndices(): columnIndices are out of the valid range on at least one process.");
2713 if (debug_ && this->isSorted()) {
2716 using exec_space =
typename local_graph_device_type::execution_space;
2717 using size_type =
typename local_graph_device_type::size_type;
2718 Kokkos::parallel_reduce(Kokkos::RangePolicy<exec_space>(0, numLocalRows),
2719 KOKKOS_LAMBDA (
const LocalOrdinal i,
int& lNotSorted)
2721 size_type rowBegin = rowPointers(i);
2722 size_type rowEnd = rowPointers(i + 1);
2723 for(size_type j = rowBegin + 1; j < rowEnd; j++)
2725 if(columnIndices(j - 1) > columnIndices(j))
2732 int globalNotSorted = 0;
2733 auto comm = this->getComm();
2734 Teuchos::reduceAll<int, int> (*comm, Teuchos::REDUCE_MAX, notSorted,
2735 Teuchos::outArg (globalNotSorted));
2736 if (globalNotSorted)
2738 std::string message;
2742 message = std::string(
"ERROR, rank ") + std::to_string(comm->getRank()) +
", CrsGraph::setAllIndices(): provided columnIndices are not sorted!\n";
2745 throw std::invalid_argument(
"CrsGraph::setAllIndices(): provided columnIndices are not sorted within rows on at least one process.");
2749 indicesAreAllocated_ =
true;
2750 indicesAreLocal_ =
true;
2751 indicesAreSorted_ =
true;
2752 noRedundancies_ =
true;
2753 lclIndsPacked_wdv= local_inds_wdv_type(columnIndices);
2754 lclIndsUnpacked_wdv = lclIndsPacked_wdv;
2755 setRowPtrs(rowPointers);
2757 set_need_sync_host_uvm_access();
2761 storageStatus_ = Details::STORAGE_1D_PACKED;
2766 numAllocForAllRows_ = 0;
2767 k_numAllocPerRow_ = decltype (k_numAllocPerRow_) ();
2769 checkInternalState ();
2773 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2777 const Teuchos::ArrayRCP<LocalOrdinal>& columnIndices)
2780 typedef typename local_graph_device_type::row_map_type row_map_type;
2781 typedef typename row_map_type::array_layout layout_type;
2782 typedef typename row_map_type::non_const_value_type row_offset_type;
2783 typedef View<
size_t*, layout_type , Kokkos::HostSpace,
2784 Kokkos::MemoryUnmanaged> input_view_type;
2785 typedef typename row_map_type::non_const_type nc_row_map_type;
2787 const size_t size =
static_cast<size_t> (rowPointers.size ());
2788 constexpr
bool same = std::is_same<size_t, row_offset_type>::value;
2789 input_view_type ptr_in (rowPointers.getRawPtr (), size);
2791 nc_row_map_type ptr_rot (
"Tpetra::CrsGraph::ptr", size);
2794 #ifdef KOKKOS_ENABLE_CXX17
2795 if constexpr (same) {
2796 using lexecution_space =
typename device_type::execution_space;
2806 input_view_type ptr_decoy (rowPointers.getRawPtr (), size);
2809 Kokkos::Impl::if_c<same,
2811 input_view_type>::select (ptr_rot, ptr_decoy),
2817 constexpr
bool inHostMemory =
2818 std::is_same<
typename row_map_type::memory_space,
2819 Kokkos::HostSpace>::value;
2830 View<size_t*, layout_type, device_type> ptr_st (
"Tpetra::CrsGraph::ptr", size);
2842 Kokkos::View<LocalOrdinal*, layout_type, device_type> k_ind =
2843 Kokkos::Compat::getKokkosViewDeepCopy<device_type> (columnIndices ());
2844 setAllIndices (ptr_rot, k_ind);
2848 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
2853 using Teuchos::Comm;
2854 using Teuchos::outArg;
2857 using Teuchos::REDUCE_MAX;
2858 using Teuchos::REDUCE_MIN;
2859 using Teuchos::reduceAll;
2864 using size_type =
typename Teuchos::Array<GO>::size_type;
2865 const char tfecfFuncName[] =
"globalAssemble: ";
2867 std::unique_ptr<std::string> prefix;
2869 prefix = this->createPrefix(
"CrsGraph",
"globalAssemble");
2870 std::ostringstream os;
2871 os << *prefix <<
"Start" << endl;
2872 std::cerr << os.str();
2874 RCP<const Comm<int> > comm = getComm ();
2876 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2877 (! isFillActive (), std::runtime_error,
"Fill must be active before "
2878 "you may call this method.");
2880 const size_t myNumNonlocalRows = this->nonlocals_.size ();
2887 const int iHaveNonlocalRows = (myNumNonlocalRows == 0) ? 0 : 1;
2888 int someoneHasNonlocalRows = 0;
2889 reduceAll<int, int> (*comm, REDUCE_MAX, iHaveNonlocalRows,
2890 outArg (someoneHasNonlocalRows));
2891 if (someoneHasNonlocalRows == 0) {
2893 std::ostringstream os;
2894 os << *prefix <<
"Done: No nonlocal rows" << endl;
2895 std::cerr << os.str();
2899 else if (verbose_) {
2900 std::ostringstream os;
2901 os << *prefix <<
"At least 1 process has nonlocal rows"
2903 std::cerr << os.str();
2916 RCP<const map_type> nonlocalRowMap;
2918 Teuchos::Array<size_t> numEntPerNonlocalRow (myNumNonlocalRows);
2920 Teuchos::Array<GO> myNonlocalGblRows (myNumNonlocalRows);
2921 size_type curPos = 0;
2922 for (
auto mapIter = this->nonlocals_.begin ();
2923 mapIter != this->nonlocals_.end ();
2924 ++mapIter, ++curPos) {
2925 myNonlocalGblRows[curPos] = mapIter->first;
2926 std::vector<GO>& gblCols = mapIter->second;
2927 std::sort (gblCols.begin (), gblCols.end ());
2928 auto vecLast = std::unique (gblCols.begin (), gblCols.end ());
2929 gblCols.erase (vecLast, gblCols.end ());
2930 numEntPerNonlocalRow[curPos] = gblCols.size ();
2941 GO myMinNonlocalGblRow = std::numeric_limits<GO>::max ();
2943 auto iter = std::min_element (myNonlocalGblRows.begin (),
2944 myNonlocalGblRows.end ());
2945 if (iter != myNonlocalGblRows.end ()) {
2946 myMinNonlocalGblRow = *iter;
2949 GO gblMinNonlocalGblRow = 0;
2950 reduceAll<int, GO> (*comm, REDUCE_MIN, myMinNonlocalGblRow,
2951 outArg (gblMinNonlocalGblRow));
2952 const GO indexBase = gblMinNonlocalGblRow;
2953 const global_size_t INV = Teuchos::OrdinalTraits<global_size_t>::invalid ();
2954 nonlocalRowMap = rcp (
new map_type (INV, myNonlocalGblRows (), indexBase, comm));
2958 std::ostringstream os;
2959 os << *prefix <<
"nonlocalRowMap->getIndexBase()="
2960 << nonlocalRowMap->getIndexBase() << endl;
2961 std::cerr << os.str();
2969 RCP<crs_graph_type> nonlocalGraph =
2970 rcp(
new crs_graph_type(nonlocalRowMap, numEntPerNonlocalRow()));
2972 size_type curPos = 0;
2973 for (
auto mapIter = this->nonlocals_.begin ();
2974 mapIter != this->nonlocals_.end ();
2975 ++mapIter, ++curPos) {
2976 const GO gblRow = mapIter->first;
2977 std::vector<GO>& gblCols = mapIter->second;
2978 const LO numEnt =
static_cast<LO
> (numEntPerNonlocalRow[curPos]);
2979 nonlocalGraph->insertGlobalIndices (gblRow, numEnt, gblCols.data ());
2983 std::ostringstream os;
2984 os << *prefix <<
"Built nonlocal graph" << endl;
2985 std::cerr << os.str();
2996 auto origRowMap = this->getRowMap ();
2997 const bool origRowMapIsOneToOne = origRowMap->isOneToOne ();
2999 if (origRowMapIsOneToOne) {
3001 std::ostringstream os;
3002 os << *prefix <<
"Original row Map is 1-to-1" << endl;
3003 std::cerr << os.str();
3005 export_type exportToOrig (nonlocalRowMap, origRowMap);
3011 std::ostringstream os;
3012 os << *prefix <<
"Original row Map is NOT 1-to-1" << endl;
3013 std::cerr << os.str();
3020 export_type exportToOneToOne (nonlocalRowMap, oneToOneRowMap);
3026 crs_graph_type oneToOneGraph (oneToOneRowMap, 0);
3030 std::ostringstream os;
3031 os << *prefix <<
"Export nonlocal graph" << endl;
3032 std::cerr << os.str();
3034 oneToOneGraph.doExport (*nonlocalGraph, exportToOneToOne,
Tpetra::INSERT);
3038 nonlocalGraph = Teuchos::null;
3041 import_type importToOrig (oneToOneRowMap, origRowMap);
3043 std::ostringstream os;
3044 os << *prefix <<
"Import nonlocal graph" << endl;
3045 std::cerr << os.str();
3054 decltype (this->nonlocals_) newNonlocals;
3055 std::swap (this->nonlocals_, newNonlocals);
3057 checkInternalState ();
3059 std::ostringstream os;
3060 os << *prefix <<
"Done" << endl;
3061 std::cerr << os.str();
3066 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3071 clearGlobalConstants();
3072 if (params != Teuchos::null) this->setParameterList (params);
3074 indicesAreSorted_ =
true;
3075 noRedundancies_ =
true;
3076 fillComplete_ =
false;
3080 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3095 Teuchos::RCP<const map_type> domMap = this->getDomainMap ();
3096 if (domMap.is_null ()) {
3097 domMap = this->getRowMap ();
3099 Teuchos::RCP<const map_type> ranMap = this->getRangeMap ();
3100 if (ranMap.is_null ()) {
3101 ranMap = this->getRowMap ();
3103 this->fillComplete (domMap, ranMap, params);
3107 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3111 const Teuchos::RCP<const map_type>& rangeMap,
3112 const Teuchos::RCP<Teuchos::ParameterList>& params)
3115 const char tfecfFuncName[] =
"fillComplete: ";
3116 const bool verbose = verbose_;
3118 std::unique_ptr<std::string> prefix;
3120 prefix = this->createPrefix(
"CrsGraph",
"fillComplete");
3121 std::ostringstream os;
3122 os << *prefix <<
"Start" << endl;
3123 std::cerr << os.str();
3126 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3127 (! isFillActive () || isFillComplete (), std::runtime_error,
3128 "Graph fill state must be active (isFillActive() "
3129 "must be true) before calling fillComplete().");
3131 const int numProcs = getComm ()->getSize ();
3139 if (! params.is_null ()) {
3140 if (params->isParameter (
"sort column map ghost gids")) {
3141 sortGhostsAssociatedWithEachProcessor_ =
3142 params->get<
bool> (
"sort column map ghost gids",
3143 sortGhostsAssociatedWithEachProcessor_);
3145 else if (params->isParameter (
"Sort column Map ghost GIDs")) {
3146 sortGhostsAssociatedWithEachProcessor_ =
3147 params->get<
bool> (
"Sort column Map ghost GIDs",
3148 sortGhostsAssociatedWithEachProcessor_);
3154 bool assertNoNonlocalInserts =
false;
3155 if (! params.is_null ()) {
3156 assertNoNonlocalInserts =
3157 params->get<
bool> (
"No Nonlocal Changes", assertNoNonlocalInserts);
3163 if (! indicesAreAllocated ()) {
3166 allocateIndices (LocalIndices, verbose);
3169 allocateIndices (GlobalIndices, verbose);
3177 const bool mayNeedGlobalAssemble = ! assertNoNonlocalInserts && numProcs > 1;
3178 if (mayNeedGlobalAssemble) {
3184 const size_t numNonlocals = nonlocals_.size();
3186 std::ostringstream os;
3187 os << *prefix <<
"Do not need to call globalAssemble; "
3188 "assertNoNonlocalInserts="
3189 << (assertNoNonlocalInserts ?
"true" :
"false")
3190 <<
"numProcs=" << numProcs
3191 <<
", nonlocals_.size()=" << numNonlocals << endl;
3192 std::cerr << os.str();
3194 const int lclNeededGlobalAssemble =
3195 (numProcs > 1 && numNonlocals != 0) ? 1 : 0;
3196 if (lclNeededGlobalAssemble != 0 && verbose) {
3197 std::ostringstream os;
3199 Details::Impl::verbosePrintMap(
3200 os, nonlocals_.begin(), nonlocals_.end(),
3201 nonlocals_.size(),
"nonlocals_");
3202 std::cerr << os.str() << endl;
3206 auto map = this->getMap();
3207 auto comm = map.is_null() ? Teuchos::null : map->getComm();
3208 int gblNeededGlobalAssemble = lclNeededGlobalAssemble;
3209 if (! comm.is_null()) {
3210 using Teuchos::REDUCE_MAX;
3211 using Teuchos::reduceAll;
3212 reduceAll(*comm, REDUCE_MAX, lclNeededGlobalAssemble,
3213 Teuchos::outArg(gblNeededGlobalAssemble));
3215 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3216 (gblNeededGlobalAssemble != 0, std::runtime_error,
3217 "nonlocals_.size()=" << numNonlocals <<
" != 0 on at "
3218 "least one process in the CrsGraph's communicator. This "
3219 "means either that you incorrectly set the "
3220 "\"No Nonlocal Changes\" fillComplete parameter to true, "
3221 "or that you inserted invalid entries. "
3222 "Rerun with the environment variable TPETRA_VERBOSE="
3223 "CrsGraph set to see the entries of nonlocals_ on every "
3224 "MPI process (WARNING: lots of output).");
3227 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3228 (lclNeededGlobalAssemble != 0, std::runtime_error,
3229 "nonlocals_.size()=" << numNonlocals <<
" != 0 on the "
3230 "calling process. This means either that you incorrectly "
3231 "set the \"No Nonlocal Changes\" fillComplete parameter "
3232 "to true, or that you inserted invalid entries. "
3233 "Rerun with the environment "
3234 "variable TPETRA_VERBOSE=CrsGraph set to see the entries "
3235 "of nonlocals_ on every MPI process (WARNING: lots of "
3242 setDomainRangeMaps (domainMap, rangeMap);
3248 Teuchos::Array<int> remotePIDs (0);
3249 const bool mustBuildColMap = ! this->hasColMap ();
3250 if (mustBuildColMap) {
3256 const std::pair<size_t, std::string> makeIndicesLocalResult =
3257 this->makeIndicesLocal(verbose);
3262 using Teuchos::REDUCE_MIN;
3263 using Teuchos::reduceAll;
3264 using Teuchos::outArg;
3266 RCP<const map_type> map = this->getMap ();
3267 RCP<const Teuchos::Comm<int> > comm;
3268 if (! map.is_null ()) {
3269 comm = map->getComm ();
3271 if (comm.is_null ()) {
3272 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3273 (makeIndicesLocalResult.first != 0, std::runtime_error,
3274 makeIndicesLocalResult.second);
3277 const int lclSuccess = (makeIndicesLocalResult.first == 0);
3279 reduceAll (*comm, REDUCE_MIN, lclSuccess, outArg (gblSuccess));
3280 if (gblSuccess != 1) {
3281 std::ostringstream os;
3282 gathervPrint (os, makeIndicesLocalResult.second, *comm);
3283 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3284 (
true, std::runtime_error, os.str ());
3293 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3294 (makeIndicesLocalResult.first != 0, std::runtime_error,
3295 makeIndicesLocalResult.second);
3301 this->sortAndMergeAllIndices (this->isSorted (), this->isMerged ());
3306 this->makeImportExport (remotePIDs, mustBuildColMap);
3309 this->fillLocalGraph (params);
3311 const bool callComputeGlobalConstants = params.get () ==
nullptr ||
3312 params->get (
"compute global constants",
true);
3313 if (callComputeGlobalConstants) {
3314 this->computeGlobalConstants ();
3317 this->computeLocalConstants ();
3319 this->fillComplete_ =
true;
3320 this->checkInternalState ();
3323 std::ostringstream os;
3324 os << *prefix <<
"Done" << endl;
3325 std::cerr << os.str();
3330 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3334 const Teuchos::RCP<const map_type>& rangeMap,
3335 const Teuchos::RCP<const import_type>& importer,
3336 const Teuchos::RCP<const export_type>& exporter,
3337 const Teuchos::RCP<Teuchos::ParameterList>& params)
3339 const char tfecfFuncName[] =
"expertStaticFillComplete: ";
3340 #ifdef HAVE_TPETRA_MMM_TIMINGS
3342 if(!params.is_null())
3343 label = params->get(
"Timer Label",label);
3344 std::string prefix = std::string(
"Tpetra ")+ label + std::string(
": ");
3345 using Teuchos::TimeMonitor;
3346 Teuchos::RCP<Teuchos::TimeMonitor> MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-Setup"))));
3350 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3351 domainMap.is_null () || rangeMap.is_null (),
3352 std::runtime_error,
"The input domain Map and range Map must be nonnull.");
3353 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3354 isFillComplete () || ! hasColMap (), std::runtime_error,
"You may not "
3355 "call this method unless the graph has a column Map.");
3356 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3357 getLocalNumRows () > 0 && rowPtrsUnpacked_host_.extent (0) == 0,
3358 std::runtime_error,
"The calling process has getLocalNumRows() = "
3359 << getLocalNumRows () <<
" > 0 rows, but the row offsets array has not "
3361 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3362 static_cast<size_t> (rowPtrsUnpacked_host_.extent (0)) != getLocalNumRows () + 1,
3363 std::runtime_error,
"The row offsets array has length " <<
3364 rowPtrsUnpacked_host_.extent (0) <<
" != getLocalNumRows()+1 = " <<
3365 (getLocalNumRows () + 1) <<
".");
3380 numAllocForAllRows_ = 0;
3381 k_numAllocPerRow_ = decltype (k_numAllocPerRow_) ();
3382 indicesAreAllocated_ =
true;
3387 indicesAreLocal_ =
true;
3388 indicesAreGlobal_ =
false;
3391 #ifdef HAVE_TPETRA_MMM_TIMINGS
3393 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-Maps"))));
3395 setDomainRangeMaps (domainMap, rangeMap);
3398 indicesAreSorted_ =
true;
3399 noRedundancies_ =
true;
3402 #ifdef HAVE_TPETRA_MMM_TIMINGS
3404 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-mIXcheckI"))));
3407 importer_ = Teuchos::null;
3408 exporter_ = Teuchos::null;
3409 if (importer != Teuchos::null) {
3410 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3411 ! importer->getSourceMap ()->isSameAs (*getDomainMap ()) ||
3412 ! importer->getTargetMap ()->isSameAs (*getColMap ()),
3413 std::invalid_argument,
": importer does not match matrix maps.");
3414 importer_ = importer;
3418 #ifdef HAVE_TPETRA_MMM_TIMINGS
3420 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-mIXcheckE"))));
3423 if (exporter != Teuchos::null) {
3424 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3425 ! exporter->getSourceMap ()->isSameAs (*getRowMap ()) ||
3426 ! exporter->getTargetMap ()->isSameAs (*getRangeMap ()),
3427 std::invalid_argument,
": exporter does not match matrix maps.");
3428 exporter_ = exporter;
3431 #ifdef HAVE_TPETRA_MMM_TIMINGS
3433 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-mIXmake"))));
3435 Teuchos::Array<int> remotePIDs (0);
3436 this->makeImportExport (remotePIDs,
false);
3438 #ifdef HAVE_TPETRA_MMM_TIMINGS
3440 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-fLG"))));
3442 this->fillLocalGraph (params);
3444 const bool callComputeGlobalConstants = params.get () ==
nullptr ||
3445 params->get (
"compute global constants",
true);
3447 if (callComputeGlobalConstants) {
3448 #ifdef HAVE_TPETRA_MMM_TIMINGS
3450 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-cGC (const)"))));
3451 #endif // HAVE_TPETRA_MMM_TIMINGS
3452 this->computeGlobalConstants ();
3455 #ifdef HAVE_TPETRA_MMM_TIMINGS
3457 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-cGC (noconst)"))));
3458 #endif // HAVE_TPETRA_MMM_TIMINGS
3459 this->computeLocalConstants ();
3462 fillComplete_ =
true;
3464 #ifdef HAVE_TPETRA_MMM_TIMINGS
3466 MM = Teuchos::rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string(
"ESFC-G-cIS"))));
3468 checkInternalState ();
3472 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3475 fillLocalGraph (
const Teuchos::RCP<Teuchos::ParameterList>& params)
3478 typedef decltype (k_numRowEntries_) row_entries_type;
3479 typedef typename local_graph_device_type::row_map_type row_map_type;
3480 typedef typename row_map_type::non_const_type non_const_row_map_type;
3481 typedef typename local_graph_device_type::entries_type::non_const_type lclinds_1d_type;
3482 const
char tfecfFuncName[] = "fillLocalGraph (called from fillComplete or "
3483 "expertStaticFillComplete): ";
3484 const
size_t lclNumRows = this->getLocalNumRows ();
3489 bool requestOptimizedStorage = true;
3490 if (! params.is_null () && ! params->get ("Optimize Storage", true)) {
3491 requestOptimizedStorage =
false;
3500 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3501 (rowPtrsUnpacked_host_.extent (0) == 0, std::logic_error,
3502 "k_rowPtrs_ has size zero, but shouldn't");
3503 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3504 (rowPtrsUnpacked_host_.extent (0) != lclNumRows + 1, std::logic_error,
3505 "rowPtrsUnpacked_host_.extent(0) = "
3506 << rowPtrsUnpacked_host_.extent (0) <<
" != (lclNumRows + 1) = "
3507 << (lclNumRows + 1) <<
".");
3508 const size_t numOffsets = rowPtrsUnpacked_host_.extent (0);
3509 const auto valToCheck = rowPtrsUnpacked_host_(numOffsets-1);
3510 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3512 lclIndsUnpacked_wdv.extent (0) != valToCheck,
3513 std::logic_error,
"numOffsets=" << numOffsets <<
" != 0 "
3514 " and lclIndsUnpacked_wdv.extent(0)=" << lclIndsUnpacked_wdv.extent(0)
3515 <<
" != k_rowPtrs_(" << numOffsets <<
")=" << valToCheck
3519 size_t allocSize = 0;
3521 allocSize = this->getLocalAllocationSize ();
3523 catch (std::logic_error& e) {
3524 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3525 (
true, std::logic_error,
"getLocalAllocationSize threw "
3526 "std::logic_error: " << e.what ());
3528 catch (std::runtime_error& e) {
3529 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3530 (
true, std::runtime_error,
"getLocalAllocationSize threw "
3531 "std::runtime_error: " << e.what ());
3533 catch (std::exception& e) {
3534 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3535 (
true, std::runtime_error,
"getLocalAllocationSize threw "
3536 "std::exception: " << e.what ());
3539 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3540 (
true, std::runtime_error,
"getLocalAllocationSize threw "
3541 "an exception not a subclass of std::exception.");
3544 if (this->getLocalNumEntries () != allocSize) {
3547 non_const_row_map_type ptr_d;
3548 row_map_type ptr_d_const;
3557 if (rowPtrsUnpacked_host_.extent (0) != 0) {
3558 const size_t numOffsets =
3559 static_cast<size_t> (rowPtrsUnpacked_host_.extent (0));
3560 const auto valToCheck = rowPtrsUnpacked_host_(numOffsets - 1);
3561 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3562 (valToCheck !=
size_t(lclIndsUnpacked_wdv.extent(0)),
3563 std::logic_error,
"(Unpacked branch) Before allocating "
3564 "or packing, k_rowPtrs_(" << (numOffsets-1) <<
")="
3565 << valToCheck <<
" != lclIndsUnpacked_wdv.extent(0)="
3566 << lclIndsUnpacked_wdv.extent (0) <<
".");
3576 size_t lclTotalNumEntries = 0;
3580 non_const_row_map_type (
"Tpetra::CrsGraph::ptr", lclNumRows + 1);
3581 ptr_d_const = ptr_d;
3585 typename row_entries_type::const_type numRowEnt_h = k_numRowEntries_;
3587 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3588 (
size_t(numRowEnt_h.extent (0)) != lclNumRows,
3589 std::logic_error,
"(Unpacked branch) "
3590 "numRowEnt_h.extent(0)=" << numRowEnt_h.extent(0)
3591 <<
" != getLocalNumRows()=" << lclNumRows <<
"");
3597 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3598 (static_cast<size_t> (ptr_d.extent (0)) != lclNumRows + 1,
3599 std::logic_error,
"(Unpacked branch) After allocating "
3600 "ptr_d, ptr_d.extent(0) = " << ptr_d.extent(0)
3601 <<
" != lclNumRows+1 = " << (lclNumRows+1) <<
".");
3602 const auto valToCheck =
3603 ::Tpetra::Details::getEntryOnHost (ptr_d, lclNumRows);
3604 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3605 (valToCheck != lclTotalNumEntries, std::logic_error,
3606 "Tpetra::CrsGraph::fillLocalGraph: In unpacked branch, "
3607 "after filling ptr_d, ptr_d(lclNumRows=" << lclNumRows
3608 <<
") = " << valToCheck <<
" != total number of entries "
3609 "on the calling process = " << lclTotalNumEntries
3615 lclinds_1d_type ind_d =
3616 lclinds_1d_type (
"Tpetra::CrsGraph::lclInd", lclTotalNumEntries);
3628 typedef pack_functor<
3629 typename local_graph_device_type::entries_type::non_const_type,
3630 typename local_inds_dualv_type::t_dev::const_type,
3632 typename local_graph_device_type::row_map_type> inds_packer_type;
3633 inds_packer_type f (ind_d,
3634 lclIndsUnpacked_wdv.getDeviceView(Access::ReadOnly),
3635 ptr_d, rowPtrsUnpacked_dev_);
3637 typedef typename decltype (ind_d)::execution_space exec_space;
3638 typedef Kokkos::RangePolicy<exec_space, LocalOrdinal> range_type;
3639 Kokkos::parallel_for (range_type (0, lclNumRows), f);
3643 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3644 (ptr_d.extent (0) == 0, std::logic_error,
3645 "(\"Optimize Storage\"=true branch) After packing, "
3646 "ptr_d.extent(0)=0. This probably means k_rowPtrs_ was "
3647 "never allocated.");
3648 if (ptr_d.extent (0) != 0) {
3649 const size_t numOffsets =
static_cast<size_t> (ptr_d.extent (0));
3650 const auto valToCheck =
3651 ::Tpetra::Details::getEntryOnHost (ptr_d, numOffsets - 1);
3652 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3653 (static_cast<size_t> (valToCheck) != ind_d.extent (0),
3654 std::logic_error,
"(\"Optimize Storage\"=true branch) "
3655 "After packing, ptr_d(" << (numOffsets-1) <<
")="
3656 << valToCheck <<
" != ind_d.extent(0)="
3657 << ind_d.extent(0) <<
".");
3661 if (requestOptimizedStorage)
3662 setRowPtrs(ptr_d_const);
3664 setRowPtrsPacked(ptr_d_const);
3665 lclIndsPacked_wdv = local_inds_wdv_type(ind_d);
3669 rowPtrsPacked_dev_ = rowPtrsUnpacked_dev_;
3670 rowPtrsPacked_host_ = rowPtrsUnpacked_host_;
3671 lclIndsPacked_wdv = lclIndsUnpacked_wdv;
3674 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3675 (rowPtrsPacked_dev_.extent (0) == 0, std::logic_error,
3676 "(\"Optimize Storage\"=false branch) "
3677 "rowPtrsPacked_dev_.extent(0) = 0. "
3678 "This probably means that "
3679 "k_rowPtrs_ was never allocated.");
3680 if (rowPtrsPacked_dev_.extent (0) != 0) {
3681 const size_t numOffsets =
3682 static_cast<size_t> (rowPtrsPacked_dev_.extent (0));
3683 const size_t valToCheck =
3684 rowPtrsPacked_host_(numOffsets - 1);
3685 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3686 (valToCheck !=
size_t(lclIndsPacked_wdv.extent (0)),
3687 std::logic_error,
"(\"Optimize Storage\"=false branch) "
3688 "rowPtrsPacked_dev_(" << (numOffsets-1) <<
")="
3690 <<
" != lclIndsPacked_wdv.extent(0)="
3691 << lclIndsPacked_wdv.extent (0) <<
".");
3697 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3698 (static_cast<size_t> (rowPtrsPacked_dev_.extent (0)) != lclNumRows + 1,
3699 std::logic_error,
"After packing, rowPtrsPacked_dev_.extent(0) = " <<
3700 rowPtrsPacked_dev_.extent (0) <<
" != lclNumRows+1 = " << (lclNumRows+1)
3702 if (rowPtrsPacked_dev_.extent (0) != 0) {
3703 const size_t numOffsets =
static_cast<size_t> (rowPtrsPacked_dev_.extent (0));
3704 const auto valToCheck = rowPtrsPacked_host_(numOffsets - 1);
3705 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3706 (static_cast<size_t> (valToCheck) != lclIndsPacked_wdv.extent (0),
3707 std::logic_error,
"After packing, rowPtrsPacked_dev_(" << (numOffsets-1)
3708 <<
") = " << valToCheck <<
" != lclIndsPacked_wdv.extent(0) = "
3709 << lclIndsPacked_wdv.extent (0) <<
".");
3713 if (requestOptimizedStorage) {
3719 k_numRowEntries_ = row_entries_type ();
3722 lclIndsUnpacked_wdv = lclIndsPacked_wdv;
3724 storageStatus_ = Details::STORAGE_1D_PACKED;
3727 set_need_sync_host_uvm_access();
3730 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3742 const char tfecfFuncName[] =
"replaceColMap: ";
3743 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3744 isLocallyIndexed () || isGloballyIndexed (), std::runtime_error,
3745 "Requires matching maps and non-static graph.");
3746 colMap_ = newColMap;
3749 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3753 const Teuchos::RCP<const import_type>& newImport,
3754 const bool sortIndicesInEachRow)
3756 using Teuchos::REDUCE_MIN;
3757 using Teuchos::reduceAll;
3759 typedef GlobalOrdinal GO;
3760 typedef LocalOrdinal LO;
3761 typedef typename local_inds_dualv_type::t_host col_inds_type;
3762 const char tfecfFuncName[] =
"reindexColumns: ";
3764 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3765 isFillComplete (), std::runtime_error,
"The graph is fill complete "
3766 "(isFillComplete() returns true). You must call resumeFill() before "
3767 "you may call this method.");
3785 const LO lclNumRows =
static_cast<LO
> (this->getLocalNumRows ());
3800 bool allCurColIndsValid =
true;
3805 bool localSuffices =
true;
3813 col_inds_type newLclInds1D;
3814 auto oldLclInds1D = lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
3819 if (indicesAreAllocated ()) {
3820 if (isLocallyIndexed ()) {
3822 const map_type& oldColMap = * (getColMap ());
3824 const size_t allocSize = this->getLocalAllocationSize ();
3825 newLclInds1D = col_inds_type(
"Tpetra::CrsGraph::lclIndsReindexedHost",
3828 for (LO lclRow = 0; lclRow < lclNumRows; ++lclRow) {
3829 const RowInfo rowInfo = this->getRowInfo (lclRow);
3830 const size_t beg = rowInfo.offset1D;
3831 const size_t end = beg + rowInfo.numEntries;
3832 for (
size_t k = beg; k < end; ++k) {
3833 const LO oldLclCol = oldLclInds1D(k);
3834 if (oldLclCol == Teuchos::OrdinalTraits<LO>::invalid ()) {
3835 allCurColIndsValid =
false;
3843 if (gblCol == Teuchos::OrdinalTraits<GO>::invalid ()) {
3844 allCurColIndsValid =
false;
3848 const LO newLclCol = newColMap->getLocalElement (gblCol);
3849 if (newLclCol == Teuchos::OrdinalTraits<LO>::invalid ()) {
3850 localSuffices =
false;
3853 newLclInds1D(k) = newLclCol;
3864 allCurColIndsValid =
false;
3881 for (LO lclRow = 0; lclRow < lclNumRows; ++lclRow) {
3882 const RowInfo rowInfo = this->getRowInfo (lclRow);
3883 auto oldGblRowView = this->getGlobalIndsViewHost (rowInfo);
3884 for (
size_t k = 0; k < rowInfo.numEntries; ++k) {
3885 const GO gblCol = oldGblRowView(k);
3886 if (! newColMap->isNodeGlobalElement (gblCol)) {
3887 localSuffices =
false;
3897 lclSuccess[0] = allCurColIndsValid ? 1 : 0;
3898 lclSuccess[1] = localSuffices ? 1 : 0;
3902 RCP<const Teuchos::Comm<int> > comm =
3903 getRowMap ().is_null () ? Teuchos::null : getRowMap ()->getComm ();
3904 if (! comm.is_null ()) {
3905 reduceAll<int, int> (*comm, REDUCE_MIN, 2, lclSuccess, gblSuccess);
3908 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3909 gblSuccess[0] == 0, std::runtime_error,
"It is not possible to continue."
3910 " The most likely reason is that the graph is locally indexed, but the "
3911 "column Map is missing (null) on some processes, due to a previous call "
3912 "to replaceColMap().");
3914 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3915 gblSuccess[1] == 0, std::runtime_error,
"On some process, the graph "
3916 "contains column indices that are in the old column Map, but not in the "
3917 "new column Map (on that process). This method does NOT redistribute "
3918 "data; it does not claim to do the work of an Import or Export operation."
3919 " This means that for all processess, the calling process MUST own all "
3920 "column indices, in both the old column Map and the new column Map. In "
3921 "this case, you will need to do an Import or Export operation to "
3922 "redistribute data.");
3925 if (isLocallyIndexed ()) {
3927 typename local_inds_dualv_type::t_dev newLclInds1D_dev(
3928 Kokkos::view_alloc(
"Tpetra::CrsGraph::lclIndReindexed",
3929 Kokkos::WithoutInitializing),
3930 newLclInds1D.extent(0));
3932 lclIndsUnpacked_wdv = local_inds_wdv_type(newLclInds1D_dev);
3941 indicesAreSorted_ =
false;
3942 if (sortIndicesInEachRow) {
3949 const bool sorted =
false;
3950 const bool merged =
true;
3951 this->sortAndMergeAllIndices (sorted, merged);
3954 colMap_ = newColMap;
3956 if (newImport.is_null ()) {
3964 if (! domainMap_.is_null ()) {
3965 if (! domainMap_->isSameAs (* newColMap)) {
3966 importer_ = Teuchos::rcp (
new import_type (domainMap_, newColMap));
3968 importer_ = Teuchos::null;
3973 importer_ = newImport;
3977 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
3982 const char prefix[] =
"Tpetra::CrsGraph::replaceDomainMap: ";
3983 TEUCHOS_TEST_FOR_EXCEPTION(
3984 colMap_.is_null (), std::invalid_argument, prefix <<
"You may not call "
3985 "this method unless the graph already has a column Map.");
3986 TEUCHOS_TEST_FOR_EXCEPTION(
3987 newDomainMap.is_null (), std::invalid_argument,
3988 prefix <<
"The new domain Map must be nonnull.");
3991 Teuchos::RCP<const import_type> newImporter = Teuchos::null;
3992 if (newDomainMap != colMap_ && (! newDomainMap->isSameAs (*colMap_))) {
3993 newImporter = rcp(
new import_type(newDomainMap, colMap_));
3995 this->replaceDomainMapAndImporter(newDomainMap, newImporter);
3998 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4002 const Teuchos::RCP<const import_type>& newImporter)
4004 const char prefix[] =
"Tpetra::CrsGraph::replaceDomainMapAndImporter: ";
4005 TEUCHOS_TEST_FOR_EXCEPTION(
4006 colMap_.is_null (), std::invalid_argument, prefix <<
"You may not call "
4007 "this method unless the graph already has a column Map.");
4008 TEUCHOS_TEST_FOR_EXCEPTION(
4009 newDomainMap.is_null (), std::invalid_argument,
4010 prefix <<
"The new domain Map must be nonnull.");
4013 if (newImporter.is_null ()) {
4018 const bool colSameAsDom = colMap_->isSameAs (*newDomainMap);
4019 TEUCHOS_TEST_FOR_EXCEPTION
4020 (!colSameAsDom, std::invalid_argument,
"If the new Import is null, "
4021 "then the new domain Map must be the same as the current column Map.");
4024 const bool colSameAsTgt =
4025 colMap_->isSameAs (* (newImporter->getTargetMap ()));
4026 const bool newDomSameAsSrc =
4027 newDomainMap->isSameAs (* (newImporter->getSourceMap ()));
4028 TEUCHOS_TEST_FOR_EXCEPTION
4029 (! colSameAsTgt || ! newDomSameAsSrc, std::invalid_argument,
"If the "
4030 "new Import is nonnull, then the current column Map must be the same "
4031 "as the new Import's target Map, and the new domain Map must be the "
4032 "same as the new Import's source Map.");
4036 domainMap_ = newDomainMap;
4037 importer_ = Teuchos::rcp_const_cast<
import_type> (newImporter);
4040 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4045 const char prefix[] =
"Tpetra::CrsGraph::replaceRangeMap: ";
4046 TEUCHOS_TEST_FOR_EXCEPTION(
4047 rowMap_.is_null (), std::invalid_argument, prefix <<
"You may not call "
4048 "this method unless the graph already has a row Map.");
4049 TEUCHOS_TEST_FOR_EXCEPTION(
4050 newRangeMap.is_null (), std::invalid_argument,
4051 prefix <<
"The new range Map must be nonnull.");
4054 Teuchos::RCP<const export_type> newExporter = Teuchos::null;
4055 if (newRangeMap != rowMap_ && (! newRangeMap->isSameAs (*rowMap_))) {
4056 newExporter = rcp(
new export_type(rowMap_, newRangeMap));
4058 this->replaceRangeMapAndExporter(newRangeMap, newExporter);
4061 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4065 const Teuchos::RCP<const export_type>& newExporter)
4067 const char prefix[] =
"Tpetra::CrsGraph::replaceRangeMapAndExporter: ";
4068 TEUCHOS_TEST_FOR_EXCEPTION(
4069 rowMap_.is_null (), std::invalid_argument, prefix <<
"You may not call "
4070 "this method unless the graph already has a column Map.");
4071 TEUCHOS_TEST_FOR_EXCEPTION(
4072 newRangeMap.is_null (), std::invalid_argument,
4073 prefix <<
"The new domain Map must be nonnull.");
4076 if (newExporter.is_null ()) {
4081 const bool rowSameAsRange = rowMap_->isSameAs (*newRangeMap);
4082 TEUCHOS_TEST_FOR_EXCEPTION
4083 (!rowSameAsRange, std::invalid_argument,
"If the new Export is null, "
4084 "then the new range Map must be the same as the current row Map.");
4087 const bool newRangeSameAsTgt =
4088 newRangeMap->isSameAs (* (newExporter->getTargetMap ()));
4089 const bool rowSameAsSrc =
4090 rowMap_->isSameAs (* (newExporter->getSourceMap ()));
4091 TEUCHOS_TEST_FOR_EXCEPTION
4092 (! rowSameAsSrc || ! newRangeSameAsTgt, std::invalid_argument,
"If the "
4093 "new Export is nonnull, then the current row Map must be the same "
4094 "as the new Export's source Map, and the new range Map must be the "
4095 "same as the new Export's target Map.");
4099 rangeMap_ = newRangeMap;
4100 exporter_ = Teuchos::rcp_const_cast<
export_type> (newExporter);
4104 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4110 lclIndsPacked_wdv.getDeviceView(Access::ReadWrite),
4111 rowPtrsPacked_dev_);
4114 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4119 return local_graph_host_type(
4120 lclIndsPacked_wdv.getHostView(Access::ReadWrite),
4121 rowPtrsPacked_host_);
4124 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4129 using ::Tpetra::Details::ProfilingRegion;
4130 using Teuchos::ArrayView;
4131 using Teuchos::outArg;
4132 using Teuchos::reduceAll;
4135 ProfilingRegion regionCGC (
"Tpetra::CrsGraph::computeGlobalConstants");
4137 this->computeLocalConstants ();
4142 if (! this->haveGlobalConstants_) {
4143 const Teuchos::Comm<int>& comm = * (this->getComm ());
4157 lcl =
static_cast<GST
> (this->getLocalNumEntries ());
4159 reduceAll<int,GST> (comm, Teuchos::REDUCE_SUM, 1, &lcl, &gbl);
4160 this->globalNumEntries_ = gbl;
4162 const GST lclMaxNumRowEnt =
static_cast<GST
> (this->nodeMaxNumRowEntries_);
4163 reduceAll<int, GST> (comm, Teuchos::REDUCE_MAX, lclMaxNumRowEnt,
4164 outArg (this->globalMaxNumRowEntries_));
4165 this->haveGlobalConstants_ =
true;
4170 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4175 using ::Tpetra::Details::ProfilingRegion;
4177 ProfilingRegion regionCLC (
"Tpetra::CrsGraph::computeLocalConstants");
4178 if (this->haveLocalConstants_) {
4183 this->nodeMaxNumRowEntries_ =
4184 Teuchos::OrdinalTraits<size_t>::invalid();
4188 auto ptr = this->rowPtrsPacked_dev_;
4189 const LO lclNumRows = ptr.extent(0) == 0 ?
4190 static_cast<LO
> (0) :
4191 (static_cast<LO> (ptr.extent(0)) - static_cast<LO> (1));
4193 const LO lclMaxNumRowEnt =
4194 ::Tpetra::Details::maxDifference (
"Tpetra::CrsGraph: nodeMaxNumRowEntries",
4196 this->nodeMaxNumRowEntries_ =
static_cast<size_t> (lclMaxNumRowEnt);
4197 this->haveLocalConstants_ =
true;
4201 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4202 std::pair<size_t, std::string>
4207 using Teuchos::arcp;
4208 using Teuchos::Array;
4210 typedef LocalOrdinal LO;
4211 typedef GlobalOrdinal GO;
4213 typedef typename local_graph_device_type::row_map_type::non_const_value_type offset_type;
4214 typedef decltype (k_numRowEntries_) row_entries_type;
4215 typedef typename row_entries_type::non_const_value_type num_ent_type;
4216 const char tfecfFuncName[] =
"makeIndicesLocal: ";
4217 ProfilingRegion regionMakeIndicesLocal (
"Tpetra::CrsGraph::makeIndicesLocal");
4219 std::unique_ptr<std::string> prefix;
4221 prefix = this->createPrefix(
"CrsGraph",
"makeIndicesLocal");
4222 std::ostringstream os;
4223 os << *prefix <<
"lclNumRows: " << getLocalNumRows() << endl;
4224 std::cerr << os.str();
4229 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4230 (! this->hasColMap (), std::logic_error,
"The graph does not have a "
4231 "column Map yet. This method should never be called in that case. "
4232 "Please report this bug to the Tpetra developers.");
4233 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4234 (this->getColMap ().is_null (), std::logic_error,
"The graph claims "
4235 "that it has a column Map, because hasColMap() returns true. However, "
4236 "the result of getColMap() is null. This should never happen. Please "
4237 "report this bug to the Tpetra developers.");
4242 size_t lclNumErrs = 0;
4243 std::ostringstream errStrm;
4245 const LO lclNumRows =
static_cast<LO
> (this->getLocalNumRows ());
4246 const map_type& colMap = * (this->getColMap ());
4248 if (this->isGloballyIndexed () && lclNumRows != 0) {
4250 typename row_entries_type::const_type h_numRowEnt =
4251 this->k_numRowEntries_;
4254 if (rowPtrsUnpacked_host_.extent (0) == 0) {
4255 errStrm <<
"k_rowPtrs_.extent(0) == 0. This should never "
4256 "happen here. Please report this bug to the Tpetra developers."
4259 return std::make_pair(Tpetra::Details::OrdinalTraits<size_t>::invalid (),
4262 const auto numEnt = rowPtrsUnpacked_host_(lclNumRows);
4271 using Kokkos::view_alloc;
4272 using Kokkos::WithoutInitializing;
4282 const std::string label (
"Tpetra::CrsGraph::lclInd");
4284 std::ostringstream os;
4285 os << *prefix <<
"(Re)allocate lclInd_wdv: old="
4286 << lclIndsUnpacked_wdv.extent(0) <<
", new=" << numEnt << endl;
4287 std::cerr << os.str();
4290 local_inds_dualv_type lclInds_dualv =
4291 local_inds_dualv_type(view_alloc(label, WithoutInitializing),
4293 lclIndsUnpacked_wdv = local_inds_wdv_type(lclInds_dualv);
4304 std::ostringstream os;
4305 os << *prefix <<
"Allocate device mirror k_numRowEnt: "
4306 << h_numRowEnt.extent(0) << endl;
4307 std::cerr << os.str();
4310 Kokkos::create_mirror_view_and_copy (
device_type (), h_numRowEnt);
4314 convertColumnIndicesFromGlobalToLocal<LO, GO, DT, offset_type, num_ent_type> (
4315 lclIndsUnpacked_wdv.getDeviceView(Access::OverwriteAll),
4316 gblInds_wdv.getDeviceView(Access::ReadOnly),
4317 rowPtrsUnpacked_dev_,
4320 if (lclNumErrs != 0) {
4321 const int myRank = [
this] () {
4322 auto map = this->getMap ();
4323 if (map.is_null ()) {
4327 auto comm = map->getComm ();
4328 return comm.is_null () ? 0 : comm->getRank ();
4331 const bool pluralNumErrs = (lclNumErrs !=
static_cast<size_t> (1));
4332 errStrm <<
"(Process " << myRank <<
") When converting column "
4333 "indices from global to local, we encountered " << lclNumErrs
4334 <<
" ind" << (pluralNumErrs ?
"ices" :
"ex")
4335 <<
" that do" << (pluralNumErrs ?
"es" :
"")
4336 <<
" not live in the column Map on this process." << endl;
4343 std::ostringstream os;
4344 os << *prefix <<
"Free gblInds_wdv: "
4345 << gblInds_wdv.extent(0) << endl;
4346 std::cerr << os.str();
4348 gblInds_wdv = global_inds_wdv_type ();
4351 this->indicesAreLocal_ =
true;
4352 this->indicesAreGlobal_ =
false;
4353 this->checkInternalState ();
4355 return std::make_pair (lclNumErrs, errStrm.str ());
4358 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4365 const char tfecfFuncName[] =
"makeColMap";
4367 ProfilingRegion regionSortAndMerge (
"Tpetra::CrsGraph::makeColMap");
4368 std::unique_ptr<std::string> prefix;
4370 prefix = this->createPrefix(
"CrsGraph", tfecfFuncName);
4371 std::ostringstream os;
4372 os << *prefix <<
"Start" << endl;
4373 std::cerr << os.str();
4381 Teuchos::RCP<const map_type> colMap = this->colMap_;
4382 const bool sortEachProcsGids =
4383 this->sortGhostsAssociatedWithEachProcessor_;
4393 using Teuchos::outArg;
4394 using Teuchos::REDUCE_MIN;
4395 using Teuchos::reduceAll;
4397 std::ostringstream errStrm;
4398 const int lclErrCode =
4400 getDomainMap (), *
this, sortEachProcsGids, &errStrm);
4401 auto comm = this->getComm ();
4402 if (! comm.is_null ()) {
4403 const int lclSuccess = (lclErrCode == 0) ? 1 : 0;
4405 reduceAll<int, int> (*comm, REDUCE_MIN, lclSuccess,
4406 outArg (gblSuccess));
4407 if (gblSuccess != 1) {
4408 std::ostringstream os;
4410 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4411 (
true, std::runtime_error,
": An error happened on at "
4412 "least one process in the CrsGraph's communicator. "
4413 "Here are all processes' error messages:" << std::endl
4420 getDomainMap (), *
this, sortEachProcsGids,
nullptr);
4425 this->colMap_ = colMap;
4427 checkInternalState ();
4429 std::ostringstream os;
4430 os << *prefix <<
"Done" << endl;
4431 std::cerr << os.str();
4436 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4442 using LO = LocalOrdinal;
4443 using host_execution_space =
4444 typename Kokkos::View<LO*, device_type>::HostMirror::
4446 using range_type = Kokkos::RangePolicy<host_execution_space, LO>;
4447 const char tfecfFuncName[] =
"sortAndMergeAllIndices";
4449 (
"Tpetra::CrsGraph::sortAndMergeAllIndices");
4451 std::unique_ptr<std::string> prefix;
4453 prefix = this->createPrefix(
"CrsGraph", tfecfFuncName);
4454 std::ostringstream os;
4455 os << *prefix <<
"Start: "
4456 <<
"sorted=" << (sorted ?
"true" :
"false")
4457 <<
", merged=" << (merged ?
"true" :
"false") << endl;
4458 std::cerr << os.str();
4460 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4461 (this->isGloballyIndexed(), std::logic_error,
4462 "This method may only be called after makeIndicesLocal." );
4463 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4464 (! merged && this->isStorageOptimized(), std::logic_error,
4465 "The graph is already storage optimized, so we shouldn't be "
4466 "merging any indices. "
4467 "Please report this bug to the Tpetra developers.");
4469 if (! sorted || ! merged) {
4470 const LO lclNumRows(this->getLocalNumRows());
4471 auto range = range_type(0, lclNumRows);
4474 size_t totalNumDups = 0;
4475 Kokkos::parallel_reduce(range,
4476 [
this, sorted, merged] (
const LO lclRow,
size_t& numDups)
4478 const RowInfo rowInfo = this->getRowInfo(lclRow);
4479 numDups += this->sortAndMergeRowIndices(rowInfo, sorted, merged);
4482 std::ostringstream os;
4483 os << *prefix <<
"totalNumDups=" << totalNumDups << endl;
4484 std::cerr << os.str();
4487 Kokkos::parallel_for(range,
4488 [
this, sorted, merged] (
const LO lclRow)
4490 const RowInfo rowInfo = this->getRowInfo(lclRow);
4491 this->sortAndMergeRowIndices(rowInfo, sorted, merged);
4494 this->indicesAreSorted_ =
true;
4495 this->noRedundancies_ =
true;
4499 std::ostringstream os;
4500 os << *prefix <<
"Done" << endl;
4501 std::cerr << os.str();
4505 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4509 const bool useRemotePIDs)
4511 using ::Tpetra::Details::ProfilingRegion;
4512 using Teuchos::ParameterList;
4515 const char tfecfFuncName[] =
"makeImportExport: ";
4516 ProfilingRegion regionMIE (
"Tpetra::CrsGraph::makeImportExport");
4518 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4519 (! this->hasColMap (), std::logic_error,
4520 "This method may not be called unless the graph has a column Map.");
4521 RCP<ParameterList> params = this->getNonconstParameterList ();
4530 if (importer_.is_null ()) {
4532 if (domainMap_ != colMap_ && (! domainMap_->isSameAs (*colMap_))) {
4533 if (params.is_null () || ! params->isSublist (
"Import")) {
4534 if (useRemotePIDs) {
4535 importer_ = rcp (
new import_type (domainMap_, colMap_, remotePIDs));
4538 importer_ = rcp (
new import_type (domainMap_, colMap_));
4542 RCP<ParameterList> importSublist = sublist (params,
"Import",
true);
4543 if (useRemotePIDs) {
4544 RCP<import_type> newImp =
4545 rcp (
new import_type (domainMap_, colMap_, remotePIDs,
4550 importer_ = rcp (
new import_type (domainMap_, colMap_, importSublist));
4558 if (exporter_.is_null ()) {
4560 if (rangeMap_ != rowMap_ && ! rangeMap_->isSameAs (*rowMap_)) {
4561 if (params.is_null () || ! params->isSublist (
"Export")) {
4562 exporter_ = rcp (
new export_type (rowMap_, rangeMap_));
4565 RCP<ParameterList> exportSublist = sublist (params,
"Export",
true);
4566 exporter_ = rcp (
new export_type (rowMap_, rangeMap_, exportSublist));
4573 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4578 std::ostringstream oss;
4579 oss << dist_object_type::description ();
4580 if (isFillComplete ()) {
4581 oss <<
"{status = fill complete"
4582 <<
", global rows = " << getGlobalNumRows()
4583 <<
", global cols = " << getGlobalNumCols()
4584 <<
", global num entries = " << getGlobalNumEntries()
4588 oss <<
"{status = fill not complete"
4589 <<
", global rows = " << getGlobalNumRows()
4596 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4600 const Teuchos::EVerbosityLevel verbLevel)
const
4602 using Teuchos::ArrayView;
4603 using Teuchos::Comm;
4605 using Teuchos::VERB_DEFAULT;
4606 using Teuchos::VERB_NONE;
4607 using Teuchos::VERB_LOW;
4608 using Teuchos::VERB_MEDIUM;
4609 using Teuchos::VERB_HIGH;
4610 using Teuchos::VERB_EXTREME;
4614 Teuchos::EVerbosityLevel vl = verbLevel;
4615 if (vl == VERB_DEFAULT) vl = VERB_LOW;
4616 RCP<const Comm<int> > comm = this->getComm();
4617 const int myImageID = comm->getRank(),
4618 numImages = comm->getSize();
4620 for (
size_t dec=10; dec<getGlobalNumRows(); dec *= 10) {
4623 width = std::max<size_t> (width,
static_cast<size_t> (11)) + 2;
4624 Teuchos::OSTab tab (out);
4632 if (vl != VERB_NONE) {
4633 if (myImageID == 0) out << this->description() << std::endl;
4635 if (isFillComplete() && myImageID == 0) {
4636 out <<
"Global max number of row entries = " << globalMaxNumRowEntries_ << std::endl;
4639 if (vl == VERB_MEDIUM || vl == VERB_HIGH || vl == VERB_EXTREME) {
4640 if (myImageID == 0) out <<
"\nRow map: " << std::endl;
4641 rowMap_->describe(out,vl);
4642 if (colMap_ != Teuchos::null) {
4643 if (myImageID == 0) out <<
"\nColumn map: " << std::endl;
4644 colMap_->describe(out,vl);
4646 if (domainMap_ != Teuchos::null) {
4647 if (myImageID == 0) out <<
"\nDomain map: " << std::endl;
4648 domainMap_->describe(out,vl);
4650 if (rangeMap_ != Teuchos::null) {
4651 if (myImageID == 0) out <<
"\nRange map: " << std::endl;
4652 rangeMap_->describe(out,vl);
4656 if (vl == VERB_MEDIUM || vl == VERB_HIGH || vl == VERB_EXTREME) {
4657 for (
int imageCtr = 0; imageCtr < numImages; ++imageCtr) {
4658 if (myImageID == imageCtr) {
4659 out <<
"Node ID = " << imageCtr << std::endl
4660 <<
"Node number of entries = " << this->getLocalNumEntries () << std::endl
4661 <<
"Node max number of entries = " << nodeMaxNumRowEntries_ << std::endl;
4662 if (! indicesAreAllocated ()) {
4663 out <<
"Indices are not allocated." << std::endl;
4672 if (vl == VERB_HIGH || vl == VERB_EXTREME) {
4673 for (
int imageCtr = 0; imageCtr < numImages; ++imageCtr) {
4674 if (myImageID == imageCtr) {
4675 out << std::setw(width) <<
"Node ID"
4676 << std::setw(width) <<
"Global Row"
4677 << std::setw(width) <<
"Num Entries";
4678 if (vl == VERB_EXTREME) {
4682 const LocalOrdinal lclNumRows =
4683 static_cast<LocalOrdinal
> (this->getLocalNumRows ());
4684 for (LocalOrdinal r=0; r < lclNumRows; ++r) {
4685 const RowInfo rowinfo = this->getRowInfo (r);
4686 GlobalOrdinal gid = rowMap_->getGlobalElement(r);
4687 out << std::setw(width) << myImageID
4688 << std::setw(width) << gid
4689 << std::setw(width) << rowinfo.numEntries;
4690 if (vl == VERB_EXTREME) {
4692 if (isGloballyIndexed()) {
4693 auto rowview = gblInds_wdv.getHostView(Access::ReadOnly);
4694 for (
size_t j=0; j < rowinfo.numEntries; ++j){
4695 GlobalOrdinal colgid = rowview[j + rowinfo.offset1D];
4696 out << colgid <<
" ";
4699 else if (isLocallyIndexed()) {
4700 auto rowview = lclIndsUnpacked_wdv.getHostView(Access::ReadOnly);
4701 for (
size_t j=0; j < rowinfo.numEntries; ++j) {
4702 LocalOrdinal collid = rowview[j + rowinfo.offset1D];
4703 out << colMap_->getGlobalElement(collid) <<
" ";
4719 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4730 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4735 const size_t numSameIDs,
4736 const Kokkos::DualView<
const local_ordinal_type*,
4737 buffer_device_type>& permuteToLIDs,
4738 const Kokkos::DualView<
const local_ordinal_type*,
4739 buffer_device_type>& permuteFromLIDs,
4743 using LO = local_ordinal_type;
4744 using GO = global_ordinal_type;
4746 const char tfecfFuncName[] =
"copyAndPermute: ";
4747 const bool verbose = verbose_;
4749 std::unique_ptr<std::string> prefix;
4751 prefix = this->createPrefix(
"CrsGraph",
"copyAndPermute");
4752 std::ostringstream os;
4753 os << *prefix << endl;
4754 std::cerr << os.str ();
4757 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4758 (permuteToLIDs.extent (0) != permuteFromLIDs.extent (0),
4759 std::runtime_error,
"permuteToLIDs.extent(0) = "
4760 << permuteToLIDs.extent (0) <<
" != permuteFromLIDs.extent(0) = "
4761 << permuteFromLIDs.extent (0) <<
".");
4765 const row_graph_type& srcRowGraph =
4766 dynamic_cast<const row_graph_type&
> (source);
4769 std::ostringstream os;
4770 os << *prefix <<
"Compute padding" << endl;
4771 std::cerr << os.str ();
4773 auto padding = computeCrsPadding(srcRowGraph, numSameIDs,
4774 permuteToLIDs, permuteFromLIDs, verbose);
4775 applyCrsPadding(*padding, verbose);
4780 const this_CRS_type* srcCrsGraph =
4781 dynamic_cast<const this_CRS_type*
> (&source);
4783 const map_type& srcRowMap = *(srcRowGraph.getRowMap());
4784 const map_type& tgtRowMap = *(getRowMap());
4785 const bool src_filled = srcRowGraph.isFillComplete();
4786 nonconst_global_inds_host_view_type row_copy;
4792 if (src_filled || srcCrsGraph ==
nullptr) {
4794 std::ostringstream os;
4795 os << *prefix <<
"src_filled || srcCrsGraph == nullptr" << endl;
4796 std::cerr << os.str ();
4803 for (
size_t i = 0; i < numSameIDs; ++i, ++myid) {
4804 const GO gid = srcRowMap.getGlobalElement (myid);
4805 size_t row_length = srcRowGraph.getNumEntriesInGlobalRow (gid);
4806 Kokkos::resize(row_copy,row_length);
4807 size_t check_row_length = 0;
4808 srcRowGraph.getGlobalRowCopy (gid, row_copy, check_row_length);
4809 this->insertGlobalIndices (gid, row_length, row_copy.data());
4813 std::ostringstream os;
4814 os << *prefix <<
"! src_filled && srcCrsGraph != nullptr" << endl;
4815 std::cerr << os.str ();
4817 for (
size_t i = 0; i < numSameIDs; ++i, ++myid) {
4818 const GO gid = srcRowMap.getGlobalElement (myid);
4819 global_inds_host_view_type row;
4820 srcCrsGraph->getGlobalRowView (gid, row);
4821 this->insertGlobalIndices (gid, row.extent(0), row.data());
4828 auto permuteToLIDs_h = permuteToLIDs.view_host ();
4829 auto permuteFromLIDs_h = permuteFromLIDs.view_host ();
4831 if (src_filled || srcCrsGraph ==
nullptr) {
4832 for (LO i = 0; i < static_cast<LO> (permuteToLIDs_h.extent (0)); ++i) {
4833 const GO mygid = tgtRowMap.getGlobalElement (permuteToLIDs_h[i]);
4834 const GO srcgid = srcRowMap.getGlobalElement (permuteFromLIDs_h[i]);
4835 size_t row_length = srcRowGraph.getNumEntriesInGlobalRow (srcgid);
4836 Kokkos::resize(row_copy,row_length);
4837 size_t check_row_length = 0;
4838 srcRowGraph.getGlobalRowCopy (srcgid, row_copy, check_row_length);
4839 this->insertGlobalIndices (mygid, row_length, row_copy.data());
4842 for (LO i = 0; i < static_cast<LO> (permuteToLIDs_h.extent (0)); ++i) {
4843 const GO mygid = tgtRowMap.getGlobalElement (permuteToLIDs_h[i]);
4844 const GO srcgid = srcRowMap.getGlobalElement (permuteFromLIDs_h[i]);
4845 global_inds_host_view_type row;
4846 srcCrsGraph->getGlobalRowView (srcgid, row);
4847 this->insertGlobalIndices (mygid, row.extent(0), row.data());
4852 std::ostringstream os;
4853 os << *prefix <<
"Done" << endl;
4854 std::cerr << os.str ();
4858 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
4860 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
4861 applyCrsPadding(
const padding_type& padding,
4864 using Details::ProfilingRegion;
4868 using row_ptrs_type =
4869 typename local_graph_device_type::row_map_type::non_const_type;
4870 using range_policy =
4871 Kokkos::RangePolicy<execution_space, Kokkos::IndexType<LO>>;
4872 const char tfecfFuncName[] =
"applyCrsPadding";
4873 ProfilingRegion regionCAP(
"Tpetra::CrsGraph::applyCrsPadding");
4875 std::unique_ptr<std::string> prefix;
4877 prefix = this->
createPrefix(
"CrsGraph", tfecfFuncName);
4878 std::ostringstream os;
4879 os << *prefix <<
"padding: ";
4882 std::cerr << os.str();
4884 const int myRank = ! verbose ? -1 : [&] () {
4885 auto map = this->getMap();
4886 if (map.is_null()) {
4889 auto comm = map->getComm();
4890 if (comm.is_null()) {
4893 return comm->getRank();
4902 if (! indicesAreAllocated()) {
4904 std::ostringstream os;
4905 os << *prefix <<
"Call allocateIndices" << endl;
4906 std::cerr << os.str();
4908 allocateIndices(GlobalIndices, verbose);
4910 TEUCHOS_ASSERT( indicesAreAllocated() );
4916 std::ostringstream os;
4917 os << *prefix <<
"Allocate row_ptrs_beg: "
4918 << rowPtrsUnpacked_dev_.extent(0) << endl;
4919 std::cerr << os.str();
4921 using Kokkos::view_alloc;
4922 using Kokkos::WithoutInitializing;
4923 row_ptrs_type row_ptrs_beg(
4924 view_alloc(
"row_ptrs_beg", WithoutInitializing),
4925 rowPtrsUnpacked_dev_.extent(0));
4929 const size_t N = row_ptrs_beg.extent(0) == 0 ? size_t(0) :
4930 size_t(row_ptrs_beg.extent(0) - 1);
4932 std::ostringstream os;
4933 os << *prefix <<
"Allocate row_ptrs_end: " << N << endl;
4934 std::cerr << os.str();
4936 row_ptrs_type row_ptrs_end(
4937 view_alloc(
"row_ptrs_end", WithoutInitializing), N);
4938 row_ptrs_type num_row_entries;
4940 const bool refill_num_row_entries = k_numRowEntries_.extent(0) != 0;
4942 execution_space().fence();
4944 if (refill_num_row_entries) {
4948 row_ptrs_type(view_alloc(
"num_row_entries", WithoutInitializing), N);
4950 Kokkos::parallel_for
4951 (
"Fill end row pointers", range_policy(0, N),
4952 KOKKOS_LAMBDA (
const size_t i) {
4953 row_ptrs_end(i) = row_ptrs_beg(i) + num_row_entries(i);
4960 Kokkos::parallel_for
4961 (
"Fill end row pointers", range_policy(0, N),
4962 KOKKOS_LAMBDA (
const size_t i) {
4963 row_ptrs_end(i) = row_ptrs_beg(i+1);
4967 if (isGloballyIndexed()) {
4969 padding, myRank, verbose);
4972 padCrsArrays(row_ptrs_beg, row_ptrs_end, lclIndsUnpacked_wdv,
4973 padding, myRank, verbose);
4976 if (refill_num_row_entries) {
4977 Kokkos::parallel_for
4978 (
"Fill num entries", range_policy(0, N),
4979 KOKKOS_LAMBDA (
const size_t i) {
4980 num_row_entries(i) = row_ptrs_end(i) - row_ptrs_beg(i);
4985 std::ostringstream os;
4986 os << *prefix <<
"Reassign k_rowPtrs_; old size: "
4987 << rowPtrsUnpacked_dev_.extent(0) <<
", new size: "
4988 << row_ptrs_beg.extent(0) << endl;
4989 std::cerr << os.str();
4990 TEUCHOS_ASSERT( rowPtrsUnpacked_dev_.extent(0) == row_ptrs_beg.extent(0) );
4993 setRowPtrsUnpacked(row_ptrs_beg);
4995 set_need_sync_host_uvm_access();
4998 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5000 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type
5002 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5004 const RowGraph<LocalOrdinal,GlobalOrdinal,Node>& source,
5005 const size_t numSameIDs,
5006 const Kokkos::DualView<
const local_ordinal_type*,
5007 buffer_device_type>& permuteToLIDs,
5008 const Kokkos::DualView<
const local_ordinal_type*,
5009 buffer_device_type>& permuteFromLIDs,
5010 const bool verbose)
const
5015 std::unique_ptr<std::string> prefix;
5018 "computeCrsPadding(same & permute)");
5019 std::ostringstream os;
5020 os << *prefix <<
"{numSameIDs: " << numSameIDs
5021 <<
", numPermutes: " << permuteFromLIDs.extent(0) <<
"}"
5023 std::cerr << os.str();
5026 const int myRank = [&] () {
5027 auto comm = rowMap_.is_null() ? Teuchos::null :
5029 return comm.is_null() ? -1 : comm->getRank();
5031 std::unique_ptr<padding_type> padding(
5032 new padding_type(myRank, numSameIDs,
5033 permuteFromLIDs.extent(0)));
5035 computeCrsPaddingForSameIDs(*padding, source,
5036 static_cast<LO>(numSameIDs));
5037 computeCrsPaddingForPermutedIDs(*padding, source, permuteToLIDs,
5042 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5044 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5045 computeCrsPaddingForSameIDs(
5046 padding_type& padding,
5047 const RowGraph<local_ordinal_type, global_ordinal_type,
5049 const local_ordinal_type numSameIDs)
const
5052 using GO = global_ordinal_type;
5053 using Details::Impl::getRowGraphGlobalRow;
5055 const char tfecfFuncName[] =
"computeCrsPaddingForSameIds";
5057 std::unique_ptr<std::string> prefix;
5058 const bool verbose = verbose_;
5060 prefix = this->
createPrefix(
"CrsGraph", tfecfFuncName);
5061 std::ostringstream os;
5062 os << *prefix <<
"numSameIDs: " << numSameIDs << endl;
5063 std::cerr << os.str();
5066 if (numSameIDs == 0) {
5070 const map_type& srcRowMap = *(source.getRowMap());
5071 const map_type& tgtRowMap = *rowMap_;
5072 using this_CRS_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
5073 const this_CRS_type* srcCrs =
dynamic_cast<const this_CRS_type*
>(&source);
5074 const bool src_is_unique =
5075 srcCrs ==
nullptr ?
false : srcCrs->isMerged();
5076 const bool tgt_is_unique = this->isMerged();
5078 std::vector<GO> srcGblColIndsScratch;
5079 std::vector<GO> tgtGblColIndsScratch;
5081 execute_sync_host_uvm_access();
5082 for (LO lclRowInd = 0; lclRowInd < numSameIDs; ++lclRowInd) {
5083 const GO srcGblRowInd = srcRowMap.getGlobalElement(lclRowInd);
5084 const GO tgtGblRowInd = tgtRowMap.getGlobalElement(lclRowInd);
5085 auto srcGblColInds = getRowGraphGlobalRow(
5086 srcGblColIndsScratch, source, srcGblRowInd);
5087 auto tgtGblColInds = getRowGraphGlobalRow(
5088 tgtGblColIndsScratch, *
this, tgtGblRowInd);
5089 padding.update_same(lclRowInd, tgtGblColInds.getRawPtr(),
5090 tgtGblColInds.size(), tgt_is_unique,
5091 srcGblColInds.getRawPtr(),
5092 srcGblColInds.size(), src_is_unique);
5095 std::ostringstream os;
5096 os << *prefix <<
"Done" << endl;
5097 std::cerr << os.str();
5101 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5103 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5104 computeCrsPaddingForPermutedIDs(
5105 padding_type& padding,
5106 const RowGraph<local_ordinal_type, global_ordinal_type,
5108 const Kokkos::DualView<
const local_ordinal_type*,
5109 buffer_device_type>& permuteToLIDs,
5110 const Kokkos::DualView<
const local_ordinal_type*,
5111 buffer_device_type>& permuteFromLIDs)
const
5114 using GO = global_ordinal_type;
5115 using Details::Impl::getRowGraphGlobalRow;
5117 const char tfecfFuncName[] =
"computeCrsPaddingForPermutedIds";
5119 std::unique_ptr<std::string> prefix;
5120 const bool verbose = verbose_;
5122 prefix = this->
createPrefix(
"CrsGraph", tfecfFuncName);
5123 std::ostringstream os;
5124 os << *prefix <<
"permuteToLIDs.extent(0): "
5125 << permuteToLIDs.extent(0)
5126 <<
", permuteFromLIDs.extent(0): "
5127 << permuteFromLIDs.extent(0) << endl;
5128 std::cerr << os.str();
5131 if (permuteToLIDs.extent(0) == 0) {
5135 const map_type& srcRowMap = *(source.getRowMap());
5136 const map_type& tgtRowMap = *rowMap_;
5137 using this_CRS_type = CrsGraph<LocalOrdinal, GlobalOrdinal, Node>;
5138 const this_CRS_type* srcCrs =
dynamic_cast<const this_CRS_type*
>(&source);
5139 const bool src_is_unique =
5140 srcCrs ==
nullptr ?
false : srcCrs->isMerged();
5141 const bool tgt_is_unique = this->isMerged();
5143 TEUCHOS_ASSERT( ! permuteToLIDs.need_sync_host() );
5144 auto permuteToLIDs_h = permuteToLIDs.view_host();
5145 TEUCHOS_ASSERT( ! permuteFromLIDs.need_sync_host() );
5146 auto permuteFromLIDs_h = permuteFromLIDs.view_host();
5148 std::vector<GO> srcGblColIndsScratch;
5149 std::vector<GO> tgtGblColIndsScratch;
5150 const LO numPermutes =
static_cast<LO
>(permuteToLIDs_h.extent(0));
5152 execute_sync_host_uvm_access();
5153 for (LO whichPermute = 0; whichPermute < numPermutes; ++whichPermute) {
5154 const LO srcLclRowInd = permuteFromLIDs_h[whichPermute];
5155 const GO srcGblRowInd = srcRowMap.getGlobalElement(srcLclRowInd);
5156 auto srcGblColInds = getRowGraphGlobalRow(
5157 srcGblColIndsScratch, source, srcGblRowInd);
5158 const LO tgtLclRowInd = permuteToLIDs_h[whichPermute];
5159 const GO tgtGblRowInd = tgtRowMap.getGlobalElement(tgtLclRowInd);
5160 auto tgtGblColInds = getRowGraphGlobalRow(
5161 tgtGblColIndsScratch, *
this, tgtGblRowInd);
5162 padding.update_permute(whichPermute, tgtLclRowInd,
5163 tgtGblColInds.getRawPtr(),
5164 tgtGblColInds.size(), tgt_is_unique,
5165 srcGblColInds.getRawPtr(),
5166 srcGblColInds.size(), src_is_unique);
5170 std::ostringstream os;
5171 os << *prefix <<
"Done" << endl;
5172 std::cerr << os.str();
5176 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5178 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type
5180 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5181 computeCrsPaddingForImports(
5182 const Kokkos::DualView<
const local_ordinal_type*,
5183 buffer_device_type>& importLIDs,
5184 Kokkos::DualView<packet_type*, buffer_device_type> imports,
5185 Kokkos::DualView<size_t*, buffer_device_type> numPacketsPerLID,
5186 const bool verbose)
const
5188 using Details::Impl::getRowGraphGlobalRow;
5191 using GO = global_ordinal_type;
5192 const char tfecfFuncName[] =
"computeCrsPaddingForImports";
5194 std::unique_ptr<std::string> prefix;
5196 prefix = this->
createPrefix(
"CrsGraph", tfecfFuncName);
5197 std::ostringstream os;
5198 os << *prefix <<
"importLIDs.extent(0): "
5199 << importLIDs.extent(0)
5200 <<
", imports.extent(0): "
5201 << imports.extent(0)
5202 <<
", numPacketsPerLID.extent(0): "
5203 << numPacketsPerLID.extent(0) << endl;
5204 std::cerr << os.str();
5207 const LO numImports =
static_cast<LO
>(importLIDs.extent(0));
5208 const int myRank = [&] () {
5209 auto comm = rowMap_.is_null() ? Teuchos::null :
5211 return comm.is_null() ? -1 : comm->getRank();
5213 std::unique_ptr<padding_type> padding(
5214 new padding_type(myRank, numImports));
5216 if (imports.need_sync_host()) {
5217 imports.sync_host();
5219 auto imports_h = imports.view_host();
5220 if (numPacketsPerLID.need_sync_host ()) {
5221 numPacketsPerLID.sync_host();
5223 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
5225 TEUCHOS_ASSERT( ! importLIDs.need_sync_host() );
5226 auto importLIDs_h = importLIDs.view_host();
5228 const map_type& tgtRowMap = *rowMap_;
5232 constexpr
bool src_is_unique =
false;
5233 const bool tgt_is_unique = isMerged();
5235 std::vector<GO> tgtGblColIndsScratch;
5237 execute_sync_host_uvm_access();
5238 for (LO whichImport = 0; whichImport < numImports; ++whichImport) {
5243 const LO origSrcNumEnt =
5244 static_cast<LO
>(numPacketsPerLID_h[whichImport]);
5245 GO*
const srcGblColInds = imports_h.data() + offset;
5247 const LO tgtLclRowInd = importLIDs_h[whichImport];
5248 const GO tgtGblRowInd =
5249 tgtRowMap.getGlobalElement(tgtLclRowInd);
5250 auto tgtGblColInds = getRowGraphGlobalRow(
5251 tgtGblColIndsScratch, *
this, tgtGblRowInd);
5252 const size_t origTgtNumEnt(tgtGblColInds.size());
5254 padding->update_import(whichImport, tgtLclRowInd,
5255 tgtGblColInds.getRawPtr(),
5256 origTgtNumEnt, tgt_is_unique,
5258 origSrcNumEnt, src_is_unique);
5259 offset += origSrcNumEnt;
5263 std::ostringstream os;
5264 os << *prefix <<
"Done" << endl;
5265 std::cerr << os.str();
5270 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5272 typename CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::padding_type
5274 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5275 computePaddingForCrsMatrixUnpack(
5276 const Kokkos::DualView<
const local_ordinal_type*,
5277 buffer_device_type>& importLIDs,
5278 Kokkos::DualView<char*, buffer_device_type> imports,
5279 Kokkos::DualView<size_t*, buffer_device_type> numPacketsPerLID,
5280 const bool verbose)
const
5282 using Details::Impl::getRowGraphGlobalRow;
5283 using Details::PackTraits;
5286 using GO = global_ordinal_type;
5287 const char tfecfFuncName[] =
"computePaddingForCrsMatrixUnpack";
5289 std::unique_ptr<std::string> prefix;
5291 prefix = this->
createPrefix(
"CrsGraph", tfecfFuncName);
5292 std::ostringstream os;
5293 os << *prefix <<
"importLIDs.extent(0): "
5294 << importLIDs.extent(0)
5295 <<
", imports.extent(0): "
5296 << imports.extent(0)
5297 <<
", numPacketsPerLID.extent(0): "
5298 << numPacketsPerLID.extent(0) << endl;
5299 std::cerr << os.str();
5301 const bool extraVerbose =
5304 const LO numImports =
static_cast<LO
>(importLIDs.extent(0));
5305 TEUCHOS_ASSERT( LO(numPacketsPerLID.extent(0)) >= numImports );
5306 const int myRank = [&] () {
5307 auto comm = rowMap_.is_null() ? Teuchos::null :
5309 return comm.is_null() ? -1 : comm->getRank();
5311 std::unique_ptr<padding_type> padding(
5312 new padding_type(myRank, numImports));
5314 if (imports.need_sync_host()) {
5315 imports.sync_host();
5317 auto imports_h = imports.view_host();
5318 if (numPacketsPerLID.need_sync_host ()) {
5319 numPacketsPerLID.sync_host();
5321 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
5323 TEUCHOS_ASSERT( ! importLIDs.need_sync_host() );
5324 auto importLIDs_h = importLIDs.view_host();
5326 const map_type& tgtRowMap = *rowMap_;
5330 constexpr
bool src_is_unique =
false;
5331 const bool tgt_is_unique = isMerged();
5333 std::vector<GO> srcGblColIndsScratch;
5334 std::vector<GO> tgtGblColIndsScratch;
5336 execute_sync_host_uvm_access();
5337 for (LO whichImport = 0; whichImport < numImports; ++whichImport) {
5342 const size_t numBytes = numPacketsPerLID_h[whichImport];
5344 std::ostringstream os;
5345 os << *prefix <<
"whichImport=" << whichImport
5346 <<
", numImports=" << numImports
5347 <<
", numBytes=" << numBytes << endl;
5348 std::cerr << os.str();
5350 if (numBytes == 0) {
5353 LO origSrcNumEnt = 0;
5354 const size_t numEntBeg = offset;
5355 const size_t numEntLen =
5357 TEUCHOS_ASSERT( numBytes >= numEntLen );
5358 TEUCHOS_ASSERT( imports_h.extent(0) >= numEntBeg + numEntLen );
5360 imports_h.data() + numEntBeg);
5362 std::ostringstream os;
5363 os << *prefix <<
"whichImport=" << whichImport
5364 <<
", numImports=" << numImports
5365 <<
", origSrcNumEnt=" << origSrcNumEnt << endl;
5366 std::cerr << os.str();
5368 TEUCHOS_ASSERT( origSrcNumEnt >= LO(0) );
5369 TEUCHOS_ASSERT( numBytes >=
size_t(numEntLen + origSrcNumEnt *
sizeof(GO)) );
5370 const size_t gidsBeg = numEntBeg + numEntLen;
5371 if (srcGblColIndsScratch.size() < size_t(origSrcNumEnt)) {
5372 srcGblColIndsScratch.resize(origSrcNumEnt);
5374 GO*
const srcGblColInds = srcGblColIndsScratch.data();
5375 PackTraits<GO>::unpackArray(srcGblColInds,
5376 imports_h.data() + gidsBeg,
5378 const LO tgtLclRowInd = importLIDs_h[whichImport];
5379 const GO tgtGblRowInd =
5380 tgtRowMap.getGlobalElement(tgtLclRowInd);
5381 auto tgtGblColInds = getRowGraphGlobalRow(
5382 tgtGblColIndsScratch, *
this, tgtGblRowInd);
5383 const size_t origNumTgtEnt(tgtGblColInds.size());
5386 std::ostringstream os;
5387 os << *prefix <<
"whichImport=" << whichImport
5388 <<
", numImports=" << numImports
5389 <<
": Call padding->update_import" << endl;
5390 std::cerr << os.str();
5392 padding->update_import(whichImport, tgtLclRowInd,
5393 tgtGblColInds.getRawPtr(),
5394 origNumTgtEnt, tgt_is_unique,
5396 origSrcNumEnt, src_is_unique);
5401 std::ostringstream os;
5402 os << *prefix <<
"Done" << endl;
5403 std::cerr << os.str();
5408 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5410 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5412 (
const SrcDistObject& source,
5413 const Kokkos::DualView<
const local_ordinal_type*,
5414 buffer_device_type>& exportLIDs,
5415 Kokkos::DualView<packet_type*,
5416 buffer_device_type>& exports,
5417 Kokkos::DualView<
size_t*,
5418 buffer_device_type> numPacketsPerLID,
5419 size_t& constantNumPackets)
5422 using GO = global_ordinal_type;
5424 using crs_graph_type =
5425 CrsGraph<local_ordinal_type, global_ordinal_type, node_type>;
5426 const char tfecfFuncName[] =
"packAndPrepare: ";
5427 ProfilingRegion region_papn (
"Tpetra::CrsGraph::packAndPrepare");
5429 const bool verbose = verbose_;
5430 std::unique_ptr<std::string> prefix;
5432 prefix = this->
createPrefix(
"CrsGraph",
"packAndPrepare");
5433 std::ostringstream os;
5434 os << *prefix <<
"Start" << endl;
5435 std::cerr << os.str();
5438 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5439 (exportLIDs.extent (0) != numPacketsPerLID.extent (0),
5441 "exportLIDs.extent(0) = " << exportLIDs.extent (0)
5442 <<
" != numPacketsPerLID.extent(0) = " << numPacketsPerLID.extent (0)
5444 const row_graph_type* srcRowGraphPtr =
5445 dynamic_cast<const row_graph_type*
> (&source);
5446 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5447 (srcRowGraphPtr ==
nullptr, std::invalid_argument,
"Source of an Export "
5448 "or Import operation to a CrsGraph must be a RowGraph with the same "
5449 "template parameters.");
5453 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5454 (this->isFillComplete (), std::runtime_error,
5455 "The target graph of an Import or Export must not be fill complete.");
5457 const crs_graph_type* srcCrsGraphPtr =
5458 dynamic_cast<const crs_graph_type*
> (&source);
5460 if (srcCrsGraphPtr ==
nullptr) {
5461 using Teuchos::ArrayView;
5465 std::ostringstream os;
5466 os << *prefix <<
"Source is a RowGraph but not a CrsGraph"
5468 std::cerr << os.str();
5475 TEUCHOS_ASSERT( ! exportLIDs.need_sync_host () );
5476 auto exportLIDs_h = exportLIDs.view_host ();
5477 ArrayView<const LO> exportLIDs_av (exportLIDs_h.data (),
5478 exportLIDs_h.extent (0));
5479 Teuchos::Array<GO> exports_a;
5481 numPacketsPerLID.clear_sync_state ();
5482 numPacketsPerLID.modify_host ();
5483 auto numPacketsPerLID_h = numPacketsPerLID.view_host ();
5484 ArrayView<size_t> numPacketsPerLID_av (numPacketsPerLID_h.data (),
5485 numPacketsPerLID_h.extent (0));
5486 srcRowGraphPtr->pack (exportLIDs_av, exports_a, numPacketsPerLID_av,
5487 constantNumPackets);
5488 const size_t newSize =
static_cast<size_t> (exports_a.size ());
5489 if (static_cast<size_t> (exports.extent (0)) != newSize) {
5490 using exports_dv_type = Kokkos::DualView<packet_type*, buffer_device_type>;
5491 exports = exports_dv_type (
"exports", newSize);
5493 Kokkos::View<
const packet_type*, Kokkos::HostSpace,
5494 Kokkos::MemoryUnmanaged> exports_a_h (exports_a.getRawPtr (), newSize);
5495 exports.clear_sync_state ();
5496 exports.modify_host ();
5501 else if (! getColMap ().is_null () &&
5502 (rowPtrsPacked_dev_.extent (0) != 0 ||
5503 getRowMap ()->getLocalNumElements () == 0)) {
5505 std::ostringstream os;
5506 os << *prefix <<
"packCrsGraphNew path" << endl;
5507 std::cerr << os.str();
5509 using export_pids_type =
5510 Kokkos::DualView<const int*, buffer_device_type>;
5511 export_pids_type exportPIDs;
5513 using NT = node_type;
5515 packCrsGraphNew<LO,GO,NT> (*srcCrsGraphPtr, exportLIDs, exportPIDs,
5516 exports, numPacketsPerLID,
5517 constantNumPackets,
false);
5520 srcCrsGraphPtr->packFillActiveNew (exportLIDs, exports, numPacketsPerLID,
5521 constantNumPackets);
5525 std::ostringstream os;
5526 os << *prefix <<
"Done" << endl;
5527 std::cerr << os.str();
5531 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5534 pack (
const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
5535 Teuchos::Array<GlobalOrdinal>& exports,
5536 const Teuchos::ArrayView<size_t>& numPacketsPerLID,
5537 size_t& constantNumPackets)
const
5539 auto col_map = this->getColMap();
5541 if( !col_map.is_null() && (rowPtrsPacked_dev_.extent(0) != 0 || getRowMap()->getLocalNumElements() ==0)) {
5543 packCrsGraph<LocalOrdinal,GlobalOrdinal,Node>(*
this, exports, numPacketsPerLID,
5544 exportLIDs, constantNumPackets);
5547 this->packFillActive(exportLIDs, exports, numPacketsPerLID,
5548 constantNumPackets);
5552 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5555 packFillActive (
const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
5556 Teuchos::Array<GlobalOrdinal>& exports,
5557 const Teuchos::ArrayView<size_t>& numPacketsPerLID,
5558 size_t& constantNumPackets)
const
5561 using LO = LocalOrdinal;
5562 using GO = GlobalOrdinal;
5563 using host_execution_space =
5564 typename Kokkos::View<size_t*, device_type>::
5565 HostMirror::execution_space;
5566 const char tfecfFuncName[] =
"packFillActive: ";
5567 const bool verbose = verbose_;
5569 const auto numExportLIDs = exportLIDs.size ();
5570 std::unique_ptr<std::string> prefix;
5572 prefix = this->createPrefix(
"CrsGraph",
"allocateIndices");
5573 std::ostringstream os;
5574 os << *prefix <<
"numExportLIDs=" << numExportLIDs << endl;
5575 std::cerr << os.str();
5577 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5578 (numExportLIDs != numPacketsPerLID.size (), std::runtime_error,
5579 "exportLIDs.size() = " << numExportLIDs <<
" != numPacketsPerLID.size()"
5580 " = " << numPacketsPerLID.size () <<
".");
5582 const map_type& rowMap = * (this->getRowMap ());
5583 const map_type*
const colMapPtr = this->colMap_.getRawPtr ();
5584 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5585 (this->isLocallyIndexed () && colMapPtr ==
nullptr, std::logic_error,
5586 "This graph claims to be locally indexed, but its column Map is nullptr. "
5587 "This should never happen. Please report this bug to the Tpetra "
5591 constantNumPackets = 0;
5595 size_t*
const numPacketsPerLID_raw = numPacketsPerLID.getRawPtr ();
5596 const LO*
const exportLIDs_raw = exportLIDs.getRawPtr ();
5603 Kokkos::RangePolicy<host_execution_space, LO> inputRange (0, numExportLIDs);
5604 size_t totalNumPackets = 0;
5605 size_t errCount = 0;
5608 typedef Kokkos::Device<host_execution_space, Kokkos::HostSpace>
5610 Kokkos::View<size_t, host_device_type> errCountView (&errCount);
5611 constexpr
size_t ONE = 1;
5613 execute_sync_host_uvm_access();
5614 Kokkos::parallel_reduce (
"Tpetra::CrsGraph::pack: totalNumPackets",
5616 [=] (
const LO& i,
size_t& curTotalNumPackets) {
5617 const GO gblRow = rowMap.getGlobalElement (exportLIDs_raw[i]);
5618 if (gblRow == Tpetra::Details::OrdinalTraits<GO>::invalid ()) {
5619 Kokkos::atomic_add (&errCountView(), ONE);
5620 numPacketsPerLID_raw[i] = 0;
5623 const size_t numEnt = this->getNumEntriesInGlobalRow (gblRow);
5624 numPacketsPerLID_raw[i] = numEnt;
5625 curTotalNumPackets += numEnt;
5631 std::ostringstream os;
5632 os << *prefix <<
"totalNumPackets=" << totalNumPackets << endl;
5633 std::cerr << os.str();
5635 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5636 (errCount != 0, std::logic_error,
"totalNumPackets count encountered "
5637 "one or more errors! errCount = " << errCount
5638 <<
", totalNumPackets = " << totalNumPackets <<
".");
5642 exports.resize (totalNumPackets);
5644 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5645 (! this->supportsRowViews (), std::logic_error,
5646 "this->supportsRowViews() returns false; this should never happen. "
5647 "Please report this bug to the Tpetra developers.");
5653 std::ostringstream os;
5654 os << *prefix <<
"Pack into exports" << endl;
5655 std::cerr << os.str();
5660 GO*
const exports_raw = exports.getRawPtr ();
5662 Kokkos::parallel_scan (
"Tpetra::CrsGraph::pack: pack from views",
5663 inputRange, [=, &prefix]
5664 (
const LO i,
size_t& exportsOffset,
const bool final) {
5665 const size_t curOffset = exportsOffset;
5666 const GO gblRow = rowMap.getGlobalElement (exportLIDs_raw[i]);
5667 const RowInfo rowInfo =
5668 this->getRowInfoFromGlobalRowIndex (gblRow);
5670 using TDO = Tpetra::Details::OrdinalTraits<size_t>;
5671 if (rowInfo.localRow == TDO::invalid ()) {
5673 std::ostringstream os;
5674 os << *prefix <<
": INVALID rowInfo: i=" << i
5675 <<
", lclRow=" << exportLIDs_raw[i] << endl;
5676 std::cerr << os.str();
5678 Kokkos::atomic_add (&errCountView(), ONE);
5680 else if (curOffset + rowInfo.numEntries > totalNumPackets) {
5682 std::ostringstream os;
5683 os << *prefix <<
": UH OH! For i=" << i <<
", lclRow="
5684 << exportLIDs_raw[i] <<
", gblRow=" << gblRow <<
", curOffset "
5685 "(= " << curOffset <<
") + numEnt (= " << rowInfo.numEntries
5686 <<
") > totalNumPackets (= " << totalNumPackets <<
")."
5688 std::cerr << os.str();
5690 Kokkos::atomic_add (&errCountView(), ONE);
5693 const LO numEnt =
static_cast<LO
> (rowInfo.numEntries);
5694 if (this->isLocallyIndexed ()) {
5695 auto lclColInds = getLocalIndsViewHost (rowInfo);
5697 for (LO k = 0; k < numEnt; ++k) {
5698 const LO lclColInd = lclColInds(k);
5699 const GO gblColInd = colMapPtr->getGlobalElement (lclColInd);
5703 exports_raw[curOffset + k] = gblColInd;
5706 exportsOffset = curOffset + numEnt;
5708 else if (this->isGloballyIndexed ()) {
5709 auto gblColInds = getGlobalIndsViewHost (rowInfo);
5711 for (LO k = 0; k < numEnt; ++k) {
5712 const GO gblColInd = gblColInds(k);
5716 exports_raw[curOffset + k] = gblColInd;
5719 exportsOffset = curOffset + numEnt;
5727 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5728 (errCount != 0, std::logic_error,
"Packing encountered "
5729 "one or more errors! errCount = " << errCount
5730 <<
", totalNumPackets = " << totalNumPackets <<
".");
5733 std::ostringstream os;
5734 os << *prefix <<
"Done" << endl;
5735 std::cerr << os.str();
5739 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5741 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5742 packFillActiveNew (
const Kokkos::DualView<
const local_ordinal_type*,
5743 buffer_device_type>& exportLIDs,
5744 Kokkos::DualView<packet_type*,
5745 buffer_device_type>& exports,
5746 Kokkos::DualView<
size_t*,
5747 buffer_device_type> numPacketsPerLID,
5748 size_t& constantNumPackets)
const
5752 using GO = global_ordinal_type;
5753 using host_execution_space =
typename Kokkos::View<
size_t*,
5754 device_type>::HostMirror::execution_space;
5755 using host_device_type =
5756 Kokkos::Device<host_execution_space, Kokkos::HostSpace>;
5757 using exports_dv_type =
5758 Kokkos::DualView<packet_type*, buffer_device_type>;
5759 const char tfecfFuncName[] =
"packFillActiveNew: ";
5760 const bool verbose = verbose_;
5762 const auto numExportLIDs = exportLIDs.extent (0);
5763 std::unique_ptr<std::string> prefix;
5765 prefix = this->
createPrefix(
"CrsGraph",
"packFillActiveNew");
5766 std::ostringstream os;
5767 os << *prefix <<
"numExportLIDs: " << numExportLIDs
5768 <<
", numPacketsPerLID.extent(0): "
5769 << numPacketsPerLID.extent(0) << endl;
5770 std::cerr << os.str();
5772 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5773 (numExportLIDs != numPacketsPerLID.extent (0), std::runtime_error,
5774 "exportLIDs.extent(0) = " << numExportLIDs
5775 <<
" != numPacketsPerLID.extent(0) = "
5776 << numPacketsPerLID.extent (0) <<
".");
5777 TEUCHOS_ASSERT( ! exportLIDs.need_sync_host () );
5778 auto exportLIDs_h = exportLIDs.view_host ();
5780 const map_type& rowMap = * (this->getRowMap ());
5781 const map_type*
const colMapPtr = this->colMap_.getRawPtr ();
5782 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5783 (this->isLocallyIndexed () && colMapPtr ==
nullptr, std::logic_error,
5784 "This graph claims to be locally indexed, but its column Map is nullptr. "
5785 "This should never happen. Please report this bug to the Tpetra "
5789 constantNumPackets = 0;
5791 numPacketsPerLID.clear_sync_state ();
5792 numPacketsPerLID.modify_host ();
5793 auto numPacketsPerLID_h = numPacketsPerLID.view_host ();
5800 using range_type = Kokkos::RangePolicy<host_execution_space, LO>;
5801 range_type inputRange (0, numExportLIDs);
5802 size_t totalNumPackets = 0;
5803 size_t errCount = 0;
5806 Kokkos::View<size_t, host_device_type> errCountView (&errCount);
5807 constexpr
size_t ONE = 1;
5810 std::ostringstream os;
5811 os << *prefix <<
"Compute totalNumPackets" << endl;
5812 std::cerr << os.str ();
5815 execute_sync_host_uvm_access();
5816 Kokkos::parallel_reduce
5817 (
"Tpetra::CrsGraph::pack: totalNumPackets",
5819 [=, &prefix] (
const LO i,
size_t& curTotalNumPackets) {
5820 const LO lclRow = exportLIDs_h[i];
5821 const GO gblRow = rowMap.getGlobalElement (lclRow);
5822 if (gblRow == Tpetra::Details::OrdinalTraits<GO>::invalid ()) {
5824 std::ostringstream os;
5825 os << *prefix <<
"For i=" << i <<
", lclRow=" << lclRow
5826 <<
" not in row Map on this process" << endl;
5827 std::cerr << os.str();
5829 Kokkos::atomic_add (&errCountView(), ONE);
5830 numPacketsPerLID_h(i) = 0;
5833 const size_t numEnt = this->getNumEntriesInGlobalRow (gblRow);
5834 numPacketsPerLID_h(i) = numEnt;
5835 curTotalNumPackets += numEnt;
5841 std::ostringstream os;
5842 os << *prefix <<
"totalNumPackets: " << totalNumPackets
5843 <<
", errCount: " << errCount << endl;
5844 std::cerr << os.str ();
5846 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5847 (errCount != 0, std::logic_error,
"totalNumPackets count encountered "
5848 "one or more errors! totalNumPackets: " << totalNumPackets
5849 <<
", errCount: " << errCount <<
".");
5852 if (
size_t(exports.extent (0)) < totalNumPackets) {
5854 exports = exports_dv_type (
"exports", totalNumPackets);
5857 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5858 (! this->supportsRowViews (), std::logic_error,
5859 "this->supportsRowViews() returns false; this should never happen. "
5860 "Please report this bug to the Tpetra developers.");
5866 std::ostringstream os;
5867 os << *prefix <<
"Pack into exports buffer" << endl;
5868 std::cerr << os.str();
5871 exports.clear_sync_state ();
5872 exports.modify_host ();
5873 auto exports_h = exports.view_host ();
5876 Kokkos::parallel_scan
5877 (
"Tpetra::CrsGraph::packFillActiveNew: Pack exports",
5878 inputRange, [=, &prefix]
5879 (
const LO i,
size_t& exportsOffset,
const bool final) {
5880 const size_t curOffset = exportsOffset;
5881 const LO lclRow = exportLIDs_h(i);
5882 const GO gblRow = rowMap.getGlobalElement (lclRow);
5883 if (gblRow == Details::OrdinalTraits<GO>::invalid ()) {
5885 std::ostringstream os;
5886 os << *prefix <<
"For i=" << i <<
", lclRow=" << lclRow
5887 <<
" not in row Map on this process" << endl;
5888 std::cerr << os.str();
5890 Kokkos::atomic_add (&errCountView(), ONE);
5894 const RowInfo rowInfo = this->getRowInfoFromGlobalRowIndex (gblRow);
5895 if (rowInfo.localRow == Details::OrdinalTraits<size_t>::invalid ()) {
5897 std::ostringstream os;
5898 os << *prefix <<
"For i=" << i <<
", lclRow=" << lclRow
5899 <<
", gblRow=" << gblRow <<
": invalid rowInfo"
5901 std::cerr << os.str();
5903 Kokkos::atomic_add (&errCountView(), ONE);
5907 if (curOffset + rowInfo.numEntries > totalNumPackets) {
5909 std::ostringstream os;
5910 os << *prefix <<
"For i=" << i <<
", lclRow=" << lclRow
5911 <<
", gblRow=" << gblRow <<
", curOffset (= "
5912 << curOffset <<
") + numEnt (= " << rowInfo.numEntries
5913 <<
") > totalNumPackets (= " << totalNumPackets
5915 std::cerr << os.str();
5917 Kokkos::atomic_add (&errCountView(), ONE);
5921 const LO numEnt =
static_cast<LO
> (rowInfo.numEntries);
5922 if (this->isLocallyIndexed ()) {
5923 auto lclColInds = getLocalIndsViewHost(rowInfo);
5925 for (LO k = 0; k < numEnt; ++k) {
5926 const LO lclColInd = lclColInds(k);
5927 const GO gblColInd = colMapPtr->getGlobalElement (lclColInd);
5931 exports_h(curOffset + k) = gblColInd;
5934 exportsOffset = curOffset + numEnt;
5936 else if (this->isGloballyIndexed ()) {
5937 auto gblColInds = getGlobalIndsViewHost(rowInfo);
5939 for (LO k = 0; k < numEnt; ++k) {
5940 const GO gblColInd = gblColInds(k);
5944 exports_h(curOffset + k) = gblColInd;
5947 exportsOffset = curOffset + numEnt;
5960 std::ostringstream os;
5961 os << *prefix <<
"errCount=" << errCount <<
"; Done" << endl;
5962 std::cerr << os.str();
5966 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
5968 CrsGraph<LocalOrdinal, GlobalOrdinal, Node>::
5970 (
const Kokkos::DualView<
const local_ordinal_type*,
5971 buffer_device_type>& importLIDs,
5972 Kokkos::DualView<packet_type*,
5973 buffer_device_type> imports,
5974 Kokkos::DualView<
size_t*,
5975 buffer_device_type> numPacketsPerLID,
5979 using Details::ProfilingRegion;
5982 using GO = global_ordinal_type;
5983 const char tfecfFuncName[] =
"unpackAndCombine";
5985 ProfilingRegion regionCGC(
"Tpetra::CrsGraph::unpackAndCombine");
5986 const bool verbose = verbose_;
5988 std::unique_ptr<std::string> prefix;
5990 prefix = this->
createPrefix(
"CrsGraph", tfecfFuncName);
5991 std::ostringstream os;
5992 os << *prefix <<
"Start" << endl;
5993 std::cerr << os.str ();
5996 auto padding = computeCrsPaddingForImports(
5997 importLIDs, imports, numPacketsPerLID, verbose);
5998 applyCrsPadding(*padding, verbose);
6000 std::ostringstream os;
6001 os << *prefix <<
"Done computing & applying padding" << endl;
6002 std::cerr << os.str ();
6023 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6024 (importLIDs.extent (0) != numPacketsPerLID.extent (0),
6025 std::runtime_error,
": importLIDs.extent(0) = "
6026 << importLIDs.extent (0) <<
" != numPacketsPerLID.extent(0) = "
6027 << numPacketsPerLID.extent (0) <<
".");
6028 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6029 (isFillComplete (), std::runtime_error,
6030 ": Import or Export operations are not allowed on a target "
6031 "CrsGraph that is fillComplete.");
6033 const size_t numImportLIDs(importLIDs.extent(0));
6034 if (numPacketsPerLID.need_sync_host()) {
6035 numPacketsPerLID.sync_host();
6037 auto numPacketsPerLID_h = numPacketsPerLID.view_host();
6038 if (imports.need_sync_host()) {
6039 imports.sync_host();
6041 auto imports_h = imports.view_host();
6042 TEUCHOS_ASSERT( ! importLIDs.need_sync_host() );
6043 auto importLIDs_h = importLIDs.view_host();
6046 Teuchos::Array<LO> lclColInds;
6047 if (isLocallyIndexed()) {
6049 std::ostringstream os;
6050 os << *prefix <<
"Preallocate local indices scratch" << endl;
6051 std::cerr << os.str();
6053 size_t maxNumInserts = 0;
6054 for (
size_t i = 0; i < numImportLIDs; ++i) {
6055 maxNumInserts = std::max (maxNumInserts, numPacketsPerLID_h[i]);
6058 std::ostringstream os;
6059 os << *prefix <<
"Local indices scratch size: "
6060 << maxNumInserts << endl;
6061 std::cerr << os.str();
6063 lclColInds.resize (maxNumInserts);
6067 std::ostringstream os;
6069 if (isGloballyIndexed()) {
6070 os <<
"Graph is globally indexed";
6073 os <<
"Graph is neither locally nor globally indexed";
6076 std::cerr << os.str();
6080 TEUCHOS_ASSERT( ! rowMap_.is_null() );
6081 const map_type& rowMap = *rowMap_;
6084 size_t importsOffset = 0;
6085 for (
size_t i = 0; i < numImportLIDs; ++i) {
6087 std::ostringstream os;
6088 os << *prefix <<
"i=" << i <<
", numImportLIDs="
6089 << numImportLIDs << endl;
6090 std::cerr << os.str();
6094 const LO lclRow = importLIDs_h[i];
6095 const GO gblRow = rowMap.getGlobalElement(lclRow);
6096 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6097 (gblRow == Teuchos::OrdinalTraits<GO>::invalid(),
6098 std::logic_error,
"importLIDs[i=" << i <<
"]="
6099 << lclRow <<
" is not in the row Map on the calling "
6101 const LO numEnt = numPacketsPerLID_h[i];
6102 const GO*
const gblColInds = (numEnt == 0) ?
nullptr :
6103 imports_h.data() + importsOffset;
6104 if (! isLocallyIndexed()) {
6105 insertGlobalIndicesFiltered(lclRow, gblColInds, numEnt);
6110 for (LO j = 0; j < numEnt; j++) {
6111 lclColInds[j] = colMap_->getLocalElement(gblColInds[j]);
6113 insertLocalIndices(lclRow, numEnt, lclColInds.data());
6115 importsOffset += numEnt;
6118 catch (std::exception& e) {
6119 TEUCHOS_TEST_FOR_EXCEPTION
6120 (
true, std::runtime_error,
6121 "Tpetra::CrsGraph::unpackAndCombine: Insert loop threw an "
6122 "exception: " << endl << e.what());
6126 std::ostringstream os;
6127 os << *prefix <<
"Done" << endl;
6128 std::cerr << os.str();
6132 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
6137 using Teuchos::Comm;
6138 using Teuchos::null;
6139 using Teuchos::ParameterList;
6145 RCP<const map_type> rowMap, domainMap, rangeMap, colMap;
6146 RCP<import_type> importer;
6147 RCP<export_type> exporter;
6150 RCP<const Comm<int> > newComm =
6151 (newMap.is_null ()) ? null : newMap->getComm ();
6153 if (! domainMap_.is_null ()) {
6154 if (domainMap_.getRawPtr () == rowMap_.getRawPtr ()) {
6161 domainMap = domainMap_->replaceCommWithSubset (newComm);
6164 if (! rangeMap_.is_null ()) {
6165 if (rangeMap_.getRawPtr () == rowMap_.getRawPtr ()) {
6172 rangeMap = rangeMap_->replaceCommWithSubset (newComm);
6175 if (! colMap_.is_null ()) {
6176 colMap = colMap_->replaceCommWithSubset (newComm);
6180 if (! newComm.is_null ()) {
6181 RCP<ParameterList> params = this->getNonconstParameterList ();
6189 if (! rangeMap_.is_null () &&
6190 rangeMap != rowMap &&
6191 ! rangeMap->isSameAs (*rowMap)) {
6192 if (params.is_null () || ! params->isSublist (
"Export")) {
6193 exporter = rcp (
new export_type (rowMap, rangeMap));
6196 RCP<ParameterList> exportSublist = sublist (params,
"Export",
true);
6197 exporter = rcp (
new export_type (rowMap, rangeMap, exportSublist));
6201 if (! domainMap_.is_null () &&
6202 domainMap != colMap &&
6203 ! domainMap->isSameAs (*colMap)) {
6204 if (params.is_null () || ! params->isSublist (
"Import")) {
6205 importer = rcp (
new import_type (domainMap, colMap));
6207 RCP<ParameterList> importSublist = sublist (params,
"Import",
true);
6208 importer = rcp (
new import_type (domainMap, colMap, importSublist));
6216 exporter_ = exporter;
6217 importer_ = importer;
6224 this->map_ = rowMap;
6225 domainMap_ = domainMap;
6226 rangeMap_ = rangeMap;
6230 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
6236 using LO = LocalOrdinal;
6237 using GO = GlobalOrdinal;
6238 const char tfecfFuncName[] =
"getLocalDiagOffsets: ";
6239 const bool verbose = verbose_;
6241 std::unique_ptr<std::string> prefix;
6243 prefix = this->createPrefix(
"CrsGraph",
"getLocalDiagOffsets");
6244 std::ostringstream os;
6245 os << *prefix <<
"offsets.extent(0)=" << offsets.extent(0)
6247 std::cerr << os.str();
6250 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6251 (! hasColMap (), std::runtime_error,
"The graph must have a column Map.");
6252 const LO lclNumRows =
static_cast<LO
> (this->getLocalNumRows ());
6253 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6254 (static_cast<LO> (offsets.extent (0)) < lclNumRows,
6255 std::invalid_argument,
"offsets.extent(0) = " <<
6256 offsets.extent (0) <<
" < getLocalNumRows() = " << lclNumRows <<
".");
6258 const map_type& rowMap = * (this->getRowMap ());
6259 const map_type& colMap = * (this->getColMap ());
6265 bool allRowMapDiagEntriesInColMap =
true;
6266 bool allDiagEntriesFound =
true;
6267 bool allOffsetsCorrect =
true;
6268 bool noOtherWeirdness =
true;
6269 using wrong_offsets_type = std::vector<std::pair<LO, size_t> >;
6270 wrong_offsets_type wrongOffsets(0);
6274 auto lclRowMap = rowMap.getLocalMap ();
6281 const bool sorted = this->isSorted ();
6282 if (isFillComplete ()) {
6283 auto lclGraph = this->getLocalGraphDevice ();
6284 ::Tpetra::Details::getGraphDiagOffsets (offsets, lclRowMap, lclColMap,
6286 lclGraph.entries, sorted);
6292 auto offsets_h = Kokkos::create_mirror_view (offsets);
6294 for (LO lclRowInd = 0; lclRowInd < lclNumRows; ++lclRowInd) {
6298 const GO gblRowInd = lclRowMap.getGlobalElement (lclRowInd);
6299 const GO gblColInd = gblRowInd;
6300 const LO lclColInd = lclColMap.getLocalElement (gblColInd);
6302 if (lclColInd == Tpetra::Details::OrdinalTraits<LO>::invalid ()) {
6303 allRowMapDiagEntriesInColMap =
false;
6304 offsets_h(lclRowInd) = Tpetra::Details::OrdinalTraits<size_t>::invalid ();
6307 const RowInfo rowInfo = this->getRowInfo (lclRowInd);
6308 if (static_cast<LO> (rowInfo.localRow) == lclRowInd &&
6309 rowInfo.numEntries > 0) {
6311 auto colInds = this->getLocalIndsViewHost (rowInfo);
6312 const size_t hint = 0;
6313 const size_t offset =
6314 KokkosSparse::findRelOffset (colInds, rowInfo.numEntries,
6315 lclColInd, hint, sorted);
6316 offsets_h(lclRowInd) = offset;
6323 typename local_inds_dualv_type::t_host::const_type lclColInds;
6325 lclColInds = this->getLocalIndsViewHost (rowInfo);
6328 noOtherWeirdness =
false;
6331 if (noOtherWeirdness) {
6332 const size_t numEnt = lclColInds.extent (0);
6333 if (offset >= numEnt) {
6336 allOffsetsCorrect =
false;
6337 wrongOffsets.push_back (std::make_pair (lclRowInd, offset));
6339 const LO actualLclColInd = lclColInds(offset);
6340 const GO actualGblColInd = lclColMap.getGlobalElement (actualLclColInd);
6341 if (actualGblColInd != gblColInd) {
6342 allOffsetsCorrect =
false;
6343 wrongOffsets.push_back (std::make_pair (lclRowInd, offset));
6350 offsets_h(lclRowInd) = Tpetra::Details::OrdinalTraits<size_t>::invalid ();
6351 allDiagEntriesFound =
false;
6359 if (verbose && wrongOffsets.size () != 0) {
6360 std::ostringstream os;
6361 os << *prefix <<
"Wrong offsets: [";
6362 for (
size_t k = 0; k < wrongOffsets.size (); ++k) {
6363 os <<
"(" << wrongOffsets[k].first <<
","
6364 << wrongOffsets[k].second <<
")";
6365 if (k + 1 < wrongOffsets.size ()) {
6370 std::cerr << os.str();
6374 using Teuchos::reduceAll;
6376 Teuchos::RCP<const Teuchos::Comm<int> > comm = this->getComm ();
6377 const bool localSuccess =
6378 allRowMapDiagEntriesInColMap && allDiagEntriesFound && allOffsetsCorrect;
6379 const int numResults = 5;
6381 lclResults[0] = allRowMapDiagEntriesInColMap ? 1 : 0;
6382 lclResults[1] = allDiagEntriesFound ? 1 : 0;
6383 lclResults[2] = allOffsetsCorrect ? 1 : 0;
6384 lclResults[3] = noOtherWeirdness ? 1 : 0;
6387 lclResults[4] = ! localSuccess ? comm->getRank () : comm->getSize ();
6395 reduceAll<int, int> (*comm, Teuchos::REDUCE_MIN,
6396 numResults, lclResults, gblResults);
6398 if (gblResults[0] != 1 || gblResults[1] != 1 || gblResults[2] != 1
6399 || gblResults[3] != 1) {
6400 std::ostringstream os;
6401 os <<
"Issue(s) that we noticed (on Process " << gblResults[4] <<
", "
6402 "possibly among others): " << endl;
6403 if (gblResults[0] == 0) {
6404 os <<
" - The column Map does not contain at least one diagonal entry "
6405 "of the graph." << endl;
6407 if (gblResults[1] == 0) {
6408 os <<
" - On one or more processes, some row does not contain a "
6409 "diagonal entry." << endl;
6411 if (gblResults[2] == 0) {
6412 os <<
" - On one or more processes, some offsets are incorrect."
6415 if (gblResults[3] == 0) {
6416 os <<
" - One or more processes had some other error."
6419 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
true, std::runtime_error, os.str());
6424 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
6430 const char tfecfFuncName[] =
"getLocalOffRankOffsets: ";
6431 const bool verbose = verbose_;
6433 std::unique_ptr<std::string> prefix;
6435 prefix = this->createPrefix(
"CrsGraph",
"getLocalOffRankOffsets");
6436 std::ostringstream os;
6437 os << *prefix <<
"offsets.extent(0)=" << offsets.extent(0)
6439 std::cerr << os.str();
6442 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6443 (! hasColMap (), std::runtime_error,
"The graph must have a column Map.");
6446 const size_t lclNumRows = this->getLocalNumRows ();
6448 if (haveLocalOffRankOffsets_ && k_offRankOffsets_.extent(0) == lclNumRows+1) {
6449 offsets = k_offRankOffsets_;
6452 haveLocalOffRankOffsets_ =
false;
6453 k_offRankOffsets_ = offset_device_view_type(Kokkos::ViewAllocateWithoutInitializing(
"offRankOffset"), lclNumRows+1);
6454 offsets = k_offRankOffsets_;
6456 const map_type& colMap = * (this->getColMap ());
6457 const map_type& domMap = * (this->getDomainMap ());
6468 TEUCHOS_ASSERT(this->isSorted ());
6469 if (isFillComplete ()) {
6470 auto lclGraph = this->getLocalGraphDevice ();
6471 ::Tpetra::Details::getGraphOffRankOffsets (k_offRankOffsets_,
6472 lclColMap, lclDomMap,
6474 haveLocalOffRankOffsets_ =
true;
6496 template<
class DeviceType,
6497 const bool memSpaceIsHostSpace =
6498 std::is_same<
typename DeviceType::memory_space,
6499 Kokkos::HostSpace>::value>
6500 struct HelpGetLocalDiagOffsets {};
6502 template<
class DeviceType>
6503 struct HelpGetLocalDiagOffsets<DeviceType, true> {
6504 typedef DeviceType device_type;
6505 typedef Kokkos::View<
size_t*, Kokkos::HostSpace,
6506 Kokkos::MemoryUnmanaged> device_offsets_type;
6507 typedef Kokkos::View<
size_t*, Kokkos::HostSpace,
6508 Kokkos::MemoryUnmanaged> host_offsets_type;
6510 static device_offsets_type
6511 getDeviceOffsets (
const host_offsets_type& hostOffsets)
6519 copyBackIfNeeded (
const host_offsets_type& ,
6520 const device_offsets_type& )
6524 template<
class DeviceType>
6525 struct HelpGetLocalDiagOffsets<DeviceType, false> {
6526 typedef DeviceType device_type;
6530 typedef Kokkos::View<size_t*, device_type> device_offsets_type;
6531 typedef Kokkos::View<
size_t*, Kokkos::HostSpace,
6532 Kokkos::MemoryUnmanaged> host_offsets_type;
6534 static device_offsets_type
6535 getDeviceOffsets (
const host_offsets_type& hostOffsets)
6539 return device_offsets_type (
"offsets", hostOffsets.extent (0));
6543 copyBackIfNeeded (
const host_offsets_type& hostOffsets,
6544 const device_offsets_type& deviceOffsets)
6553 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
6558 typedef LocalOrdinal LO;
6559 const char tfecfFuncName[] =
"getLocalDiagOffsets: ";
6560 TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6561 (! this->hasColMap (), std::runtime_error,
6562 "The graph does not yet have a column Map.");
6563 const LO myNumRows =
static_cast<LO
> (this->getLocalNumRows ());
6564 if (static_cast<LO> (offsets.size ()) != myNumRows) {
6568 offsets.resize (myNumRows);
6580 typedef HelpGetLocalDiagOffsets<device_type> helper_type;
6581 typedef typename helper_type::host_offsets_type host_offsets_type;
6583 host_offsets_type hostOffsets (offsets.getRawPtr (), myNumRows);
6585 auto deviceOffsets = helper_type::getDeviceOffsets (hostOffsets);
6587 this->getLocalDiagOffsets (deviceOffsets);
6588 helper_type::copyBackIfNeeded (hostOffsets, deviceOffsets);
6591 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
6598 template <
class LocalOrdinal,
class GlobalOrdinal,
class Node>
6602 const ::Tpetra::Details::Transfer<LocalOrdinal, GlobalOrdinal, Node>& rowTransfer,
6603 const Teuchos::RCP<const ::Tpetra::Details::Transfer<LocalOrdinal, GlobalOrdinal, Node> > & domainTransfer,
6604 const Teuchos::RCP<const map_type>& domainMap,
6605 const Teuchos::RCP<const map_type>& rangeMap,
6606 const Teuchos::RCP<Teuchos::ParameterList>& params)
const
6612 using Teuchos::ArrayRCP;
6613 using Teuchos::ArrayView;
6614 using Teuchos::Comm;
6615 using Teuchos::ParameterList;
6618 #ifdef HAVE_TPETRA_MMM_TIMINGS
6620 using Teuchos::TimeMonitor;
6623 using LO = LocalOrdinal;
6624 using GO = GlobalOrdinal;
6625 using NT = node_type;
6629 const char* prefix =
"Tpetra::CrsGraph::transferAndFillComplete: ";
6631 #ifdef HAVE_TPETRA_MMM_TIMINGS
6633 if(!params.is_null()) label = params->get(
"Timer Label", label);
6634 string prefix2 = string(
"Tpetra ")+ label + std::string(
": CrsGraph TAFC ");
6635 RCP<TimeMonitor> MM =
6636 rcp(
new TimeMonitor(*TimeMonitor::getNewTimer(prefix2+
string(
"Pack-1"))));