Tpetra parallel linear algebra  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Tpetra_CrsMatrix_def.hpp
Go to the documentation of this file.
1 // @HEADER
2 // *****************************************************************************
3 // Tpetra: Templated Linear Algebra Services Package
4 //
5 // Copyright 2008 NTESS and the Tpetra contributors.
6 // SPDX-License-Identifier: BSD-3-Clause
7 // *****************************************************************************
8 // @HEADER
9 
10 #ifndef TPETRA_CRSMATRIX_DEF_HPP
11 #define TPETRA_CRSMATRIX_DEF_HPP
12 
20 
21 #include "Tpetra_Import_Util.hpp"
22 #include "Tpetra_Import_Util2.hpp"
23 #include "Tpetra_RowMatrix.hpp"
24 #include "Tpetra_LocalCrsMatrixOperator.hpp"
25 
26 #include "Tpetra_Details_Behavior.hpp"
27 #include "Tpetra_Details_castAwayConstDualView.hpp"
28 #include "Tpetra_Details_computeOffsets.hpp"
29 #include "Tpetra_Details_copyOffsets.hpp"
30 #include "Tpetra_Details_createMirrorView.hpp"
31 #include "Tpetra_Details_gathervPrint.hpp"
32 #include "Tpetra_Details_getDiagCopyWithoutOffsets.hpp"
33 #include "Tpetra_Details_leftScaleLocalCrsMatrix.hpp"
34 #include "Tpetra_Details_Profiling.hpp"
35 #include "Tpetra_Details_rightScaleLocalCrsMatrix.hpp"
36 #include "Tpetra_Details_ScalarViewTraits.hpp"
37 #include "Tpetra_Details_copyConvert.hpp"
38 #include "Tpetra_Details_iallreduce.hpp"
39 #include "Tpetra_Details_getEntryOnHost.hpp"
40 #include "Tpetra_Details_packCrsMatrix.hpp"
41 #include "Tpetra_Details_unpackCrsMatrixAndCombine.hpp"
42 #include "Tpetra_Details_crsUtils.hpp"
43 #include "Teuchos_FancyOStream.hpp"
44 #include "Teuchos_RCP.hpp"
45 #include "Teuchos_DataAccess.hpp"
46 #include "Teuchos_SerialDenseMatrix.hpp" // unused here, could delete
47 #include "KokkosBlas1_scal.hpp"
48 #include "KokkosSparse_getDiagCopy.hpp"
49 #include "KokkosSparse_spmv.hpp"
50 
51 #include <memory>
52 #include <sstream>
53 #include <typeinfo>
54 #include <utility>
55 #include <vector>
56 
57 namespace Tpetra {
58 
59 namespace { // (anonymous)
60 
61  template<class T, class BinaryFunction>
62  T atomic_binary_function_update (T* const dest,
63  const T& inputVal,
64  BinaryFunction f)
65  {
66  T oldVal = *dest;
67  T assume;
68 
69  // NOTE (mfh 30 Nov 2015) I do NOT need a fence here for IBM
70  // POWER architectures, because 'newval' depends on 'assume',
71  // which depends on 'oldVal', which depends on '*dest'. This
72  // sets up a chain of read dependencies that should ensure
73  // correct behavior given a sane memory model.
74  do {
75  assume = oldVal;
76  T newVal = f (assume, inputVal);
77  oldVal = Kokkos::atomic_compare_exchange (dest, assume, newVal);
78  } while (assume != oldVal);
79 
80  return oldVal;
81  }
82 } // namespace (anonymous)
83 
84 //
85 // Users must never rely on anything in the Details namespace.
86 //
87 namespace Details {
88 
98 template<class Scalar>
99 struct AbsMax {
101  Scalar operator() (const Scalar& x, const Scalar& y) {
102  typedef Teuchos::ScalarTraits<Scalar> STS;
103  return std::max (STS::magnitude (x), STS::magnitude (y));
104  }
105 };
106 
107 } // namespace Details
108 } // namespace Tpetra
109 
110 namespace Tpetra {
111 
112  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
113  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
114  CrsMatrix (const Teuchos::RCP<const map_type>& rowMap,
115  size_t maxNumEntriesPerRow,
116  const Teuchos::RCP<Teuchos::ParameterList>& params) :
117  dist_object_type (rowMap)
118  {
119  const char tfecfFuncName[] = "CrsMatrix(RCP<const Map>, size_t "
120  "[, RCP<ParameterList>]): ";
121  Teuchos::RCP<crs_graph_type> graph;
122  try {
123  graph = Teuchos::rcp (new crs_graph_type (rowMap, maxNumEntriesPerRow,
124  params));
125  }
126  catch (std::exception& e) {
127  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
128  (true, std::runtime_error, "CrsGraph constructor (RCP<const Map>, "
129  "size_t [, RCP<ParameterList>]) threw an exception: "
130  << e.what ());
131  }
132  // myGraph_ not null means that the matrix owns the graph. That's
133  // different than the const CrsGraph constructor, where the matrix
134  // does _not_ own the graph.
135  myGraph_ = graph;
136  staticGraph_ = myGraph_;
137  resumeFill (params);
138  checkInternalState ();
139  }
140 
141  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
142  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
143  CrsMatrix (const Teuchos::RCP<const map_type>& rowMap,
144  const Teuchos::ArrayView<const size_t>& numEntPerRowToAlloc,
145  const Teuchos::RCP<Teuchos::ParameterList>& params) :
146  dist_object_type (rowMap)
147  {
148  const char tfecfFuncName[] = "CrsMatrix(RCP<const Map>, "
149  "ArrayView<const size_t>[, RCP<ParameterList>]): ";
150  Teuchos::RCP<crs_graph_type> graph;
151  try {
152  using Teuchos::rcp;
153  graph = rcp(new crs_graph_type(rowMap, numEntPerRowToAlloc,
154  params));
155  }
156  catch (std::exception& e) {
157  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
158  (true, std::runtime_error, "CrsGraph constructor "
159  "(RCP<const Map>, ArrayView<const size_t>"
160  "[, RCP<ParameterList>]) threw an exception: "
161  << e.what ());
162  }
163  // myGraph_ not null means that the matrix owns the graph. That's
164  // different than the const CrsGraph constructor, where the matrix
165  // does _not_ own the graph.
166  myGraph_ = graph;
167  staticGraph_ = graph;
168  resumeFill (params);
169  checkInternalState ();
170  }
171 
172  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
173  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
174  CrsMatrix (const Teuchos::RCP<const map_type>& rowMap,
175  const Teuchos::RCP<const map_type>& colMap,
176  const size_t maxNumEntPerRow,
177  const Teuchos::RCP<Teuchos::ParameterList>& params) :
178  dist_object_type (rowMap)
179  {
180  const char tfecfFuncName[] = "CrsMatrix(RCP<const Map>, "
181  "RCP<const Map>, size_t[, RCP<ParameterList>]): ";
182  const char suffix[] =
183  " Please report this bug to the Tpetra developers.";
184 
185  // An artifact of debugging something a while back.
186  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
187  (! staticGraph_.is_null (), std::logic_error,
188  "staticGraph_ is not null at the beginning of the constructor."
189  << suffix);
190  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
191  (! myGraph_.is_null (), std::logic_error,
192  "myGraph_ is not null at the beginning of the constructor."
193  << suffix);
194  Teuchos::RCP<crs_graph_type> graph;
195  try {
196  graph = Teuchos::rcp (new crs_graph_type (rowMap, colMap,
197  maxNumEntPerRow,
198  params));
199  }
200  catch (std::exception& e) {
201  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
202  (true, std::runtime_error, "CrsGraph constructor (RCP<const Map>, "
203  "RCP<const Map>, size_t[, RCP<ParameterList>]) threw an "
204  "exception: " << e.what ());
205  }
206  // myGraph_ not null means that the matrix owns the graph. That's
207  // different than the const CrsGraph constructor, where the matrix
208  // does _not_ own the graph.
209  myGraph_ = graph;
210  staticGraph_ = myGraph_;
211  resumeFill (params);
212  checkInternalState ();
213  }
214 
215  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
216  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
217  CrsMatrix (const Teuchos::RCP<const map_type>& rowMap,
218  const Teuchos::RCP<const map_type>& colMap,
219  const Teuchos::ArrayView<const size_t>& numEntPerRowToAlloc,
220  const Teuchos::RCP<Teuchos::ParameterList>& params) :
221  dist_object_type (rowMap)
222  {
223  const char tfecfFuncName[] =
224  "CrsMatrix(RCP<const Map>, RCP<const Map>, "
225  "ArrayView<const size_t>[, RCP<ParameterList>]): ";
226  Teuchos::RCP<crs_graph_type> graph;
227  try {
228  graph = Teuchos::rcp (new crs_graph_type (rowMap, colMap,
229  numEntPerRowToAlloc,
230  params));
231  }
232  catch (std::exception& e) {
233  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
234  (true, std::runtime_error, "CrsGraph constructor (RCP<const Map>, "
235  "RCP<const Map>, ArrayView<const size_t>[, "
236  "RCP<ParameterList>]) threw an exception: " << e.what ());
237  }
238  // myGraph_ not null means that the matrix owns the graph. That's
239  // different than the const CrsGraph constructor, where the matrix
240  // does _not_ own the graph.
241  myGraph_ = graph;
242  staticGraph_ = graph;
243  resumeFill (params);
244  checkInternalState ();
245  }
246 
247 
248  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
249  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
250  CrsMatrix (const Teuchos::RCP<const crs_graph_type>& graph,
251  const Teuchos::RCP<Teuchos::ParameterList>& /* params */) :
252  dist_object_type (graph->getRowMap ()),
253  staticGraph_ (graph),
254  storageStatus_ (Details::STORAGE_1D_PACKED)
255  {
256  using std::endl;
257  typedef typename local_matrix_device_type::values_type values_type;
258  const char tfecfFuncName[] = "CrsMatrix(RCP<const CrsGraph>[, "
259  "RCP<ParameterList>]): ";
260  const bool verbose = Details::Behavior::verbose("CrsMatrix");
261 
262  std::unique_ptr<std::string> prefix;
263  if (verbose) {
264  prefix = this->createPrefix("CrsMatrix", "CrsMatrix(graph,params)");
265  std::ostringstream os;
266  os << *prefix << "Start" << endl;
267  std::cerr << os.str ();
268  }
269 
270  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
271  (graph.is_null (), std::runtime_error, "Input graph is null.");
272  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
273  (! graph->isFillComplete (), std::runtime_error, "Input graph "
274  "is not fill complete. You must call fillComplete on the "
275  "graph before using it to construct a CrsMatrix. Note that "
276  "calling resumeFill on the graph makes it not fill complete, "
277  "even if you had previously called fillComplete. In that "
278  "case, you must call fillComplete on the graph again.");
279 
280  // The graph is fill complete, so it is locally indexed and has a
281  // fixed structure. This means we can allocate the (1-D) array of
282  // values and build the local matrix right now. Note that the
283  // local matrix's number of columns comes from the column Map, not
284  // the domain Map.
285 
286  const size_t numEnt = graph->lclIndsPacked_wdv.extent (0);
287  if (verbose) {
288  std::ostringstream os;
289  os << *prefix << "Allocate values: " << numEnt << endl;
290  std::cerr << os.str ();
291  }
292 
293  values_type val ("Tpetra::CrsMatrix::values", numEnt);
294  valuesPacked_wdv = values_wdv_type(val);
295  valuesUnpacked_wdv = valuesPacked_wdv;
296 
297  checkInternalState ();
298 
299  if (verbose) {
300  std::ostringstream os;
301  os << *prefix << "Done" << endl;
302  std::cerr << os.str ();
303  }
304  }
305 
306  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
307  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
308  CrsMatrix(CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>& matrix,
309  const Teuchos::RCP<const crs_graph_type>& graph,
310  const Teuchos::RCP<Teuchos::ParameterList>& params) :
311  dist_object_type (graph->getRowMap ()),
312  staticGraph_ (graph),
313  storageStatus_ (matrix.storageStatus_)
314  {
315  const char tfecfFuncName[] = "CrsMatrix(RCP<const CrsGraph>, "
316  "local_matrix_device_type::values_type, "
317  "[,RCP<ParameterList>]): ";
318  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
319  (graph.is_null (), std::runtime_error, "Input graph is null.");
320  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
321  (! graph->isFillComplete (), std::runtime_error, "Input graph "
322  "is not fill complete. You must call fillComplete on the "
323  "graph before using it to construct a CrsMatrix. Note that "
324  "calling resumeFill on the graph makes it not fill complete, "
325  "even if you had previously called fillComplete. In that "
326  "case, you must call fillComplete on the graph again.");
327 
328  size_t numValuesPacked = graph->lclIndsPacked_wdv.extent(0);
329  valuesPacked_wdv = values_wdv_type(matrix.valuesPacked_wdv, 0, numValuesPacked);
330 
331  size_t numValuesUnpacked = graph->lclIndsUnpacked_wdv.extent(0);
332  valuesUnpacked_wdv = values_wdv_type(matrix.valuesUnpacked_wdv, 0, numValuesUnpacked);
333 
334  checkInternalState();
335  }
336 
337 
338  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
339  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
340  CrsMatrix (const Teuchos::RCP<const crs_graph_type>& graph,
341  const typename local_matrix_device_type::values_type& values,
342  const Teuchos::RCP<Teuchos::ParameterList>& /* params */) :
343  dist_object_type (graph->getRowMap ()),
344  staticGraph_ (graph),
345  storageStatus_ (Details::STORAGE_1D_PACKED)
346  {
347  const char tfecfFuncName[] = "CrsMatrix(RCP<const CrsGraph>, "
348  "local_matrix_device_type::values_type, "
349  "[,RCP<ParameterList>]): ";
350  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
351  (graph.is_null (), std::runtime_error, "Input graph is null.");
352  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
353  (! graph->isFillComplete (), std::runtime_error, "Input graph "
354  "is not fill complete. You must call fillComplete on the "
355  "graph before using it to construct a CrsMatrix. Note that "
356  "calling resumeFill on the graph makes it not fill complete, "
357  "even if you had previously called fillComplete. In that "
358  "case, you must call fillComplete on the graph again.");
359 
360  // The graph is fill complete, so it is locally indexed and has a
361  // fixed structure. This means we can allocate the (1-D) array of
362  // values and build the local matrix right now. Note that the
363  // local matrix's number of columns comes from the column Map, not
364  // the domain Map.
365 
366  valuesPacked_wdv = values_wdv_type(values);
367  valuesUnpacked_wdv = valuesPacked_wdv;
368 
369  // FIXME (22 Jun 2016) I would very much like to get rid of
370  // k_values1D_ at some point. I find it confusing to have all
371  // these extra references lying around.
372  // KDDKDD ALMOST THERE, MARK!
373 // k_values1D_ = valuesUnpacked_wdv.getDeviceView(Access::ReadWrite);
374 
375  checkInternalState ();
376  }
377 
378  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
379  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
380  CrsMatrix (const Teuchos::RCP<const map_type>& rowMap,
381  const Teuchos::RCP<const map_type>& colMap,
382  const typename local_graph_device_type::row_map_type& rowPointers,
383  const typename local_graph_device_type::entries_type::non_const_type& columnIndices,
384  const typename local_matrix_device_type::values_type& values,
385  const Teuchos::RCP<Teuchos::ParameterList>& params) :
386  dist_object_type (rowMap),
387  storageStatus_ (Details::STORAGE_1D_PACKED)
388  {
389  using Details::getEntryOnHost;
390  using Teuchos::RCP;
391  using std::endl;
392  const char tfecfFuncName[] = "Tpetra::CrsMatrix(RCP<const Map>, "
393  "RCP<const Map>, ptr, ind, val[, params]): ";
394  const char suffix[] =
395  ". Please report this bug to the Tpetra developers.";
396  const bool debug = Details::Behavior::debug("CrsMatrix");
397  const bool verbose = Details::Behavior::verbose("CrsMatrix");
398 
399  std::unique_ptr<std::string> prefix;
400  if (verbose) {
401  prefix = this->createPrefix(
402  "CrsMatrix", "CrsMatrix(rowMap,colMap,ptr,ind,val[,params])");
403  std::ostringstream os;
404  os << *prefix << "Start" << endl;
405  std::cerr << os.str ();
406  }
407 
408  // Check the user's input. Note that this might throw only on
409  // some processes but not others, causing deadlock. We prefer
410  // deadlock due to exceptions to segfaults, because users can
411  // catch exceptions.
412  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
413  (values.extent(0) != columnIndices.extent(0),
414  std::invalid_argument, "values.extent(0)=" << values.extent(0)
415  << " != columnIndices.extent(0) = " << columnIndices.extent(0)
416  << ".");
417  if (debug && rowPointers.extent(0) != 0) {
418  const size_t numEnt =
419  getEntryOnHost(rowPointers, rowPointers.extent(0) - 1);
420  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
421  (numEnt != size_t(columnIndices.extent(0)) ||
422  numEnt != size_t(values.extent(0)),
423  std::invalid_argument, "Last entry of rowPointers says that "
424  "the matrix has " << numEnt << " entr"
425  << (numEnt != 1 ? "ies" : "y") << ", but the dimensions of "
426  "columnIndices and values don't match this. "
427  "columnIndices.extent(0)=" << columnIndices.extent (0)
428  << " and values.extent(0)=" << values.extent (0) << ".");
429  }
430 
431  RCP<crs_graph_type> graph;
432  try {
433  graph = Teuchos::rcp (new crs_graph_type (rowMap, colMap, rowPointers,
434  columnIndices, params));
435  }
436  catch (std::exception& e) {
437  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
438  (true, std::runtime_error, "CrsGraph constructor (RCP<const Map>, "
439  "RCP<const Map>, ptr, ind[, params]) threw an exception: "
440  << e.what ());
441  }
442  // The newly created CrsGraph _must_ have a local graph at this
443  // point. We don't really care whether CrsGraph's constructor
444  // deep-copies or shallow-copies the input, but the dimensions
445  // have to be right. That's how we tell whether the CrsGraph has
446  // a local graph.
447  auto lclGraph = graph->getLocalGraphDevice ();
448  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
449  (lclGraph.row_map.extent (0) != rowPointers.extent (0) ||
450  lclGraph.entries.extent (0) != columnIndices.extent (0),
451  std::logic_error, "CrsGraph's constructor (rowMap, colMap, ptr, "
452  "ind[, params]) did not set the local graph correctly." << suffix);
453  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
454  (lclGraph.entries.extent (0) != values.extent (0),
455  std::logic_error, "CrsGraph's constructor (rowMap, colMap, ptr, ind[, "
456  "params]) did not set the local graph correctly. "
457  "lclGraph.entries.extent(0) = " << lclGraph.entries.extent (0)
458  << " != values.extent(0) = " << values.extent (0) << suffix);
459 
460  // myGraph_ not null means that the matrix owns the graph. This
461  // is true because the column indices come in as nonconst,
462  // implying shared ownership.
463  myGraph_ = graph;
464  staticGraph_ = graph;
465 
466  // The graph may not be fill complete yet. However, it is locally
467  // indexed (since we have a column Map) and has a fixed structure
468  // (due to the input arrays). This means we can allocate the
469  // (1-D) array of values and build the local matrix right now.
470  // Note that the local matrix's number of columns comes from the
471  // column Map, not the domain Map.
472 
473  valuesPacked_wdv = values_wdv_type(values);
474  valuesUnpacked_wdv = valuesPacked_wdv;
475 
476  // FIXME (22 Jun 2016) I would very much like to get rid of
477  // k_values1D_ at some point. I find it confusing to have all
478  // these extra references lying around.
479 // this->k_values1D_ = valuesPacked_wdv.getDeviceView(Access::ReadWrite);
480 
481  checkInternalState ();
482  if (verbose) {
483  std::ostringstream os;
484  os << *prefix << "Done" << endl;
485  std::cerr << os.str();
486  }
487  }
488 
489  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
490  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
491  CrsMatrix (const Teuchos::RCP<const map_type>& rowMap,
492  const Teuchos::RCP<const map_type>& colMap,
493  const Teuchos::ArrayRCP<size_t>& ptr,
494  const Teuchos::ArrayRCP<LocalOrdinal>& ind,
495  const Teuchos::ArrayRCP<Scalar>& val,
496  const Teuchos::RCP<Teuchos::ParameterList>& params) :
497  dist_object_type (rowMap),
498  storageStatus_ (Details::STORAGE_1D_PACKED)
499  {
500  using Kokkos::Compat::getKokkosViewDeepCopy;
501  using Teuchos::av_reinterpret_cast;
502  using Teuchos::RCP;
503  using values_type = typename local_matrix_device_type::values_type;
504  using IST = impl_scalar_type;
505  const char tfecfFuncName[] = "Tpetra::CrsMatrix(RCP<const Map>, "
506  "RCP<const Map>, ptr, ind, val[, params]): ";
507 
508  RCP<crs_graph_type> graph;
509  try {
510  graph = Teuchos::rcp (new crs_graph_type (rowMap, colMap, ptr,
511  ind, params));
512  }
513  catch (std::exception& e) {
514  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
515  (true, std::runtime_error, "CrsGraph constructor (RCP<const Map>, "
516  "RCP<const Map>, ArrayRCP<size_t>, ArrayRCP<LocalOrdinal>[, "
517  "RCP<ParameterList>]) threw an exception: " << e.what ());
518  }
519  // myGraph_ not null means that the matrix owns the graph. This
520  // is true because the column indices come in as nonconst,
521  // implying shared ownership.
522  myGraph_ = graph;
523  staticGraph_ = graph;
524 
525  // The graph may not be fill complete yet. However, it is locally
526  // indexed (since we have a column Map) and has a fixed structure
527  // (due to the input arrays). This means we can allocate the
528  // (1-D) array of values and build the local matrix right now.
529  // Note that the local matrix's number of columns comes from the
530  // column Map, not the domain Map.
531 
532  // The graph _must_ have a local graph at this point. We don't
533  // really care whether CrsGraph's constructor deep-copies or
534  // shallow-copies the input, but the dimensions have to be right.
535  // That's how we tell whether the CrsGraph has a local graph.
536  auto lclGraph = staticGraph_->getLocalGraphDevice ();
537  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
538  (size_t (lclGraph.row_map.extent (0)) != size_t (ptr.size ()) ||
539  size_t (lclGraph.entries.extent (0)) != size_t (ind.size ()),
540  std::logic_error, "CrsGraph's constructor (rowMap, colMap, "
541  "ptr, ind[, params]) did not set the local graph correctly. "
542  "Please report this bug to the Tpetra developers.");
543 
544  values_type valIn =
545  getKokkosViewDeepCopy<device_type> (av_reinterpret_cast<IST> (val ()));
546  valuesPacked_wdv = values_wdv_type(valIn);
547  valuesUnpacked_wdv = valuesPacked_wdv;
548 
549  // FIXME (22 Jun 2016) I would very much like to get rid of
550  // k_values1D_ at some point. I find it confusing to have all
551  // these extra references lying around.
552 // this->k_values1D_ = valuesPacked_wdv.getDeviceView(Access::ReadWrite);
553 
554  checkInternalState ();
555  }
556 
557  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
558  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
559  CrsMatrix (const Teuchos::RCP<const map_type>& rowMap,
560  const Teuchos::RCP<const map_type>& colMap,
561  const local_matrix_device_type& lclMatrix,
562  const Teuchos::RCP<Teuchos::ParameterList>& params) :
563  dist_object_type (rowMap),
564  storageStatus_ (Details::STORAGE_1D_PACKED),
565  fillComplete_ (true)
566  {
567  const char tfecfFuncName[] = "Tpetra::CrsMatrix(RCP<const Map>, "
568  "RCP<const Map>, local_matrix_device_type[, RCP<ParameterList>]): ";
569  const char suffix[] =
570  " Please report this bug to the Tpetra developers.";
571 
572  Teuchos::RCP<crs_graph_type> graph;
573  try {
574  graph = Teuchos::rcp (new crs_graph_type (rowMap, colMap,
575  lclMatrix.graph, params));
576  }
577  catch (std::exception& e) {
578  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
579  (true, std::runtime_error, "CrsGraph constructor (RCP<const Map>, "
580  "RCP<const Map>, local_graph_device_type[, RCP<ParameterList>]) threw an "
581  "exception: " << e.what ());
582  }
583  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
584  (!graph->isFillComplete (), std::logic_error, "CrsGraph constructor (RCP"
585  "<const Map>, RCP<const Map>, local_graph_device_type[, RCP<ParameterList>]) "
586  "did not produce a fill-complete graph. Please report this bug to the "
587  "Tpetra developers.");
588  // myGraph_ not null means that the matrix owns the graph. This
589  // is true because the column indices come in as nonconst through
590  // the matrix, implying shared ownership.
591  myGraph_ = graph;
592  staticGraph_ = graph;
593 
594  valuesPacked_wdv = values_wdv_type(lclMatrix.values);
595  valuesUnpacked_wdv = valuesPacked_wdv;
596 
597  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
598  (isFillActive (), std::logic_error,
599  "At the end of a CrsMatrix constructor that should produce "
600  "a fillComplete matrix, isFillActive() is true." << suffix);
601  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
602  (! isFillComplete (), std::logic_error, "At the end of a "
603  "CrsMatrix constructor that should produce a fillComplete "
604  "matrix, isFillComplete() is false." << suffix);
605  checkInternalState ();
606  }
607 
608  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
609  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
610  CrsMatrix (const local_matrix_device_type& lclMatrix,
611  const Teuchos::RCP<const map_type>& rowMap,
612  const Teuchos::RCP<const map_type>& colMap,
613  const Teuchos::RCP<const map_type>& domainMap,
614  const Teuchos::RCP<const map_type>& rangeMap,
615  const Teuchos::RCP<Teuchos::ParameterList>& params) :
616  dist_object_type (rowMap),
617  storageStatus_ (Details::STORAGE_1D_PACKED),
618  fillComplete_ (true)
619  {
620  const char tfecfFuncName[] = "Tpetra::CrsMatrix(RCP<const Map>, "
621  "RCP<const Map>, RCP<const Map>, RCP<const Map>, "
622  "local_matrix_device_type[, RCP<ParameterList>]): ";
623  const char suffix[] =
624  " Please report this bug to the Tpetra developers.";
625 
626  Teuchos::RCP<crs_graph_type> graph;
627  try {
628  graph = Teuchos::rcp (new crs_graph_type (lclMatrix.graph, rowMap, colMap,
629  domainMap, rangeMap, params));
630  }
631  catch (std::exception& e) {
632  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
633  (true, std::runtime_error, "CrsGraph constructor (RCP<const Map>, "
634  "RCP<const Map>, RCP<const Map>, RCP<const Map>, local_graph_device_type[, "
635  "RCP<ParameterList>]) threw an exception: " << e.what ());
636  }
637  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
638  (! graph->isFillComplete (), std::logic_error, "CrsGraph "
639  "constructor (RCP<const Map>, RCP<const Map>, RCP<const Map>, "
640  "RCP<const Map>, local_graph_device_type[, RCP<ParameterList>]) did "
641  "not produce a fillComplete graph." << suffix);
642  // myGraph_ not null means that the matrix owns the graph. This
643  // is true because the column indices come in as nonconst through
644  // the matrix, implying shared ownership.
645  myGraph_ = graph;
646  staticGraph_ = graph;
647 
648  valuesPacked_wdv = values_wdv_type(lclMatrix.values);
649  valuesUnpacked_wdv = valuesPacked_wdv;
650 
651  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
652  (isFillActive (), std::logic_error,
653  "At the end of a CrsMatrix constructor that should produce "
654  "a fillComplete matrix, isFillActive() is true." << suffix);
655  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
656  (! isFillComplete (), std::logic_error, "At the end of a "
657  "CrsMatrix constructor that should produce a fillComplete "
658  "matrix, isFillComplete() is false." << suffix);
659  checkInternalState ();
660  }
661 
662  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
663  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
664  CrsMatrix (const local_matrix_device_type& lclMatrix,
665  const Teuchos::RCP<const map_type>& rowMap,
666  const Teuchos::RCP<const map_type>& colMap,
667  const Teuchos::RCP<const map_type>& domainMap,
668  const Teuchos::RCP<const map_type>& rangeMap,
669  const Teuchos::RCP<const import_type>& importer,
670  const Teuchos::RCP<const export_type>& exporter,
671  const Teuchos::RCP<Teuchos::ParameterList>& params) :
672  dist_object_type (rowMap),
673  storageStatus_ (Details::STORAGE_1D_PACKED),
674  fillComplete_ (true)
675  {
676  using Teuchos::rcp;
677  const char tfecfFuncName[] = "Tpetra::CrsMatrix"
678  "(lclMat,Map,Map,Map,Map,Import,Export,params): ";
679  const char suffix[] =
680  " Please report this bug to the Tpetra developers.";
681 
682  Teuchos::RCP<crs_graph_type> graph;
683  try {
684  graph = rcp (new crs_graph_type (lclMatrix.graph, rowMap, colMap,
685  domainMap, rangeMap, importer,
686  exporter, params));
687  }
688  catch (std::exception& e) {
689  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
690  (true, std::runtime_error, "CrsGraph constructor "
691  "(local_graph_device_type, Map, Map, Map, Map, Import, Export, "
692  "params) threw: " << e.what ());
693  }
694  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
695  (!graph->isFillComplete (), std::logic_error, "CrsGraph "
696  "constructor (local_graph_device_type, Map, Map, Map, Map, Import, "
697  "Export, params) did not produce a fill-complete graph. "
698  "Please report this bug to the Tpetra developers.");
699  // myGraph_ not null means that the matrix owns the graph. This
700  // is true because the column indices come in as nonconst through
701  // the matrix, implying shared ownership.
702  myGraph_ = graph;
703  staticGraph_ = graph;
704 
705  valuesPacked_wdv = values_wdv_type(lclMatrix.values);
706  valuesUnpacked_wdv = valuesPacked_wdv;
707 
708  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
709  (isFillActive (), std::logic_error,
710  "At the end of a CrsMatrix constructor that should produce "
711  "a fillComplete matrix, isFillActive() is true." << suffix);
712  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
713  (! isFillComplete (), std::logic_error, "At the end of a "
714  "CrsMatrix constructor that should produce a fillComplete "
715  "matrix, isFillComplete() is false." << suffix);
716  checkInternalState ();
717  }
718 
719  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
720  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
721  CrsMatrix (const CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>& source,
722  const Teuchos::DataAccess copyOrView):
723  dist_object_type (source.getCrsGraph()->getRowMap ()),
724  staticGraph_ (source.getCrsGraph()),
725  storageStatus_ (source.storageStatus_)
726  {
727  const char tfecfFuncName[] = "Tpetra::CrsMatrix("
728  "const CrsMatrix&, const Teuchos::DataAccess): ";
729  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
730  (! source.isFillComplete (), std::invalid_argument,
731  "Source graph must be fillComplete().");
732 
733  if (copyOrView == Teuchos::Copy) {
734  using values_type = typename local_matrix_device_type::values_type;
735  auto vals = source.getLocalValuesDevice (Access::ReadOnly);
736  using Kokkos::view_alloc;
737  using Kokkos::WithoutInitializing;
738  values_type newvals (view_alloc ("val", WithoutInitializing),
739  vals.extent (0));
740  // DEEP_COPY REVIEW - DEVICE-TO_DEVICE
741  Kokkos::deep_copy (newvals, vals);
742  valuesPacked_wdv = values_wdv_type(newvals);
743  valuesUnpacked_wdv = valuesPacked_wdv;
744  fillComplete (source.getDomainMap (), source.getRangeMap ());
745  }
746  else if (copyOrView == Teuchos::View) {
747  valuesPacked_wdv = values_wdv_type(source.valuesPacked_wdv);
748  valuesUnpacked_wdv = values_wdv_type(source.valuesUnpacked_wdv);
749  fillComplete (source.getDomainMap (), source.getRangeMap ());
750  }
751  else {
752  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
753  (true, std::invalid_argument, "Second argument 'copyOrView' "
754  "has an invalid value " << copyOrView << ". Valid values "
755  "include Teuchos::Copy = " << Teuchos::Copy << " and "
756  "Teuchos::View = " << Teuchos::View << ".");
757  }
758  checkInternalState();
759  }
760 
761  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
762  void
763  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
764  swap(CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node> & crs_matrix)
765  {
766  std::swap(crs_matrix.importMV_, this->importMV_);
767  std::swap(crs_matrix.exportMV_, this->exportMV_);
768  std::swap(crs_matrix.staticGraph_, this->staticGraph_);
769  std::swap(crs_matrix.myGraph_, this->myGraph_);
770  std::swap(crs_matrix.valuesPacked_wdv, this->valuesPacked_wdv);
771  std::swap(crs_matrix.valuesUnpacked_wdv, this->valuesUnpacked_wdv);
772  std::swap(crs_matrix.storageStatus_, this->storageStatus_);
773  std::swap(crs_matrix.fillComplete_, this->fillComplete_);
774  std::swap(crs_matrix.nonlocals_, this->nonlocals_);
775  }
776 
777  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
778  Teuchos::RCP<const Teuchos::Comm<int> >
779  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
780  getComm () const {
781  return getCrsGraphRef ().getComm ();
782  }
783 
784  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
785  bool
786  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
787  isFillComplete () const {
788  return fillComplete_;
789  }
790 
791  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
792  bool
793  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
794  isFillActive () const {
795  return ! fillComplete_;
796  }
797 
798  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
799  bool
800  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
801  isStorageOptimized () const {
802  return this->getCrsGraphRef ().isStorageOptimized ();
803  }
804 
805  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
806  bool
807  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
808  isLocallyIndexed () const {
809  return getCrsGraphRef ().isLocallyIndexed ();
810  }
811 
812  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
813  bool
814  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
815  isGloballyIndexed () const {
816  return getCrsGraphRef ().isGloballyIndexed ();
817  }
818 
819  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
820  bool
821  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
822  hasColMap () const {
823  return getCrsGraphRef ().hasColMap ();
824  }
825 
826  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
827  global_size_t
828  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
829  getGlobalNumEntries () const {
830  return getCrsGraphRef ().getGlobalNumEntries ();
831  }
832 
833  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
834  size_t
835  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
836  getLocalNumEntries () const {
837  return getCrsGraphRef ().getLocalNumEntries ();
838  }
839 
840  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
841  global_size_t
842  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
843  getGlobalNumRows () const {
844  return getCrsGraphRef ().getGlobalNumRows ();
845  }
846 
847  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
848  global_size_t
849  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
850  getGlobalNumCols () const {
851  return getCrsGraphRef ().getGlobalNumCols ();
852  }
853 
854  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
855  size_t
856  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
857  getLocalNumRows () const {
858  return getCrsGraphRef ().getLocalNumRows ();
859  }
860 
861 
862  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
863  size_t
864  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
865  getLocalNumCols () const {
866  return getCrsGraphRef ().getLocalNumCols ();
867  }
868 
869 
870  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
871  size_t
872  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
873  getNumEntriesInGlobalRow (GlobalOrdinal globalRow) const {
874  return getCrsGraphRef ().getNumEntriesInGlobalRow (globalRow);
875  }
876 
877  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
878  size_t
879  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
880  getNumEntriesInLocalRow (LocalOrdinal localRow) const {
881  return getCrsGraphRef ().getNumEntriesInLocalRow (localRow);
882  }
883 
884  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
885  size_t
886  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
887  getGlobalMaxNumRowEntries () const {
888  return getCrsGraphRef ().getGlobalMaxNumRowEntries ();
889  }
890 
891  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
892  size_t
893  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
894  getLocalMaxNumRowEntries () const {
895  return getCrsGraphRef ().getLocalMaxNumRowEntries ();
896  }
897 
898  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
899  GlobalOrdinal
900  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
901  getIndexBase () const {
902  return getRowMap ()->getIndexBase ();
903  }
904 
905  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
906  Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> >
907  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
908  getRowMap () const {
909  return getCrsGraphRef ().getRowMap ();
910  }
911 
912  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
913  Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> >
914  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
915  getColMap () const {
916  return getCrsGraphRef ().getColMap ();
917  }
918 
919  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
920  Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> >
921  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
922  getDomainMap () const {
923  return getCrsGraphRef ().getDomainMap ();
924  }
925 
926  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
927  Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> >
928  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
929  getRangeMap () const {
930  return getCrsGraphRef ().getRangeMap ();
931  }
932 
933  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
934  Teuchos::RCP<const RowGraph<LocalOrdinal, GlobalOrdinal, Node> >
935  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
936  getGraph () const {
937  if (staticGraph_ != Teuchos::null) {
938  return staticGraph_;
939  }
940  return myGraph_;
941  }
942 
943  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
944  Teuchos::RCP<const CrsGraph<LocalOrdinal, GlobalOrdinal, Node> >
945  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
946  getCrsGraph () const {
947  if (staticGraph_ != Teuchos::null) {
948  return staticGraph_;
949  }
950  return myGraph_;
951  }
952 
953  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
954  const CrsGraph<LocalOrdinal, GlobalOrdinal, Node>&
955  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
956  getCrsGraphRef () const
957  {
958 #ifdef HAVE_TPETRA_DEBUG
959  constexpr bool debug = true;
960 #else
961  constexpr bool debug = false;
962 #endif // HAVE_TPETRA_DEBUG
963 
964  if (! this->staticGraph_.is_null ()) {
965  return * (this->staticGraph_);
966  }
967  else {
968  if (debug) {
969  const char tfecfFuncName[] = "getCrsGraphRef: ";
970  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
971  (this->myGraph_.is_null (), std::logic_error,
972  "Both staticGraph_ and myGraph_ are null. "
973  "Please report this bug to the Tpetra developers.");
974  }
975  return * (this->myGraph_);
976  }
977  }
978 
979  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
980  typename CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::local_matrix_device_type
981  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
982  getLocalMatrixDevice () const
983  {
984  auto numCols = staticGraph_->getColMap()->getLocalNumElements();
985  return local_matrix_device_type("Tpetra::CrsMatrix::lclMatrixDevice",
986  numCols,
987  valuesPacked_wdv.getDeviceView(Access::ReadWrite),
988  staticGraph_->getLocalGraphDevice());
989  }
990 
991  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
992  typename CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::local_matrix_host_type
993  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
994  getLocalMatrixHost () const
995  {
996  auto numCols = staticGraph_->getColMap()->getLocalNumElements();
997  return local_matrix_host_type("Tpetra::CrsMatrix::lclMatrixHost", numCols,
998  valuesPacked_wdv.getHostView(Access::ReadWrite),
999  staticGraph_->getLocalGraphHost());
1000  }
1001 
1002 #if KOKKOSKERNELS_VERSION < 40299
1003 // KDDKDD NOT SURE WHY THIS MUST RETURN A SHARED_PTR
1004  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
1005  std::shared_ptr<typename CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::local_multiply_op_type>
1006  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
1007  getLocalMultiplyOperator () const
1008  {
1009  auto localMatrix = getLocalMatrixDevice();
1010 #if defined(KOKKOSKERNELS_ENABLE_TPL_CUSPARSE) || defined(KOKKOSKERNELS_ENABLE_TPL_ROCSPARSE) || defined(KOKKOSKERNELS_ENABLE_TPL_MKL)
1011  if(this->getLocalNumEntries() <= size_t(Teuchos::OrdinalTraits<LocalOrdinal>::max()))
1012  {
1013  if(this->ordinalRowptrs.data() == nullptr)
1014  {
1015  auto originalRowptrs = localMatrix.graph.row_map;
1016  //create LocalOrdinal-typed copy of the local graph's rowptrs.
1017  //This enables the LocalCrsMatrixOperator to use cuSPARSE SpMV.
1018  this->ordinalRowptrs = ordinal_rowptrs_type(
1019  Kokkos::ViewAllocateWithoutInitializing("CrsMatrix::ordinalRowptrs"), originalRowptrs.extent(0));
1020  auto ordinalRowptrs_ = this->ordinalRowptrs; //don't want to capture 'this'
1021  Kokkos::parallel_for("CrsMatrix::getLocalMultiplyOperator::convertRowptrs",
1022  Kokkos::RangePolicy<execution_space>(0, originalRowptrs.extent(0)),
1023  KOKKOS_LAMBDA(LocalOrdinal i)
1024  {
1025  ordinalRowptrs_(i) = originalRowptrs(i);
1026  });
1027  }
1028  //return local operator using ordinalRowptrs
1029  return std::make_shared<local_multiply_op_type>(
1030  std::make_shared<local_matrix_device_type>(localMatrix), this->ordinalRowptrs);
1031  }
1032 #endif
1033 // KDDKDD NOT SURE WHY THIS MUST RETURN A SHARED_PTR
1034  return std::make_shared<local_multiply_op_type>(
1035  std::make_shared<local_matrix_device_type>(localMatrix));
1036  }
1037 #endif
1038 
1039  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
1040  bool
1041  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
1042  isStaticGraph () const {
1043  return myGraph_.is_null ();
1044  }
1045 
1046  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
1047  bool
1048  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
1049  hasTransposeApply () const {
1050  return true;
1051  }
1052 
1053  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
1054  bool
1055  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
1056  supportsRowViews () const {
1057  return true;
1058  }
1059 
1060  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
1061  void
1062  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
1063  allocateValues (ELocalGlobal lg, GraphAllocationStatus gas,
1064  const bool verbose)
1065  {
1066  using Details::Behavior;
1067  using Details::ProfilingRegion;
1068  using std::endl;
1069  const char tfecfFuncName[] = "allocateValues: ";
1070  const char suffix[] =
1071  " Please report this bug to the Tpetra developers.";
1072  ProfilingRegion region("Tpetra::CrsMatrix::allocateValues");
1073 
1074  std::unique_ptr<std::string> prefix;
1075  if (verbose) {
1076  prefix = this->createPrefix("CrsMatrix", "allocateValues");
1077  std::ostringstream os;
1078  os << *prefix << "lg: "
1079  << (lg == LocalIndices ? "Local" : "Global") << "Indices"
1080  << ", gas: Graph"
1081  << (gas == GraphAlreadyAllocated ? "Already" : "NotYet")
1082  << "Allocated" << endl;
1083  std::cerr << os.str();
1084  }
1085 
1086  const bool debug = Behavior::debug("CrsMatrix");
1087  if (debug) {
1088  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1089  (this->staticGraph_.is_null (), std::logic_error,
1090  "staticGraph_ is null." << suffix);
1091 
1092  // If the graph indices are already allocated, then gas should be
1093  // GraphAlreadyAllocated. Otherwise, gas should be
1094  // GraphNotYetAllocated.
1095  if ((gas == GraphAlreadyAllocated) !=
1096  staticGraph_->indicesAreAllocated ()) {
1097  const char err1[] = "The caller has asserted that the graph "
1098  "is ";
1099  const char err2[] = "already allocated, but the static graph "
1100  "says that its indices are ";
1101  const char err3[] = "already allocated. ";
1102  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1103  (gas == GraphAlreadyAllocated &&
1104  ! staticGraph_->indicesAreAllocated (), std::logic_error,
1105  err1 << err2 << "not " << err3 << suffix);
1106  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1107  (gas != GraphAlreadyAllocated &&
1108  staticGraph_->indicesAreAllocated (), std::logic_error,
1109  err1 << "not " << err2 << err3 << suffix);
1110  }
1111 
1112  // If the graph is unallocated, then it had better be a
1113  // matrix-owned graph. ("Matrix-owned graph" means that the
1114  // matrix gets to define the graph structure. If the CrsMatrix
1115  // constructor that takes an RCP<const CrsGraph> was used, then
1116  // the matrix does _not_ own the graph.)
1117  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1118  (! this->staticGraph_->indicesAreAllocated () &&
1119  this->myGraph_.is_null (), std::logic_error,
1120  "The static graph says that its indices are not allocated, "
1121  "but the graph is not owned by the matrix." << suffix);
1122  }
1123 
1124  if (gas == GraphNotYetAllocated) {
1125  if (debug) {
1126  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1127  (this->myGraph_.is_null (), std::logic_error,
1128  "gas = GraphNotYetAllocated, but myGraph_ is null." << suffix);
1129  }
1130  try {
1131  this->myGraph_->allocateIndices (lg, verbose);
1132  }
1133  catch (std::exception& e) {
1134  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1135  (true, std::runtime_error, "CrsGraph::allocateIndices "
1136  "threw an exception: " << e.what ());
1137  }
1138  catch (...) {
1139  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1140  (true, std::runtime_error, "CrsGraph::allocateIndices "
1141  "threw an exception not a subclass of std::exception.");
1142  }
1143  }
1144 
1145  // Allocate matrix values.
1146  const size_t lclTotalNumEntries = this->staticGraph_->getLocalAllocationSize();
1147  if (debug) {
1148  const size_t lclNumRows = this->staticGraph_->getLocalNumRows ();
1149  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1150  (this->staticGraph_->getRowPtrsUnpackedHost()(lclNumRows) != lclTotalNumEntries, std::logic_error,
1151  "length of staticGraph's lclIndsUnpacked does not match final entry of rowPtrsUnapcked_host." << suffix);
1152  }
1153 
1154  // Allocate array of (packed???) matrix values.
1155  using values_type = typename local_matrix_device_type::values_type;
1156  if (verbose) {
1157  std::ostringstream os;
1158  os << *prefix << "Allocate values_wdv: Pre "
1159  << valuesUnpacked_wdv.extent(0) << ", post "
1160  << lclTotalNumEntries << endl;
1161  std::cerr << os.str();
1162  }
1163 // this->k_values1D_ =
1164  valuesUnpacked_wdv = values_wdv_type(
1165  values_type("Tpetra::CrsMatrix::values",
1166  lclTotalNumEntries));
1167  }
1168 
1169 
1170  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
1171  void
1172  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
1173  fillLocalGraphAndMatrix (const Teuchos::RCP<Teuchos::ParameterList>& params)
1174  {
1175  using ::Tpetra::Details::computeOffsetsFromCounts;
1176  using ::Tpetra::Details::getEntryOnHost;
1177  using Teuchos::arcp_const_cast;
1178  using Teuchos::Array;
1179  using Teuchos::ArrayRCP;
1180  using Teuchos::null;
1181  using Teuchos::RCP;
1182  using Teuchos::rcp;
1183  using std::endl;
1184  using row_map_type = typename local_graph_device_type::row_map_type;
1185  using lclinds_1d_type = typename Graph::local_graph_device_type::entries_type::non_const_type;
1186  using values_type = typename local_matrix_device_type::values_type;
1187  Details::ProfilingRegion regionFLGAM
1188  ("Tpetra::CrsMatrix::fillLocalGraphAndMatrix");
1189 
1190  const char tfecfFuncName[] = "fillLocalGraphAndMatrix (called from "
1191  "fillComplete or expertStaticFillComplete): ";
1192  const char suffix[] =
1193  " Please report this bug to the Tpetra developers.";
1194  const bool debug = Details::Behavior::debug("CrsMatrix");
1195  const bool verbose = Details::Behavior::verbose("CrsMatrix");
1196 
1197  std::unique_ptr<std::string> prefix;
1198  if (verbose) {
1199  prefix = this->createPrefix("CrsMatrix", "fillLocalGraphAndMatrix");
1200  std::ostringstream os;
1201  os << *prefix << endl;
1202  std::cerr << os.str ();
1203  }
1204 
1205  if (debug) {
1206  // fillComplete() only calls fillLocalGraphAndMatrix() if the
1207  // matrix owns the graph, which means myGraph_ is not null.
1208  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1209  (myGraph_.is_null (), std::logic_error, "The nonconst graph "
1210  "(myGraph_) is null. This means that the matrix has a "
1211  "const (a.k.a. \"static\") graph. fillComplete or "
1212  "expertStaticFillComplete should never call "
1213  "fillLocalGraphAndMatrix in that case." << suffix);
1214  }
1215 
1216  const size_t lclNumRows = this->getLocalNumRows ();
1217 
1218  // This method's goal is to fill in the three arrays (compressed
1219  // sparse row format) that define the sparse graph's and matrix's
1220  // structure, and the sparse matrix's values.
1221  //
1222  // Get references to the data in myGraph_, so we can modify them
1223  // as well. Note that we only call fillLocalGraphAndMatrix() if
1224  // the matrix owns the graph, which means myGraph_ is not null.
1225 
1226  // NOTE: This does not work correctly w/ GCC 12.3 + CUDA due to a compiler bug.
1227  // See: https://github.com/trilinos/Trilinos/issues/12237
1228  //using row_entries_type = decltype (myGraph_->k_numRowEntries_);
1229  using row_entries_type = typename crs_graph_type::num_row_entries_type;
1230 
1231  typename Graph::local_graph_device_type::row_map_type curRowOffsets =
1232  myGraph_->rowPtrsUnpacked_dev_;
1233 
1234  if (debug) {
1235  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1236  (curRowOffsets.extent (0) == 0, std::logic_error,
1237  "curRowOffsets.extent(0) == 0.");
1238  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1239  (curRowOffsets.extent (0) != lclNumRows + 1, std::logic_error,
1240  "curRowOffsets.extent(0) = "
1241  << curRowOffsets.extent (0) << " != lclNumRows + 1 = "
1242  << (lclNumRows + 1) << ".");
1243  const size_t numOffsets = curRowOffsets.extent (0);
1244  const auto valToCheck = myGraph_->getRowPtrsUnpackedHost()(numOffsets - 1);
1245  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1246  (numOffsets != 0 &&
1247  myGraph_->lclIndsUnpacked_wdv.extent (0) != valToCheck,
1248  std::logic_error, "numOffsets = " <<
1249  numOffsets << " != 0 and myGraph_->lclIndsUnpacked_wdv.extent(0) = "
1250  << myGraph_->lclIndsUnpacked_wdv.extent (0) << " != curRowOffsets("
1251  << numOffsets << ") = " << valToCheck << ".");
1252  }
1253 
1254  if (myGraph_->getLocalNumEntries() !=
1255  myGraph_->getLocalAllocationSize()) {
1256 
1257  // Use the nonconst version of row_map_type for k_ptrs,
1258  // because row_map_type is const and we need to modify k_ptrs here.
1259  typename row_map_type::non_const_type k_ptrs;
1260  row_map_type k_ptrs_const;
1261  lclinds_1d_type k_inds;
1262  values_type k_vals;
1263 
1264  if (verbose) {
1265  std::ostringstream os;
1266  const auto numEnt = myGraph_->getLocalNumEntries();
1267  const auto allocSize = myGraph_->getLocalAllocationSize();
1268  os << *prefix << "Unpacked 1-D storage: numEnt=" << numEnt
1269  << ", allocSize=" << allocSize << endl;
1270  std::cerr << os.str ();
1271  }
1272  // The matrix's current 1-D storage is "unpacked." This means
1273  // the row offsets may differ from what the final row offsets
1274  // should be. This could happen, for example, if the user
1275  // set an upper
1276  // bound on the number of entries per row, but didn't fill all
1277  // those entries.
1278  if (debug && curRowOffsets.extent (0) != 0) {
1279  const size_t numOffsets =
1280  static_cast<size_t> (curRowOffsets.extent (0));
1281  const auto valToCheck = myGraph_->getRowPtrsUnpackedHost()(numOffsets - 1);
1282  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1283  (static_cast<size_t> (valToCheck) !=
1284  static_cast<size_t> (valuesUnpacked_wdv.extent (0)),
1285  std::logic_error, "(unpacked branch) Before "
1286  "allocating or packing, curRowOffsets(" << (numOffsets-1)
1287  << ") = " << valToCheck << " != valuesUnpacked_wdv.extent(0)"
1288  " = " << valuesUnpacked_wdv.extent (0) << ".");
1289  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1290  (static_cast<size_t> (valToCheck) !=
1291  static_cast<size_t> (myGraph_->lclIndsUnpacked_wdv.extent (0)),
1292  std::logic_error, "(unpacked branch) Before "
1293  "allocating or packing, curRowOffsets(" << (numOffsets-1)
1294  << ") = " << valToCheck
1295  << " != myGraph_->lclIndsUnpacked_wdv.extent(0) = "
1296  << myGraph_->lclIndsUnpacked_wdv.extent (0) << ".");
1297  }
1298  // Pack the row offsets into k_ptrs, by doing a sum-scan of
1299  // the array of valid entry counts per row.
1300 
1301  // Total number of entries in the matrix on the calling
1302  // process. We will compute this in the loop below. It's
1303  // cheap to compute and useful as a sanity check.
1304  size_t lclTotalNumEntries = 0;
1305  {
1306  // Allocate the packed row offsets array. We use a nonconst
1307  // temporary (packedRowOffsets) here, because k_ptrs is
1308  // const. We will assign packedRowOffsets to k_ptrs below.
1309  if (verbose) {
1310  std::ostringstream os;
1311  os << *prefix << "Allocate packed row offsets: "
1312  << (lclNumRows+1) << endl;
1313  std::cerr << os.str ();
1314  }
1315  typename row_map_type::non_const_type
1316  packedRowOffsets ("Tpetra::CrsGraph::ptr", lclNumRows + 1);
1317  typename row_entries_type::const_type numRowEnt_h =
1318  myGraph_->k_numRowEntries_;
1319  // We're computing offsets on device. This function can
1320  // handle numRowEnt_h being a host View.
1321  lclTotalNumEntries =
1322  computeOffsetsFromCounts (packedRowOffsets, numRowEnt_h);
1323  // packedRowOffsets is modifiable; k_ptrs isn't, so we have
1324  // to use packedRowOffsets in the loop above and assign here.
1325  k_ptrs = packedRowOffsets;
1326  k_ptrs_const = k_ptrs;
1327  }
1328 
1329  if (debug) {
1330  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1331  (static_cast<size_t> (k_ptrs.extent (0)) != lclNumRows + 1,
1332  std::logic_error,
1333  "(unpacked branch) After packing k_ptrs, "
1334  "k_ptrs.extent(0) = " << k_ptrs.extent (0) << " != "
1335  "lclNumRows+1 = " << (lclNumRows+1) << ".");
1336  const auto valToCheck = getEntryOnHost (k_ptrs, lclNumRows);
1337  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1338  (valToCheck != lclTotalNumEntries, std::logic_error,
1339  "(unpacked branch) After filling k_ptrs, "
1340  "k_ptrs(lclNumRows=" << lclNumRows << ") = " << valToCheck
1341  << " != total number of entries on the calling process = "
1342  << lclTotalNumEntries << ".");
1343  }
1344 
1345  // Allocate the arrays of packed column indices and values.
1346  if (verbose) {
1347  std::ostringstream os;
1348  os << *prefix << "Allocate packed local column indices: "
1349  << lclTotalNumEntries << endl;
1350  std::cerr << os.str ();
1351  }
1352  k_inds = lclinds_1d_type ("Tpetra::CrsGraph::lclInds", lclTotalNumEntries);
1353  if (verbose) {
1354  std::ostringstream os;
1355  os << *prefix << "Allocate packed values: "
1356  << lclTotalNumEntries << endl;
1357  std::cerr << os.str ();
1358  }
1359  k_vals = values_type ("Tpetra::CrsMatrix::values", lclTotalNumEntries);
1360 
1361  // curRowOffsets (myGraph_->rowPtrsUnpacked_) (???), lclIndsUnpacked_wdv,
1362  // and valuesUnpacked_wdv are currently unpacked. Pack them, using
1363  // the packed row offsets array k_ptrs that we created above.
1364  //
1365  // FIXME (mfh 06 Aug 2014) If "Optimize Storage" is false, we
1366  // need to keep around the unpacked row offsets, column
1367  // indices, and values arrays.
1368 
1369  // Pack the column indices from unpacked lclIndsUnpacked_wdv into
1370  // packed k_inds. We will replace lclIndsUnpacked_wdv below.
1371  using inds_packer_type = pack_functor<
1372  typename Graph::local_graph_device_type::entries_type::non_const_type,
1373  typename Graph::local_inds_dualv_type::t_dev::const_type,
1374  typename Graph::local_graph_device_type::row_map_type::non_const_type,
1375  typename Graph::local_graph_device_type::row_map_type>;
1376  inds_packer_type indsPacker (
1377  k_inds,
1378  myGraph_->lclIndsUnpacked_wdv.getDeviceView(Access::ReadOnly),
1379  k_ptrs, curRowOffsets);
1380  using exec_space = typename decltype (k_inds)::execution_space;
1381  using range_type = Kokkos::RangePolicy<exec_space, LocalOrdinal>;
1382  Kokkos::parallel_for
1383  ("Tpetra::CrsMatrix pack column indices",
1384  range_type (0, lclNumRows), indsPacker);
1385 
1386  // Pack the values from unpacked valuesUnpacked_wdv into packed
1387  // k_vals. We will replace valuesPacked_wdv below.
1388  using vals_packer_type = pack_functor<
1389  typename values_type::non_const_type,
1390  typename values_type::const_type,
1391  typename row_map_type::non_const_type,
1392  typename row_map_type::const_type>;
1393  vals_packer_type valsPacker (
1394  k_vals,
1395  this->valuesUnpacked_wdv.getDeviceView(Access::ReadOnly),
1396  k_ptrs, curRowOffsets);
1397  Kokkos::parallel_for ("Tpetra::CrsMatrix pack values",
1398  range_type (0, lclNumRows), valsPacker);
1399 
1400  if (debug) {
1401  const char myPrefix[] = "(\"Optimize Storage\""
1402  "=true branch) After packing, ";
1403  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1404  (k_ptrs.extent (0) == 0, std::logic_error, myPrefix
1405  << "k_ptrs.extent(0) = 0. This probably means that "
1406  "rowPtrsUnpacked_ was never allocated.");
1407  if (k_ptrs.extent (0) != 0) {
1408  const size_t numOffsets (k_ptrs.extent (0));
1409  const auto valToCheck =
1410  getEntryOnHost (k_ptrs, numOffsets - 1);
1411  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1412  (size_t (valToCheck) != k_vals.extent (0),
1413  std::logic_error, myPrefix <<
1414  "k_ptrs(" << (numOffsets-1) << ") = " << valToCheck <<
1415  " != k_vals.extent(0) = " << k_vals.extent (0) << ".");
1416  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1417  (size_t (valToCheck) != k_inds.extent (0),
1418  std::logic_error, myPrefix <<
1419  "k_ptrs(" << (numOffsets-1) << ") = " << valToCheck <<
1420  " != k_inds.extent(0) = " << k_inds.extent (0) << ".");
1421  }
1422  }
1423  // Build the local graph.
1424  myGraph_->setRowPtrsPacked(k_ptrs_const);
1425  myGraph_->lclIndsPacked_wdv =
1426  typename crs_graph_type::local_inds_wdv_type(k_inds);
1427  valuesPacked_wdv = values_wdv_type(k_vals);
1428  }
1429  else { // We don't have to pack, so just set the pointers.
1430  // FIXME KDDKDD https://github.com/trilinos/Trilinos/issues/9657
1431  // FIXME? This is already done in the graph fill call - need to avoid the memcpy to host
1432  myGraph_->rowPtrsPacked_dev_ = myGraph_->rowPtrsUnpacked_dev_;
1433  myGraph_->rowPtrsPacked_host_ = myGraph_->rowPtrsUnpacked_host_;
1434  myGraph_->packedUnpackedRowPtrsMatch_ = true;
1435  myGraph_->lclIndsPacked_wdv = myGraph_->lclIndsUnpacked_wdv;
1436  valuesPacked_wdv = valuesUnpacked_wdv;
1437 
1438  if (verbose) {
1439  std::ostringstream os;
1440  os << *prefix << "Storage already packed: rowPtrsUnpacked_: "
1441  << myGraph_->getRowPtrsUnpackedHost().extent(0) << ", lclIndsUnpacked_wdv: "
1442  << myGraph_->lclIndsUnpacked_wdv.extent(0) << ", valuesUnpacked_wdv: "
1443  << valuesUnpacked_wdv.extent(0) << endl;
1444  std::cerr << os.str();
1445  }
1446 
1447  if (debug) {
1448  const char myPrefix[] =
1449  "(\"Optimize Storage\"=false branch) ";
1450  auto rowPtrsUnpackedHost = myGraph_->getRowPtrsUnpackedHost();
1451  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1452  (myGraph_->rowPtrsUnpacked_dev_.extent (0) == 0, std::logic_error, myPrefix
1453  << "myGraph->rowPtrsUnpacked_dev_.extent(0) = 0. This probably means "
1454  "that rowPtrsUnpacked_ was never allocated.");
1455  if (myGraph_->rowPtrsUnpacked_dev_.extent (0) != 0) {
1456  const size_t numOffsets = rowPtrsUnpackedHost.extent (0);
1457  const auto valToCheck = rowPtrsUnpackedHost(numOffsets - 1);
1458  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1459  (size_t (valToCheck) != valuesPacked_wdv.extent (0),
1460  std::logic_error, myPrefix <<
1461  "k_ptrs_const(" << (numOffsets-1) << ") = " << valToCheck
1462  << " != valuesPacked_wdv.extent(0) = "
1463  << valuesPacked_wdv.extent (0) << ".");
1464  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1465  (size_t (valToCheck) != myGraph_->lclIndsPacked_wdv.extent (0),
1466  std::logic_error, myPrefix <<
1467  "k_ptrs_const(" << (numOffsets-1) << ") = " << valToCheck
1468  << " != myGraph_->lclIndsPacked.extent(0) = "
1469  << myGraph_->lclIndsPacked_wdv.extent (0) << ".");
1470  }
1471  }
1472  }
1473 
1474  if (debug) {
1475  const char myPrefix[] = "After packing, ";
1476  auto rowPtrsPackedHost = myGraph_->getRowPtrsPackedHost();
1477  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1478  (size_t (rowPtrsPackedHost.extent (0)) != size_t (lclNumRows + 1),
1479  std::logic_error, myPrefix << "myGraph_->rowPtrsPacked_host_.extent(0) = "
1480  << rowPtrsPackedHost.extent (0) << " != lclNumRows+1 = " <<
1481  (lclNumRows+1) << ".");
1482  if (rowPtrsPackedHost.extent (0) != 0) {
1483  const size_t numOffsets (rowPtrsPackedHost.extent (0));
1484  const size_t valToCheck = rowPtrsPackedHost(numOffsets-1);
1485  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1486  (valToCheck != size_t (valuesPacked_wdv.extent (0)),
1487  std::logic_error, myPrefix << "k_ptrs_const(" <<
1488  (numOffsets-1) << ") = " << valToCheck
1489  << " != valuesPacked_wdv.extent(0) = "
1490  << valuesPacked_wdv.extent (0) << ".");
1491  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1492  (valToCheck != size_t (myGraph_->lclIndsPacked_wdv.extent (0)),
1493  std::logic_error, myPrefix << "k_ptrs_const(" <<
1494  (numOffsets-1) << ") = " << valToCheck
1495  << " != myGraph_->lclIndsPacked_wdvk_inds.extent(0) = "
1496  << myGraph_->lclIndsPacked_wdv.extent (0) << ".");
1497  }
1498  }
1499 
1500  // May we ditch the old allocations for the packed (and otherwise
1501  // "optimized") allocations, later in this routine? Optimize
1502  // storage if the graph is not static, or if the graph already has
1503  // optimized storage.
1504  const bool defaultOptStorage =
1505  ! isStaticGraph () || staticGraph_->isStorageOptimized ();
1506  const bool requestOptimizedStorage =
1507  (! params.is_null () &&
1508  params->get ("Optimize Storage", defaultOptStorage)) ||
1509  (params.is_null () && defaultOptStorage);
1510 
1511  // The graph has optimized storage when indices are allocated,
1512  // myGraph_->k_numRowEntries_ is empty, and there are more than
1513  // zero rows on this process.
1514  if (requestOptimizedStorage) {
1515  // Free the old, unpacked, unoptimized allocations.
1516  // Free graph data structures that are only needed for
1517  // unpacked 1-D storage.
1518  if (verbose) {
1519  std::ostringstream os;
1520  os << *prefix << "Optimizing storage: free k_numRowEntries_: "
1521  << myGraph_->k_numRowEntries_.extent(0) << endl;
1522  std::cerr << os.str();
1523  }
1524 
1525  myGraph_->k_numRowEntries_ = row_entries_type ();
1526 
1527  // Keep the new 1-D packed allocations.
1528  // FIXME KDDKDD https://github.com/trilinos/Trilinos/issues/9657
1529  // We directly set the memory spaces to avoid a memcpy from device to host
1530  myGraph_->rowPtrsUnpacked_dev_ = myGraph_->rowPtrsPacked_dev_;
1531  myGraph_->rowPtrsUnpacked_host_ = myGraph_->rowPtrsPacked_host_;
1532  myGraph_->packedUnpackedRowPtrsMatch_ = true;
1533  myGraph_->lclIndsUnpacked_wdv = myGraph_->lclIndsPacked_wdv;
1534  valuesUnpacked_wdv = valuesPacked_wdv;
1535 
1536  myGraph_->storageStatus_ = Details::STORAGE_1D_PACKED;
1537  this->storageStatus_ = Details::STORAGE_1D_PACKED;
1538  }
1539  else {
1540  if (verbose) {
1541  std::ostringstream os;
1542  os << *prefix << "User requested NOT to optimize storage"
1543  << endl;
1544  std::cerr << os.str();
1545  }
1546  }
1547  }
1548 
1549  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
1550  void
1551  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
1552  fillLocalMatrix (const Teuchos::RCP<Teuchos::ParameterList>& params)
1553  {
1554  using ::Tpetra::Details::ProfilingRegion;
1555  using Teuchos::ArrayRCP;
1556  using Teuchos::Array;
1557  using Teuchos::null;
1558  using Teuchos::RCP;
1559  using Teuchos::rcp;
1560  using std::endl;
1561  using row_map_type = typename Graph::local_graph_device_type::row_map_type;
1562  using non_const_row_map_type = typename row_map_type::non_const_type;
1563  using values_type = typename local_matrix_device_type::values_type;
1564  ProfilingRegion regionFLM("Tpetra::CrsMatrix::fillLocalMatrix");
1565  const size_t lclNumRows = getLocalNumRows();
1566 
1567  const bool verbose = Details::Behavior::verbose("CrsMatrix");
1568  std::unique_ptr<std::string> prefix;
1569  if (verbose) {
1570  prefix = this->createPrefix("CrsMatrix", "fillLocalMatrix");
1571  std::ostringstream os;
1572  os << *prefix << "lclNumRows: " << lclNumRows << endl;
1573  std::cerr << os.str ();
1574  }
1575 
1576  // The goals of this routine are first, to allocate and fill
1577  // packed 1-D storage (see below for an explanation) in the vals
1578  // array, and second, to give vals to the local matrix and
1579  // finalize the local matrix. We only need k_ptrs, the packed 1-D
1580  // row offsets, within the scope of this routine, since we're only
1581  // filling the local matrix here (use fillLocalGraphAndMatrix() to
1582  // fill both the graph and the matrix at the same time).
1583 
1584  // get data from staticGraph_
1585  size_t nodeNumEntries = staticGraph_->getLocalNumEntries ();
1586  size_t nodeNumAllocated = staticGraph_->getLocalAllocationSize ();
1587  row_map_type k_rowPtrs = staticGraph_->rowPtrsPacked_dev_;
1588 
1589  row_map_type k_ptrs; // "packed" row offsets array
1590  values_type k_vals; // "packed" values array
1591 
1592  // May we ditch the old allocations for the packed (and otherwise
1593  // "optimized") allocations, later in this routine? Request
1594  // optimized storage by default.
1595  bool requestOptimizedStorage = true;
1596  const bool default_OptimizeStorage =
1597  ! isStaticGraph() || staticGraph_->isStorageOptimized();
1598  if (! params.is_null() &&
1599  ! params->get("Optimize Storage", default_OptimizeStorage)) {
1600  requestOptimizedStorage = false;
1601  }
1602  // If we're not allowed to change a static graph, then we can't
1603  // change the storage of the matrix, either. This means that if
1604  // the graph's storage isn't already optimized, we can't optimize
1605  // the matrix's storage either. Check and give warning, as
1606  // appropriate.
1607  if (! staticGraph_->isStorageOptimized () &&
1608  requestOptimizedStorage) {
1609  TPETRA_ABUSE_WARNING
1610  (true, std::runtime_error, "You requested optimized storage "
1611  "by setting the \"Optimize Storage\" flag to \"true\" in "
1612  "the ParameterList, or by virtue of default behavior. "
1613  "However, the associated CrsGraph was filled separately and "
1614  "requested not to optimize storage. Therefore, the "
1615  "CrsMatrix cannot optimize storage.");
1616  requestOptimizedStorage = false;
1617  }
1618 
1619  // NOTE: This does not work correctly w/ GCC 12.3 + CUDA due to a compiler bug.
1620  // See: https://github.com/trilinos/Trilinos/issues/12237
1621  //using row_entries_type = decltype (staticGraph_->k_numRowEntries_);
1622  using row_entries_type = typename crs_graph_type::num_row_entries_type;
1623 
1624  // The matrix's values are currently
1625  // stored in a 1-D format. However, this format is "unpacked";
1626  // it doesn't necessarily have the same row offsets as indicated
1627  // by the ptrs array returned by allocRowPtrs. This could
1628  // happen, for example, if the user
1629  // fixed the number of matrix entries in
1630  // each row, but didn't fill all those entries.
1631  //
1632  // As above, we don't need to keep the "packed" row offsets
1633  // array ptrs here, but we do need it here temporarily, so we
1634  // have to allocate it. We'll free ptrs later in this method.
1635  //
1636  // Note that this routine checks whether storage has already
1637  // been packed. This is a common case for solution of nonlinear
1638  // PDEs using the finite element method, as long as the
1639  // structure of the sparse matrix does not change between linear
1640  // solves.
1641  if (nodeNumEntries != nodeNumAllocated) {
1642  if (verbose) {
1643  std::ostringstream os;
1644  os << *prefix << "Unpacked 1-D storage: numEnt="
1645  << nodeNumEntries << ", allocSize=" << nodeNumAllocated
1646  << endl;
1647  std::cerr << os.str();
1648  }
1649  // We have to pack the 1-D storage, since the user didn't fill
1650  // up all requested storage.
1651  if (verbose) {
1652  std::ostringstream os;
1653  os << *prefix << "Allocate packed row offsets: "
1654  << (lclNumRows+1) << endl;
1655  std::cerr << os.str();
1656  }
1657  non_const_row_map_type tmpk_ptrs ("Tpetra::CrsGraph::ptr",
1658  lclNumRows+1);
1659  // Total number of entries in the matrix on the calling
1660  // process. We will compute this in the loop below. It's
1661  // cheap to compute and useful as a sanity check.
1662  size_t lclTotalNumEntries = 0;
1663  k_ptrs = tmpk_ptrs;
1664  {
1665  typename row_entries_type::const_type numRowEnt_h =
1666  staticGraph_->k_numRowEntries_;
1667  // This function can handle the counts being a host View.
1668  lclTotalNumEntries =
1669  Details::computeOffsetsFromCounts (tmpk_ptrs, numRowEnt_h);
1670  }
1671 
1672  // Allocate the "packed" values array.
1673  // It has exactly the right number of entries.
1674  if (verbose) {
1675  std::ostringstream os;
1676  os << *prefix << "Allocate packed values: "
1677  << lclTotalNumEntries << endl;
1678  std::cerr << os.str ();
1679  }
1680  k_vals = values_type ("Tpetra::CrsMatrix::val", lclTotalNumEntries);
1681 
1682  // Pack values_wdv into k_vals. We will replace values_wdv below.
1683  pack_functor<
1684  typename values_type::non_const_type,
1685  typename values_type::const_type,
1686  typename row_map_type::non_const_type,
1687  typename row_map_type::const_type> valsPacker
1688  (k_vals, valuesUnpacked_wdv.getDeviceView(Access::ReadOnly),
1689  tmpk_ptrs, k_rowPtrs);
1690 
1691  using exec_space = typename decltype (k_vals)::execution_space;
1692  using range_type = Kokkos::RangePolicy<exec_space, LocalOrdinal>;
1693  Kokkos::parallel_for ("Tpetra::CrsMatrix pack values",
1694  range_type (0, lclNumRows), valsPacker);
1695  valuesPacked_wdv = values_wdv_type(k_vals);
1696  }
1697  else { // We don't have to pack, so just set the pointer.
1698  valuesPacked_wdv = valuesUnpacked_wdv;
1699  if (verbose) {
1700  std::ostringstream os;
1701  os << *prefix << "Storage already packed: "
1702  << "valuesUnpacked_wdv: " << valuesUnpacked_wdv.extent(0) << endl;
1703  std::cerr << os.str();
1704  }
1705  }
1706 
1707  // May we ditch the old allocations for the packed one?
1708  if (requestOptimizedStorage) {
1709  // The user requested optimized storage, so we can dump the
1710  // unpacked 1-D storage, and keep the packed storage.
1711  valuesUnpacked_wdv = valuesPacked_wdv;
1712 // k_values1D_ = valuesPacked_wdv.getDeviceView(Access::ReadWrite);
1713  this->storageStatus_ = Details::STORAGE_1D_PACKED;
1714  }
1715  }
1716 
1717  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
1718  void
1719  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
1720  insertIndicesAndValues (crs_graph_type& graph,
1721  RowInfo& rowInfo,
1722  const typename crs_graph_type::SLocalGlobalViews& newInds,
1723  const Teuchos::ArrayView<impl_scalar_type>& oldRowVals,
1724  const Teuchos::ArrayView<const impl_scalar_type>& newRowVals,
1725  const ELocalGlobal lg,
1726  const ELocalGlobal I)
1727  {
1728  const size_t oldNumEnt = rowInfo.numEntries;
1729  const size_t numInserted = graph.insertIndices (rowInfo, newInds, lg, I);
1730 
1731  // Use of memcpy here works around an issue with GCC >= 4.9.0,
1732  // that probably relates to scalar_type vs. impl_scalar_type
1733  // aliasing. See history of Tpetra_CrsGraph_def.hpp for
1734  // details; look for GCC_WORKAROUND macro definition.
1735  if (numInserted > 0) {
1736  const size_t startOffset = oldNumEnt;
1737  memcpy ((void*) &oldRowVals[startOffset], &newRowVals[0],
1738  numInserted * sizeof (impl_scalar_type));
1739  }
1740  }
1741 
1742  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
1743  void
1744  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
1745  insertLocalValues (const LocalOrdinal lclRow,
1746  const Teuchos::ArrayView<const LocalOrdinal>& indices,
1747  const Teuchos::ArrayView<const Scalar>& values,
1748  const CombineMode CM)
1749  {
1750  using std::endl;
1751  const char tfecfFuncName[] = "insertLocalValues: ";
1752 
1753  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1754  (! this->isFillActive (), std::runtime_error,
1755  "Fill is not active. After calling fillComplete, you must call "
1756  "resumeFill before you may insert entries into the matrix again.");
1757  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1758  (this->isStaticGraph (), std::runtime_error,
1759  "Cannot insert indices with static graph; use replaceLocalValues() "
1760  "instead.");
1761  // At this point, we know that myGraph_ is nonnull.
1762  crs_graph_type& graph = * (this->myGraph_);
1763  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1764  (graph.colMap_.is_null (), std::runtime_error,
1765  "Cannot insert local indices without a column map.");
1766  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1767  (graph.isGloballyIndexed (),
1768  std::runtime_error, "Graph indices are global; use "
1769  "insertGlobalValues().");
1770  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1771  (values.size () != indices.size (), std::runtime_error,
1772  "values.size() = " << values.size ()
1773  << " != indices.size() = " << indices.size () << ".");
1774  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
1775  ! graph.rowMap_->isNodeLocalElement (lclRow), std::runtime_error,
1776  "Local row index " << lclRow << " does not belong to this process.");
1777 
1778  if (! graph.indicesAreAllocated ()) {
1779  // We only allocate values at most once per process, so it's OK
1780  // to check TPETRA_VERBOSE here.
1781  const bool verbose = Details::Behavior::verbose("CrsMatrix");
1782  this->allocateValues (LocalIndices, GraphNotYetAllocated, verbose);
1783  }
1784 
1785 #ifdef HAVE_TPETRA_DEBUG
1786  const size_t numEntriesToAdd = static_cast<size_t> (indices.size ());
1787  // In a debug build, test whether any of the given column indices
1788  // are not in the column Map. Keep track of the invalid column
1789  // indices so we can tell the user about them.
1790  {
1791  using Teuchos::toString;
1792 
1793  const map_type& colMap = * (graph.colMap_);
1794  Teuchos::Array<LocalOrdinal> badColInds;
1795  bool allInColMap = true;
1796  for (size_t k = 0; k < numEntriesToAdd; ++k) {
1797  if (! colMap.isNodeLocalElement (indices[k])) {
1798  allInColMap = false;
1799  badColInds.push_back (indices[k]);
1800  }
1801  }
1802  if (! allInColMap) {
1803  std::ostringstream os;
1804  os << "You attempted to insert entries in owned row " << lclRow
1805  << ", at the following column indices: " << toString (indices)
1806  << "." << endl;
1807  os << "Of those, the following indices are not in the column Map on "
1808  "this process: " << toString (badColInds) << "." << endl << "Since "
1809  "the matrix has a column Map already, it is invalid to insert "
1810  "entries at those locations.";
1811  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1812  (true, std::invalid_argument, os.str ());
1813  }
1814  }
1815 #endif // HAVE_TPETRA_DEBUG
1816 
1817  RowInfo rowInfo = graph.getRowInfo (lclRow);
1818 
1819  auto valsView = this->getValuesViewHostNonConst(rowInfo);
1820  if (CM == ADD) {
1821  auto fun = [&](size_t const k, size_t const /*start*/, size_t const offset) {
1822  valsView[offset] += values[k]; };
1823  std::function<void(size_t const, size_t const, size_t const)> cb(std::ref(fun));
1824  graph.insertLocalIndicesImpl(lclRow, indices, cb);
1825  } else if (CM == INSERT) {
1826  auto fun = [&](size_t const k, size_t const /*start*/, size_t const offset) {
1827  valsView[offset] = values[k]; };
1828  std::function<void(size_t const, size_t const, size_t const)> cb(std::ref(fun));
1829  graph.insertLocalIndicesImpl(lclRow, indices, cb);
1830  } else {
1831  std::ostringstream os;
1832  os << "You attempted to use insertLocalValues with CombineMode " << combineModeToString(CM)
1833  << "but this has not been implemented." << endl;
1834  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1835  (true, std::invalid_argument, os.str ());
1836  }
1837  }
1838 
1839  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
1840  void
1841  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
1842  insertLocalValues (const LocalOrdinal localRow,
1843  const LocalOrdinal numEnt,
1844  const Scalar vals[],
1845  const LocalOrdinal cols[],
1846  const CombineMode CM)
1847  {
1848  Teuchos::ArrayView<const LocalOrdinal> colsT (cols, numEnt);
1849  Teuchos::ArrayView<const Scalar> valsT (vals, numEnt);
1850  this->insertLocalValues (localRow, colsT, valsT, CM);
1851  }
1852 
1853  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
1854  void
1855  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
1856  insertGlobalValuesImpl (crs_graph_type& graph,
1857  RowInfo& rowInfo,
1858  const GlobalOrdinal gblColInds[],
1859  const impl_scalar_type vals[],
1860  const size_t numInputEnt)
1861  {
1862 #ifdef HAVE_TPETRA_DEBUG
1863  const char tfecfFuncName[] = "insertGlobalValuesImpl: ";
1864  const size_t origNumEnt = graph.getNumEntriesInLocalRow (rowInfo.localRow);
1865  const size_t curNumEnt = rowInfo.numEntries;
1866 #endif // HAVE_TPETRA_DEBUG
1867 
1868  if (! graph.indicesAreAllocated ()) {
1869  // We only allocate values at most once per process, so it's OK
1870  // to check TPETRA_VERBOSE here.
1871  using ::Tpetra::Details::Behavior;
1872  const bool verbose = Behavior::verbose("CrsMatrix");
1873  this->allocateValues (GlobalIndices, GraphNotYetAllocated, verbose);
1874  // mfh 23 Jul 2017: allocateValues invalidates existing
1875  // getRowInfo results. Once we get rid of lazy graph
1876  // allocation, we'll be able to move the getRowInfo call outside
1877  // of this method.
1878  rowInfo = graph.getRowInfo (rowInfo.localRow);
1879  }
1880 
1881  auto valsView = this->getValuesViewHostNonConst(rowInfo);
1882  auto fun = [&](size_t const k, size_t const /*start*/, size_t const offset){
1883  valsView[offset] += vals[k];
1884  };
1885  std::function<void(size_t const, size_t const, size_t const)> cb(std::ref(fun));
1886 #ifdef HAVE_TPETRA_DEBUG
1887  //numInserted is only used inside the debug code below.
1888  auto numInserted =
1889 #endif
1890  graph.insertGlobalIndicesImpl(rowInfo, gblColInds, numInputEnt, cb);
1891 
1892 #ifdef HAVE_TPETRA_DEBUG
1893  size_t newNumEnt = curNumEnt + numInserted;
1894  const size_t chkNewNumEnt =
1895  graph.getNumEntriesInLocalRow (rowInfo.localRow);
1896  if (chkNewNumEnt != newNumEnt) {
1897  std::ostringstream os;
1898  os << std::endl << "newNumEnt = " << newNumEnt
1899  << " != graph.getNumEntriesInLocalRow(" << rowInfo.localRow
1900  << ") = " << chkNewNumEnt << "." << std::endl
1901  << "\torigNumEnt: " << origNumEnt << std::endl
1902  << "\tnumInputEnt: " << numInputEnt << std::endl
1903  << "\tgblColInds: [";
1904  for (size_t k = 0; k < numInputEnt; ++k) {
1905  os << gblColInds[k];
1906  if (k + size_t (1) < numInputEnt) {
1907  os << ",";
1908  }
1909  }
1910  os << "]" << std::endl
1911  << "\tvals: [";
1912  for (size_t k = 0; k < numInputEnt; ++k) {
1913  os << vals[k];
1914  if (k + size_t (1) < numInputEnt) {
1915  os << ",";
1916  }
1917  }
1918  os << "]" << std::endl;
1919 
1920  if (this->supportsRowViews ()) {
1921  values_host_view_type vals2;
1922  if (this->isGloballyIndexed ()) {
1923  global_inds_host_view_type gblColInds2;
1924  const GlobalOrdinal gblRow =
1925  graph.rowMap_->getGlobalElement (rowInfo.localRow);
1926  if (gblRow ==
1927  Tpetra::Details::OrdinalTraits<GlobalOrdinal>::invalid ()) {
1928  os << "Local row index " << rowInfo.localRow << " is invalid!"
1929  << std::endl;
1930  }
1931  else {
1932  bool getViewThrew = false;
1933  try {
1934  this->getGlobalRowView (gblRow, gblColInds2, vals2);
1935  }
1936  catch (std::exception& e) {
1937  getViewThrew = true;
1938  os << "getGlobalRowView threw exception:" << std::endl
1939  << e.what () << std::endl;
1940  }
1941  if (! getViewThrew) {
1942  os << "\tNew global column indices: ";
1943  for (size_t jjj = 0; jjj < gblColInds2.extent(0); jjj++)
1944  os << gblColInds2[jjj] << " ";
1945  os << std::endl;
1946  os << "\tNew values: ";
1947  for (size_t jjj = 0; jjj < vals2.extent(0); jjj++)
1948  os << vals2[jjj] << " ";
1949  os << std::endl;
1950  }
1951  }
1952  }
1953  else if (this->isLocallyIndexed ()) {
1954  local_inds_host_view_type lclColInds2;
1955  this->getLocalRowView (rowInfo.localRow, lclColInds2, vals2);
1956  os << "\tNew local column indices: ";
1957  for (size_t jjj = 0; jjj < lclColInds2.extent(0); jjj++)
1958  os << lclColInds2[jjj] << " ";
1959  os << std::endl;
1960  os << "\tNew values: ";
1961  for (size_t jjj = 0; jjj < vals2.extent(0); jjj++)
1962  os << vals2[jjj] << " ";
1963  os << std::endl;
1964  }
1965  }
1966 
1967  os << "Please report this bug to the Tpetra developers.";
1968  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1969  (true, std::logic_error, os.str ());
1970  }
1971 #endif // HAVE_TPETRA_DEBUG
1972  }
1973 
1974  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
1975  void
1976  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
1977  insertGlobalValues (const GlobalOrdinal gblRow,
1978  const Teuchos::ArrayView<const GlobalOrdinal>& indices,
1979  const Teuchos::ArrayView<const Scalar>& values)
1980  {
1981  using Teuchos::toString;
1982  using std::endl;
1983  typedef impl_scalar_type IST;
1984  typedef LocalOrdinal LO;
1985  typedef GlobalOrdinal GO;
1986  typedef Tpetra::Details::OrdinalTraits<LO> OTLO;
1987  typedef typename Teuchos::ArrayView<const GO>::size_type size_type;
1988  const char tfecfFuncName[] = "insertGlobalValues: ";
1989 
1990 #ifdef HAVE_TPETRA_DEBUG
1991  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1992  (values.size () != indices.size (), std::runtime_error,
1993  "values.size() = " << values.size () << " != indices.size() = "
1994  << indices.size () << ".");
1995 #endif // HAVE_TPETRA_DEBUG
1996 
1997  // getRowMap() is not thread safe, because it increments RCP's
1998  // reference count. getCrsGraphRef() is thread safe.
1999  const map_type& rowMap = * (this->getCrsGraphRef ().rowMap_);
2000  const LO lclRow = rowMap.getLocalElement (gblRow);
2001 
2002  if (lclRow == OTLO::invalid ()) {
2003  // Input row is _not_ owned by the calling process.
2004  //
2005  // See a note (now deleted) from mfh 14 Dec 2012: If input row
2006  // is not in the row Map, it doesn't matter whether or not the
2007  // graph is static; the data just get stashed for later use by
2008  // globalAssemble().
2009  this->insertNonownedGlobalValues (gblRow, indices, values);
2010  }
2011  else { // Input row _is_ owned by the calling process
2012  if (this->isStaticGraph ()) {
2013  // Uh oh! Not allowed to insert into owned rows in that case.
2014  const int myRank = rowMap.getComm ()->getRank ();
2015  const int numProcs = rowMap.getComm ()->getSize ();
2016  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2017  (true, std::runtime_error,
2018  "The matrix was constructed with a constant (\"static\") graph, "
2019  "yet the given global row index " << gblRow << " is in the row "
2020  "Map on the calling process (with rank " << myRank << ", of " <<
2021  numProcs << " process(es)). In this case, you may not insert "
2022  "new entries into rows owned by the calling process.");
2023  }
2024 
2025  crs_graph_type& graph = * (this->myGraph_);
2026  const IST* const inputVals =
2027  reinterpret_cast<const IST*> (values.getRawPtr ());
2028  const GO* const inputGblColInds = indices.getRawPtr ();
2029  const size_t numInputEnt = indices.size ();
2030  RowInfo rowInfo = graph.getRowInfo (lclRow);
2031 
2032  // If the matrix has a column Map, check at this point whether
2033  // the column indices belong to the column Map.
2034  //
2035  // FIXME (mfh 16 May 2013) We may want to consider deferring the
2036  // test to the CrsGraph method, since it may have to do this
2037  // anyway.
2038  if (! graph.colMap_.is_null ()) {
2039  const map_type& colMap = * (graph.colMap_);
2040  // In a debug build, keep track of the nonowned ("bad") column
2041  // indices, so that we can display them in the exception
2042  // message. In a release build, just ditch the loop early if
2043  // we encounter a nonowned column index.
2044 #ifdef HAVE_TPETRA_DEBUG
2045  Teuchos::Array<GO> badColInds;
2046 #endif // HAVE_TPETRA_DEBUG
2047  const size_type numEntriesToInsert = indices.size ();
2048  bool allInColMap = true;
2049  for (size_type k = 0; k < numEntriesToInsert; ++k) {
2050  if (! colMap.isNodeGlobalElement (indices[k])) {
2051  allInColMap = false;
2052 #ifdef HAVE_TPETRA_DEBUG
2053  badColInds.push_back (indices[k]);
2054 #else
2055  break;
2056 #endif // HAVE_TPETRA_DEBUG
2057  }
2058  }
2059  if (! allInColMap) {
2060  std::ostringstream os;
2061  os << "You attempted to insert entries in owned row " << gblRow
2062  << ", at the following column indices: " << toString (indices)
2063  << "." << endl;
2064 #ifdef HAVE_TPETRA_DEBUG
2065  os << "Of those, the following indices are not in the column Map "
2066  "on this process: " << toString (badColInds) << "." << endl
2067  << "Since the matrix has a column Map already, it is invalid "
2068  "to insert entries at those locations.";
2069 #else
2070  os << "At least one of those indices is not in the column Map "
2071  "on this process." << endl << "It is invalid to insert into "
2072  "columns not in the column Map on the process that owns the "
2073  "row.";
2074 #endif // HAVE_TPETRA_DEBUG
2075  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2076  (true, std::invalid_argument, os.str ());
2077  }
2078  }
2079 
2080  this->insertGlobalValuesImpl (graph, rowInfo, inputGblColInds,
2081  inputVals, numInputEnt);
2082  }
2083  }
2084 
2085 
2086  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2087  void
2088  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2089  insertGlobalValues (const GlobalOrdinal globalRow,
2090  const LocalOrdinal numEnt,
2091  const Scalar vals[],
2092  const GlobalOrdinal inds[])
2093  {
2094  Teuchos::ArrayView<const GlobalOrdinal> indsT (inds, numEnt);
2095  Teuchos::ArrayView<const Scalar> valsT (vals, numEnt);
2096  this->insertGlobalValues (globalRow, indsT, valsT);
2097  }
2098 
2099 
2100  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2101  void
2102  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2103  insertGlobalValuesFiltered(
2104  const GlobalOrdinal gblRow,
2105  const Teuchos::ArrayView<const GlobalOrdinal>& indices,
2106  const Teuchos::ArrayView<const Scalar>& values,
2107  const bool debug)
2108  {
2109  typedef impl_scalar_type IST;
2110  typedef LocalOrdinal LO;
2111  typedef GlobalOrdinal GO;
2112  typedef Tpetra::Details::OrdinalTraits<LO> OTLO;
2113  const char tfecfFuncName[] = "insertGlobalValuesFiltered: ";
2114 
2115  if (debug) {
2116  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2117  (values.size () != indices.size (), std::runtime_error,
2118  "values.size() = " << values.size () << " != indices.size() = "
2119  << indices.size () << ".");
2120  }
2121 
2122  // getRowMap() is not thread safe, because it increments RCP's
2123  // reference count. getCrsGraphRef() is thread safe.
2124  const map_type& rowMap = * (this->getCrsGraphRef ().rowMap_);
2125  const LO lclRow = rowMap.getLocalElement (gblRow);
2126  if (lclRow == OTLO::invalid ()) {
2127  // Input row is _not_ owned by the calling process.
2128  //
2129  // See a note (now deleted) from mfh 14 Dec 2012: If input row
2130  // is not in the row Map, it doesn't matter whether or not the
2131  // graph is static; the data just get stashed for later use by
2132  // globalAssemble().
2133  this->insertNonownedGlobalValues (gblRow, indices, values);
2134  }
2135  else { // Input row _is_ owned by the calling process
2136  if (this->isStaticGraph ()) {
2137  // Uh oh! Not allowed to insert into owned rows in that case.
2138  const int myRank = rowMap.getComm ()->getRank ();
2139  const int numProcs = rowMap.getComm ()->getSize ();
2140  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2141  (true, std::runtime_error,
2142  "The matrix was constructed with a constant (\"static\") graph, "
2143  "yet the given global row index " << gblRow << " is in the row "
2144  "Map on the calling process (with rank " << myRank << ", of " <<
2145  numProcs << " process(es)). In this case, you may not insert "
2146  "new entries into rows owned by the calling process.");
2147  }
2148 
2149  crs_graph_type& graph = * (this->myGraph_);
2150  const IST* const inputVals =
2151  reinterpret_cast<const IST*> (values.getRawPtr ());
2152  const GO* const inputGblColInds = indices.getRawPtr ();
2153  const size_t numInputEnt = indices.size ();
2154  RowInfo rowInfo = graph.getRowInfo (lclRow);
2155 
2156  if (!graph.colMap_.is_null() && graph.isLocallyIndexed()) {
2157  // This branch is similar in function to the following branch, but for
2158  // the special case that the target graph is locally indexed.
2159  // In this case, we cannot simply filter
2160  // out global indices that don't exist on the receiving process and
2161  // insert the remaining (global) indices, but we must convert them (the
2162  // remaining global indices) to local and call `insertLocalValues`.
2163  const map_type& colMap = * (graph.colMap_);
2164  size_t curOffset = 0;
2165  while (curOffset < numInputEnt) {
2166  // Find a sequence of input indices that are in the column Map on the
2167  // calling process. Doing a sequence at a time, instead of one at a
2168  // time, amortizes some overhead.
2169  Teuchos::Array<LO> lclIndices;
2170  size_t endOffset = curOffset;
2171  for ( ; endOffset < numInputEnt; ++endOffset) {
2172  auto lclIndex = colMap.getLocalElement(inputGblColInds[endOffset]);
2173  if (lclIndex != OTLO::invalid())
2174  lclIndices.push_back(lclIndex);
2175  else
2176  break;
2177  }
2178  // curOffset, endOffset: half-exclusive range of indices in the column
2179  // Map on the calling process. If endOffset == curOffset, the range is
2180  // empty.
2181  const LO numIndInSeq = (endOffset - curOffset);
2182  if (numIndInSeq != 0) {
2183  this->insertLocalValues(lclRow, lclIndices(), values(curOffset, numIndInSeq));
2184  }
2185  // Invariant before the increment line: Either endOffset ==
2186  // numInputEnt, or inputGblColInds[endOffset] is not in the column Map
2187  // on the calling process.
2188  if (debug) {
2189  const bool invariant = endOffset == numInputEnt ||
2190  colMap.getLocalElement (inputGblColInds[endOffset]) == OTLO::invalid ();
2191  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2192  (! invariant, std::logic_error, std::endl << "Invariant failed!");
2193  }
2194  curOffset = endOffset + 1;
2195  }
2196  }
2197  else if (! graph.colMap_.is_null ()) { // We have a column Map.
2198  const map_type& colMap = * (graph.colMap_);
2199  size_t curOffset = 0;
2200  while (curOffset < numInputEnt) {
2201  // Find a sequence of input indices that are in the column
2202  // Map on the calling process. Doing a sequence at a time,
2203  // instead of one at a time, amortizes some overhead.
2204  size_t endOffset = curOffset;
2205  for ( ; endOffset < numInputEnt &&
2206  colMap.getLocalElement (inputGblColInds[endOffset]) != OTLO::invalid ();
2207  ++endOffset)
2208  {}
2209  // curOffset, endOffset: half-exclusive range of indices in
2210  // the column Map on the calling process. If endOffset ==
2211  // curOffset, the range is empty.
2212  const LO numIndInSeq = (endOffset - curOffset);
2213  if (numIndInSeq != 0) {
2214  rowInfo = graph.getRowInfo(lclRow); // KDD 5/19 Need fresh RowInfo in each loop iteration
2215  this->insertGlobalValuesImpl (graph, rowInfo,
2216  inputGblColInds + curOffset,
2217  inputVals + curOffset,
2218  numIndInSeq);
2219  }
2220  // Invariant before the increment line: Either endOffset ==
2221  // numInputEnt, or inputGblColInds[endOffset] is not in the
2222  // column Map on the calling process.
2223  if (debug) {
2224  const bool invariant = endOffset == numInputEnt ||
2225  colMap.getLocalElement (inputGblColInds[endOffset]) == OTLO::invalid ();
2226  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
2227  (! invariant, std::logic_error, std::endl << "Invariant failed!");
2228  }
2229  curOffset = endOffset + 1;
2230  }
2231  }
2232  else { // we don't have a column Map.
2233  this->insertGlobalValuesImpl (graph, rowInfo, inputGblColInds,
2234  inputVals, numInputEnt);
2235  }
2236  }
2237  }
2238 
2239  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2240  void
2241  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2242  insertGlobalValuesFilteredChecked(
2243  const GlobalOrdinal gblRow,
2244  const Teuchos::ArrayView<const GlobalOrdinal>& indices,
2245  const Teuchos::ArrayView<const Scalar>& values,
2246  const char* const prefix,
2247  const bool debug,
2248  const bool verbose)
2249  {
2250  using Details::verbosePrintArray;
2251  using std::endl;
2252 
2253  try {
2254  insertGlobalValuesFiltered(gblRow, indices, values, debug);
2255  }
2256  catch(std::exception& e) {
2257  std::ostringstream os;
2258  if (verbose) {
2259  const size_t maxNumToPrint =
2260  Details::Behavior::verbosePrintCountThreshold();
2261  os << *prefix << ": insertGlobalValuesFiltered threw an "
2262  "exception: " << e.what() << endl
2263  << "Global row index: " << gblRow << endl;
2264  verbosePrintArray(os, indices, "Global column indices",
2265  maxNumToPrint);
2266  os << endl;
2267  verbosePrintArray(os, values, "Values", maxNumToPrint);
2268  os << endl;
2269  }
2270  else {
2271  os << ": insertGlobalValuesFiltered threw an exception: "
2272  << e.what();
2273  }
2274  TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error, os.str());
2275  }
2276  }
2277 
2278  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2279  LocalOrdinal
2280  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2281  replaceLocalValuesImpl (impl_scalar_type rowVals[],
2282  const crs_graph_type& graph,
2283  const RowInfo& rowInfo,
2284  const LocalOrdinal inds[],
2285  const impl_scalar_type newVals[],
2286  const LocalOrdinal numElts)
2287  {
2288  typedef LocalOrdinal LO;
2289  typedef GlobalOrdinal GO;
2290  const bool sorted = graph.isSorted ();
2291 
2292  size_t hint = 0; // Guess for the current index k into rowVals
2293  LO numValid = 0; // number of valid local column indices
2294 
2295  if (graph.isLocallyIndexed ()) {
2296  // Get a view of the column indices in the row. This amortizes
2297  // the cost of getting the view over all the entries of inds.
2298  auto colInds = graph.getLocalIndsViewHost (rowInfo);
2299 
2300  for (LO j = 0; j < numElts; ++j) {
2301  const LO lclColInd = inds[j];
2302  const size_t offset =
2303  KokkosSparse::findRelOffset (colInds, rowInfo.numEntries,
2304  lclColInd, hint, sorted);
2305  if (offset != rowInfo.numEntries) {
2306  rowVals[offset] = newVals[j];
2307  hint = offset + 1;
2308  ++numValid;
2309  }
2310  }
2311  }
2312  else if (graph.isGloballyIndexed ()) {
2313  if (graph.colMap_.is_null ()) {
2314  return Teuchos::OrdinalTraits<LO>::invalid ();
2315  }
2316  const map_type colMap = * (graph.colMap_);
2317 
2318  // Get a view of the column indices in the row. This amortizes
2319  // the cost of getting the view over all the entries of inds.
2320  auto colInds = graph.getGlobalIndsViewHost (rowInfo);
2321 
2322  for (LO j = 0; j < numElts; ++j) {
2323  const GO gblColInd = colMap.getGlobalElement (inds[j]);
2324  if (gblColInd != Teuchos::OrdinalTraits<GO>::invalid ()) {
2325  const size_t offset =
2326  KokkosSparse::findRelOffset (colInds, rowInfo.numEntries,
2327  gblColInd, hint, sorted);
2328  if (offset != rowInfo.numEntries) {
2329  rowVals[offset] = newVals[j];
2330  hint = offset + 1;
2331  ++numValid;
2332  }
2333  }
2334  }
2335  }
2336  // NOTE (mfh 26 Jun 2014, 26 Nov 2015) In the current version of
2337  // CrsGraph and CrsMatrix, it's possible for a matrix (or graph)
2338  // to be neither locally nor globally indexed on a process.
2339  // This means that the graph or matrix has no entries on that
2340  // process. Epetra also works like this. It's related to lazy
2341  // allocation (on first insertion, not at graph / matrix
2342  // construction). Lazy allocation will go away because it is
2343  // not thread scalable.
2344 
2345  return numValid;
2346  }
2347 
2348  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2349  LocalOrdinal
2350  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2351  replaceLocalValues (const LocalOrdinal localRow,
2352  const Teuchos::ArrayView<const LocalOrdinal>& lclCols,
2353  const Teuchos::ArrayView<const Scalar>& vals)
2354  {
2355  typedef LocalOrdinal LO;
2356 
2357  const LO numInputEnt = static_cast<LO> (lclCols.size ());
2358  if (static_cast<LO> (vals.size ()) != numInputEnt) {
2359  return Teuchos::OrdinalTraits<LO>::invalid ();
2360  }
2361  const LO* const inputInds = lclCols.getRawPtr ();
2362  const Scalar* const inputVals = vals.getRawPtr ();
2363  return this->replaceLocalValues (localRow, numInputEnt,
2364  inputVals, inputInds);
2365  }
2366 
2367  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2368  typename CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2369  local_ordinal_type
2370  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2371  replaceLocalValues(
2372  const local_ordinal_type localRow,
2373  const Kokkos::View<const local_ordinal_type*, Kokkos::AnonymousSpace>& inputInds,
2374  const Kokkos::View<const impl_scalar_type*, Kokkos::AnonymousSpace>& inputVals)
2375  {
2376  using LO = local_ordinal_type;
2377  const LO numInputEnt = inputInds.extent(0);
2378  if (numInputEnt != static_cast<LO>(inputVals.extent(0))) {
2379  return Teuchos::OrdinalTraits<LO>::invalid();
2380  }
2381  const Scalar* const inVals =
2382  reinterpret_cast<const Scalar*>(inputVals.data());
2383  return this->replaceLocalValues(localRow, numInputEnt,
2384  inVals, inputInds.data());
2385  }
2386 
2387  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2388  LocalOrdinal
2389  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2390  replaceLocalValues (const LocalOrdinal localRow,
2391  const LocalOrdinal numEnt,
2392  const Scalar inputVals[],
2393  const LocalOrdinal inputCols[])
2394  {
2395  typedef impl_scalar_type IST;
2396  typedef LocalOrdinal LO;
2397 
2398  if (! this->isFillActive () || this->staticGraph_.is_null ()) {
2399  // Fill must be active and the "nonconst" graph must exist.
2400  return Teuchos::OrdinalTraits<LO>::invalid ();
2401  }
2402  const crs_graph_type& graph = * (this->staticGraph_);
2403  const RowInfo rowInfo = graph.getRowInfo (localRow);
2404 
2405  if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid ()) {
2406  // The calling process does not own this row, so it is not
2407  // allowed to modify its values.
2408  return static_cast<LO> (0);
2409  }
2410  auto curRowVals = this->getValuesViewHostNonConst (rowInfo);
2411  const IST* const inVals = reinterpret_cast<const IST*> (inputVals);
2412  return this->replaceLocalValuesImpl (curRowVals.data (), graph, rowInfo,
2413  inputCols, inVals, numEnt);
2414  }
2415 
2416  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2417  LocalOrdinal
2418  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2419  replaceGlobalValuesImpl (impl_scalar_type rowVals[],
2420  const crs_graph_type& graph,
2421  const RowInfo& rowInfo,
2422  const GlobalOrdinal inds[],
2423  const impl_scalar_type newVals[],
2424  const LocalOrdinal numElts)
2425  {
2426  Teuchos::ArrayView<const GlobalOrdinal> indsT(inds, numElts);
2427  auto fun =
2428  [&](size_t const k, size_t const /*start*/, size_t const offset) {
2429  rowVals[offset] = newVals[k];
2430  };
2431  std::function<void(size_t const, size_t const, size_t const)> cb(std::ref(fun));
2432  return graph.findGlobalIndices(rowInfo, indsT, cb);
2433  }
2434 
2435  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2436  LocalOrdinal
2437  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2438  replaceGlobalValues (const GlobalOrdinal globalRow,
2439  const Teuchos::ArrayView<const GlobalOrdinal>& inputGblColInds,
2440  const Teuchos::ArrayView<const Scalar>& inputVals)
2441  {
2442  typedef LocalOrdinal LO;
2443 
2444  const LO numInputEnt = static_cast<LO> (inputGblColInds.size ());
2445  if (static_cast<LO> (inputVals.size ()) != numInputEnt) {
2446  return Teuchos::OrdinalTraits<LO>::invalid ();
2447  }
2448  return this->replaceGlobalValues (globalRow, numInputEnt,
2449  inputVals.getRawPtr (),
2450  inputGblColInds.getRawPtr ());
2451  }
2452 
2453  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2454  LocalOrdinal
2455  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2456  replaceGlobalValues (const GlobalOrdinal globalRow,
2457  const LocalOrdinal numEnt,
2458  const Scalar inputVals[],
2459  const GlobalOrdinal inputGblColInds[])
2460  {
2461  typedef impl_scalar_type IST;
2462  typedef LocalOrdinal LO;
2463 
2464  if (! this->isFillActive () || this->staticGraph_.is_null ()) {
2465  // Fill must be active and the "nonconst" graph must exist.
2466  return Teuchos::OrdinalTraits<LO>::invalid ();
2467  }
2468  const crs_graph_type& graph = * (this->staticGraph_);
2469 
2470  const RowInfo rowInfo = graph.getRowInfoFromGlobalRowIndex (globalRow);
2471  if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid ()) {
2472  // The input local row is invalid on the calling process,
2473  // which means that the calling process summed 0 entries.
2474  return static_cast<LO> (0);
2475  }
2476 
2477  auto curRowVals = this->getValuesViewHostNonConst (rowInfo);
2478  const IST* const inVals = reinterpret_cast<const IST*> (inputVals);
2479  return this->replaceGlobalValuesImpl (curRowVals.data (), graph, rowInfo,
2480  inputGblColInds, inVals, numEnt);
2481  }
2482 
2483  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2484  typename CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2485  local_ordinal_type
2486  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2487  replaceGlobalValues(
2488  const global_ordinal_type globalRow,
2489  const Kokkos::View<const global_ordinal_type*, Kokkos::AnonymousSpace>& inputInds,
2490  const Kokkos::View<const impl_scalar_type*, Kokkos::AnonymousSpace>& inputVals)
2491  {
2492  // We use static_assert here to check the template parameters,
2493  // rather than std::enable_if (e.g., on the return value, to
2494  // enable compilation only if the template parameters match the
2495  // desired attributes). This turns obscure link errors into
2496  // clear compilation errors. It also makes the return value a
2497  // lot easier to see.
2498  using LO = local_ordinal_type;
2499  const LO numInputEnt = static_cast<LO>(inputInds.extent(0));
2500  if (static_cast<LO>(inputVals.extent(0)) != numInputEnt) {
2501  return Teuchos::OrdinalTraits<LO>::invalid();
2502  }
2503  const Scalar* const inVals =
2504  reinterpret_cast<const Scalar*>(inputVals.data());
2505  return this->replaceGlobalValues(globalRow, numInputEnt, inVals,
2506  inputInds.data());
2507  }
2508 
2509  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2510  LocalOrdinal
2511  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2512  sumIntoGlobalValuesImpl (impl_scalar_type rowVals[],
2513  const crs_graph_type& graph,
2514  const RowInfo& rowInfo,
2515  const GlobalOrdinal inds[],
2516  const impl_scalar_type newVals[],
2517  const LocalOrdinal numElts,
2518  const bool atomic)
2519  {
2520  typedef LocalOrdinal LO;
2521  typedef GlobalOrdinal GO;
2522 
2523  const bool sorted = graph.isSorted ();
2524 
2525  size_t hint = 0; // guess at the index's relative offset in the row
2526  LO numValid = 0; // number of valid input column indices
2527 
2528  if (graph.isLocallyIndexed ()) {
2529  // NOTE (mfh 04 Nov 2015) Dereferencing an RCP or reading its
2530  // pointer does NOT change its reference count. Thus, this
2531  // code is still thread safe.
2532  if (graph.colMap_.is_null ()) {
2533  // NO input column indices are valid in this case, since if
2534  // the column Map is null on the calling process, then the
2535  // calling process owns no graph entries.
2536  return numValid;
2537  }
2538  const map_type& colMap = * (graph.colMap_);
2539 
2540  // Get a view of the column indices in the row. This amortizes
2541  // the cost of getting the view over all the entries of inds.
2542  auto colInds = graph.getLocalIndsViewHost (rowInfo);
2543  const LO LINV = Teuchos::OrdinalTraits<LO>::invalid ();
2544 
2545  for (LO j = 0; j < numElts; ++j) {
2546  const LO lclColInd = colMap.getLocalElement (inds[j]);
2547  if (lclColInd != LINV) {
2548  const size_t offset =
2549  KokkosSparse::findRelOffset (colInds, rowInfo.numEntries,
2550  lclColInd, hint, sorted);
2551  if (offset != rowInfo.numEntries) {
2552  if (atomic) {
2553  Kokkos::atomic_add (&rowVals[offset], newVals[j]);
2554  }
2555  else {
2556  rowVals[offset] += newVals[j];
2557  }
2558  hint = offset + 1;
2559  numValid++;
2560  }
2561  }
2562  }
2563  }
2564  else if (graph.isGloballyIndexed ()) {
2565  // Get a view of the column indices in the row. This amortizes
2566  // the cost of getting the view over all the entries of inds.
2567  auto colInds = graph.getGlobalIndsViewHost (rowInfo);
2568 
2569  for (LO j = 0; j < numElts; ++j) {
2570  const GO gblColInd = inds[j];
2571  const size_t offset =
2572  KokkosSparse::findRelOffset (colInds, rowInfo.numEntries,
2573  gblColInd, hint, sorted);
2574  if (offset != rowInfo.numEntries) {
2575  if (atomic) {
2576  Kokkos::atomic_add (&rowVals[offset], newVals[j]);
2577  }
2578  else {
2579  rowVals[offset] += newVals[j];
2580  }
2581  hint = offset + 1;
2582  numValid++;
2583  }
2584  }
2585  }
2586  // If the graph is neither locally nor globally indexed on the
2587  // calling process, that means the calling process has no graph
2588  // entries. Thus, none of the input column indices are valid.
2589 
2590  return numValid;
2591  }
2592 
2593  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2594  LocalOrdinal
2595  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2596  sumIntoGlobalValues (const GlobalOrdinal gblRow,
2597  const Teuchos::ArrayView<const GlobalOrdinal>& inputGblColInds,
2598  const Teuchos::ArrayView<const Scalar>& inputVals,
2599  const bool atomic)
2600  {
2601  typedef LocalOrdinal LO;
2602 
2603  const LO numInputEnt = static_cast<LO> (inputGblColInds.size ());
2604  if (static_cast<LO> (inputVals.size ()) != numInputEnt) {
2605  return Teuchos::OrdinalTraits<LO>::invalid ();
2606  }
2607  return this->sumIntoGlobalValues (gblRow, numInputEnt,
2608  inputVals.getRawPtr (),
2609  inputGblColInds.getRawPtr (),
2610  atomic);
2611  }
2612 
2613  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2614  LocalOrdinal
2615  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2616  sumIntoGlobalValues (const GlobalOrdinal gblRow,
2617  const LocalOrdinal numInputEnt,
2618  const Scalar inputVals[],
2619  const GlobalOrdinal inputGblColInds[],
2620  const bool atomic)
2621  {
2622  typedef impl_scalar_type IST;
2623  typedef LocalOrdinal LO;
2624  typedef GlobalOrdinal GO;
2625 
2626  if (! this->isFillActive () || this->staticGraph_.is_null ()) {
2627  // Fill must be active and the "nonconst" graph must exist.
2628  return Teuchos::OrdinalTraits<LO>::invalid ();
2629  }
2630  const crs_graph_type& graph = * (this->staticGraph_);
2631 
2632  const RowInfo rowInfo = graph.getRowInfoFromGlobalRowIndex (gblRow);
2633  if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid ()) {
2634  // mfh 23 Mar 2017, 26 Jul 2017: This branch may not be not
2635  // thread safe in a debug build, in part because it uses
2636  // Teuchos::ArrayView, and in part because of the data structure
2637  // used to stash outgoing entries.
2638  using Teuchos::ArrayView;
2639  ArrayView<const GO> inputGblColInds_av(
2640  numInputEnt == 0 ? nullptr : inputGblColInds,
2641  numInputEnt);
2642  ArrayView<const Scalar> inputVals_av(
2643  numInputEnt == 0 ? nullptr :
2644  inputVals, numInputEnt);
2645  // gblRow is not in the row Map on the calling process, so stash
2646  // the given entries away in a separate data structure.
2647  // globalAssemble() (called during fillComplete()) will exchange
2648  // that data and sum it in using sumIntoGlobalValues().
2649  this->insertNonownedGlobalValues (gblRow, inputGblColInds_av,
2650  inputVals_av);
2651  // FIXME (mfh 08 Jul 2014) It's not clear what to return here,
2652  // since we won't know whether the given indices were valid
2653  // until globalAssemble (called in fillComplete) is called.
2654  // That's why insertNonownedGlobalValues doesn't return
2655  // anything. Just for consistency, I'll return the number of
2656  // entries that the user gave us.
2657  return numInputEnt;
2658  }
2659  else { // input row is in the row Map on the calling process
2660  auto curRowVals = this->getValuesViewHostNonConst (rowInfo);
2661  const IST* const inVals = reinterpret_cast<const IST*> (inputVals);
2662  return this->sumIntoGlobalValuesImpl (curRowVals.data (), graph, rowInfo,
2663  inputGblColInds, inVals,
2664  numInputEnt, atomic);
2665  }
2666  }
2667 
2668  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2669  LocalOrdinal
2670  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2671  transformLocalValues (const LocalOrdinal lclRow,
2672  const LocalOrdinal numInputEnt,
2673  const impl_scalar_type inputVals[],
2674  const LocalOrdinal inputCols[],
2675  std::function<impl_scalar_type (const impl_scalar_type&, const impl_scalar_type&) > f,
2676  const bool atomic)
2677  {
2678  using Tpetra::Details::OrdinalTraits;
2679  typedef LocalOrdinal LO;
2680 
2681  if (! this->isFillActive () || this->staticGraph_.is_null ()) {
2682  // Fill must be active and the "nonconst" graph must exist.
2683  return Teuchos::OrdinalTraits<LO>::invalid ();
2684  }
2685  const crs_graph_type& graph = * (this->staticGraph_);
2686  const RowInfo rowInfo = graph.getRowInfo (lclRow);
2687 
2688  if (rowInfo.localRow == OrdinalTraits<size_t>::invalid ()) {
2689  // The calling process does not own this row, so it is not
2690  // allowed to modify its values.
2691  return static_cast<LO> (0);
2692  }
2693  auto curRowVals = this->getValuesViewHostNonConst (rowInfo);
2694  return this->transformLocalValues (curRowVals.data (), graph,
2695  rowInfo, inputCols, inputVals,
2696  numInputEnt, f, atomic);
2697  }
2698 
2699  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2700  LocalOrdinal
2701  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2702  transformGlobalValues (const GlobalOrdinal gblRow,
2703  const LocalOrdinal numInputEnt,
2704  const impl_scalar_type inputVals[],
2705  const GlobalOrdinal inputCols[],
2706  std::function<impl_scalar_type (const impl_scalar_type&, const impl_scalar_type&) > f,
2707  const bool atomic)
2708  {
2709  using Tpetra::Details::OrdinalTraits;
2710  typedef LocalOrdinal LO;
2711 
2712  if (! this->isFillActive () || this->staticGraph_.is_null ()) {
2713  // Fill must be active and the "nonconst" graph must exist.
2714  return OrdinalTraits<LO>::invalid ();
2715  }
2716  const crs_graph_type& graph = * (this->staticGraph_);
2717  const RowInfo rowInfo = graph.getRowInfoFromGlobalRowIndex (gblRow);
2718 
2719  if (rowInfo.localRow == OrdinalTraits<size_t>::invalid ()) {
2720  // The calling process does not own this row, so it is not
2721  // allowed to modify its values.
2722  return static_cast<LO> (0);
2723  }
2724  auto curRowVals = this->getValuesViewHostNonConst (rowInfo);
2725  return this->transformGlobalValues (curRowVals.data (), graph,
2726  rowInfo, inputCols, inputVals,
2727  numInputEnt, f, atomic);
2728  }
2729 
2730  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2731  LocalOrdinal
2732  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2733  transformLocalValues (impl_scalar_type rowVals[],
2734  const crs_graph_type& graph,
2735  const RowInfo& rowInfo,
2736  const LocalOrdinal inds[],
2737  const impl_scalar_type newVals[],
2738  const LocalOrdinal numElts,
2739  std::function<impl_scalar_type (const impl_scalar_type&, const impl_scalar_type&) > f,
2740  const bool atomic)
2741  {
2742  typedef impl_scalar_type ST;
2743  typedef LocalOrdinal LO;
2744  typedef GlobalOrdinal GO;
2745 
2746  //if (newVals.extent (0) != inds.extent (0)) {
2747  // The sizes of the input arrays must match.
2748  //return Tpetra::Details::OrdinalTraits<LO>::invalid ();
2749  //}
2750  //const LO numElts = static_cast<LO> (inds.extent (0));
2751  const bool sorted = graph.isSorted ();
2752 
2753  LO numValid = 0; // number of valid input column indices
2754  size_t hint = 0; // Guess for the current index k into rowVals
2755 
2756  if (graph.isLocallyIndexed ()) {
2757  // Get a view of the column indices in the row. This amortizes
2758  // the cost of getting the view over all the entries of inds.
2759  auto colInds = graph.getLocalIndsViewHost (rowInfo);
2760 
2761  for (LO j = 0; j < numElts; ++j) {
2762  const LO lclColInd = inds[j];
2763  const size_t offset =
2764  KokkosSparse::findRelOffset (colInds, rowInfo.numEntries,
2765  lclColInd, hint, sorted);
2766  if (offset != rowInfo.numEntries) {
2767  if (atomic) {
2768  // NOTE (mfh 30 Nov 2015) The commented-out code is
2769  // wrong because another thread may have changed
2770  // rowVals[offset] between those two lines of code.
2771  //
2772  //const ST newVal = f (rowVals[offset], newVals[j]);
2773  //Kokkos::atomic_assign (&rowVals[offset], newVal);
2774 
2775  ST* const dest = &rowVals[offset];
2776  (void) atomic_binary_function_update (dest, newVals[j], f);
2777  }
2778  else {
2779  // use binary function f
2780  rowVals[offset] = f (rowVals[offset], newVals[j]);
2781  }
2782  hint = offset + 1;
2783  ++numValid;
2784  }
2785  }
2786  }
2787  else if (graph.isGloballyIndexed ()) {
2788  // NOTE (mfh 26 Nov 2015) Dereferencing an RCP or reading its
2789  // pointer does NOT change its reference count. Thus, this
2790  // code is still thread safe.
2791  if (graph.colMap_.is_null ()) {
2792  // NO input column indices are valid in this case. Either
2793  // the column Map hasn't been set yet (so local indices
2794  // don't exist yet), or the calling process owns no graph
2795  // entries.
2796  return numValid;
2797  }
2798  const map_type& colMap = * (graph.colMap_);
2799  // Get a view of the column indices in the row. This amortizes
2800  // the cost of getting the view over all the entries of inds.
2801  auto colInds = graph.getGlobalIndsViewHost (rowInfo);
2802 
2803  const GO GINV = Teuchos::OrdinalTraits<GO>::invalid ();
2804  for (LO j = 0; j < numElts; ++j) {
2805  const GO gblColInd = colMap.getGlobalElement (inds[j]);
2806  if (gblColInd != GINV) {
2807  const size_t offset =
2808  KokkosSparse::findRelOffset (colInds, rowInfo.numEntries,
2809  gblColInd, hint, sorted);
2810  if (offset != rowInfo.numEntries) {
2811  if (atomic) {
2812  // NOTE (mfh 30 Nov 2015) The commented-out code is
2813  // wrong because another thread may have changed
2814  // rowVals[offset] between those two lines of code.
2815  //
2816  //const ST newVal = f (rowVals[offset], newVals[j]);
2817  //Kokkos::atomic_assign (&rowVals[offset], newVal);
2818 
2819  ST* const dest = &rowVals[offset];
2820  (void) atomic_binary_function_update (dest, newVals[j], f);
2821  }
2822  else {
2823  // use binary function f
2824  rowVals[offset] = f (rowVals[offset], newVals[j]);
2825  }
2826  hint = offset + 1;
2827  numValid++;
2828  }
2829  }
2830  }
2831  }
2832  // If the graph is neither locally nor globally indexed on the
2833  // calling process, that means the calling process has no graph
2834  // entries. Thus, none of the input column indices are valid.
2835 
2836  return numValid;
2837  }
2838 
2839  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2840  LocalOrdinal
2841  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2842  transformGlobalValues (impl_scalar_type rowVals[],
2843  const crs_graph_type& graph,
2844  const RowInfo& rowInfo,
2845  const GlobalOrdinal inds[],
2846  const impl_scalar_type newVals[],
2847  const LocalOrdinal numElts,
2848  std::function<impl_scalar_type (const impl_scalar_type&, const impl_scalar_type&) > f,
2849  const bool atomic)
2850  {
2851  typedef impl_scalar_type ST;
2852  typedef LocalOrdinal LO;
2853  typedef GlobalOrdinal GO;
2854 
2855  //if (newVals.extent (0) != inds.extent (0)) {
2856  // The sizes of the input arrays must match.
2857  //return Tpetra::Details::OrdinalTraits<LO>::invalid ();
2858  //}
2859  //const LO numElts = static_cast<LO> (inds.extent (0));
2860  const bool sorted = graph.isSorted ();
2861 
2862  LO numValid = 0; // number of valid input column indices
2863  size_t hint = 0; // Guess for the current index k into rowVals
2864 
2865  if (graph.isGloballyIndexed ()) {
2866  // Get a view of the column indices in the row. This amortizes
2867  // the cost of getting the view over all the entries of inds.
2868  auto colInds = graph.getGlobalIndsViewHost (rowInfo);
2869 
2870  for (LO j = 0; j < numElts; ++j) {
2871  const GO gblColInd = inds[j];
2872  const size_t offset =
2873  KokkosSparse::findRelOffset (colInds, rowInfo.numEntries,
2874  gblColInd, hint, sorted);
2875  if (offset != rowInfo.numEntries) {
2876  if (atomic) {
2877  // NOTE (mfh 30 Nov 2015) The commented-out code is
2878  // wrong because another thread may have changed
2879  // rowVals[offset] between those two lines of code.
2880  //
2881  //const ST newVal = f (rowVals[offset], newVals[j]);
2882  //Kokkos::atomic_assign (&rowVals[offset], newVal);
2883 
2884  ST* const dest = &rowVals[offset];
2885  (void) atomic_binary_function_update (dest, newVals[j], f);
2886  }
2887  else {
2888  // use binary function f
2889  rowVals[offset] = f (rowVals[offset], newVals[j]);
2890  }
2891  hint = offset + 1;
2892  ++numValid;
2893  }
2894  }
2895  }
2896  else if (graph.isLocallyIndexed ()) {
2897  // NOTE (mfh 26 Nov 2015) Dereferencing an RCP or reading its
2898  // pointer does NOT change its reference count. Thus, this
2899  // code is still thread safe.
2900  if (graph.colMap_.is_null ()) {
2901  // NO input column indices are valid in this case. Either the
2902  // column Map hasn't been set yet (so local indices don't
2903  // exist yet), or the calling process owns no graph entries.
2904  return numValid;
2905  }
2906  const map_type& colMap = * (graph.colMap_);
2907  // Get a view of the column indices in the row. This amortizes
2908  // the cost of getting the view over all the entries of inds.
2909  auto colInds = graph.getLocalIndsViewHost (rowInfo);
2910 
2911  const LO LINV = Teuchos::OrdinalTraits<LO>::invalid ();
2912  for (LO j = 0; j < numElts; ++j) {
2913  const LO lclColInd = colMap.getLocalElement (inds[j]);
2914  if (lclColInd != LINV) {
2915  const size_t offset =
2916  KokkosSparse::findRelOffset (colInds, rowInfo.numEntries,
2917  lclColInd, hint, sorted);
2918  if (offset != rowInfo.numEntries) {
2919  if (atomic) {
2920  // NOTE (mfh 30 Nov 2015) The commented-out code is
2921  // wrong because another thread may have changed
2922  // rowVals[offset] between those two lines of code.
2923  //
2924  //const ST newVal = f (rowVals[offset], newVals[j]);
2925  //Kokkos::atomic_assign (&rowVals[offset], newVal);
2926 
2927  ST* const dest = &rowVals[offset];
2928  (void) atomic_binary_function_update (dest, newVals[j], f);
2929  }
2930  else {
2931  // use binary function f
2932  rowVals[offset] = f (rowVals[offset], newVals[j]);
2933  }
2934  hint = offset + 1;
2935  numValid++;
2936  }
2937  }
2938  }
2939  }
2940  // If the graph is neither locally nor globally indexed on the
2941  // calling process, that means the calling process has no graph
2942  // entries. Thus, none of the input column indices are valid.
2943 
2944  return numValid;
2945  }
2946 
2947  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
2948  LocalOrdinal
2949  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
2950  sumIntoLocalValuesImpl (impl_scalar_type rowVals[],
2951  const crs_graph_type& graph,
2952  const RowInfo& rowInfo,
2953  const LocalOrdinal inds[],
2954  const impl_scalar_type newVals[],
2955  const LocalOrdinal numElts,
2956  const bool atomic)
2957  {
2958  typedef LocalOrdinal LO;
2959  typedef GlobalOrdinal GO;
2960 
2961  const bool sorted = graph.isSorted ();
2962 
2963  size_t hint = 0; // Guess for the current index k into rowVals
2964  LO numValid = 0; // number of valid local column indices
2965 
2966  if (graph.isLocallyIndexed ()) {
2967  // Get a view of the column indices in the row. This amortizes
2968  // the cost of getting the view over all the entries of inds.
2969  auto colInds = graph.getLocalIndsViewHost (rowInfo);
2970 
2971  for (LO j = 0; j < numElts; ++j) {
2972  const LO lclColInd = inds[j];
2973  const size_t offset =
2974  KokkosSparse::findRelOffset (colInds, rowInfo.numEntries,
2975  lclColInd, hint, sorted);
2976  if (offset != rowInfo.numEntries) {
2977  if (atomic) {
2978  Kokkos::atomic_add (&rowVals[offset], newVals[j]);
2979  }
2980  else {
2981  rowVals[offset] += newVals[j];
2982  }
2983  hint = offset + 1;
2984  ++numValid;
2985  }
2986  }
2987  }
2988  else if (graph.isGloballyIndexed ()) {
2989  if (graph.colMap_.is_null ()) {
2990  return Teuchos::OrdinalTraits<LO>::invalid ();
2991  }
2992  const map_type colMap = * (graph.colMap_);
2993 
2994  // Get a view of the column indices in the row. This amortizes
2995  // the cost of getting the view over all the entries of inds.
2996  auto colInds = graph.getGlobalIndsViewHost (rowInfo);
2997 
2998  for (LO j = 0; j < numElts; ++j) {
2999  const GO gblColInd = colMap.getGlobalElement (inds[j]);
3000  if (gblColInd != Teuchos::OrdinalTraits<GO>::invalid ()) {
3001  const size_t offset =
3002  KokkosSparse::findRelOffset (colInds, rowInfo.numEntries,
3003  gblColInd, hint, sorted);
3004  if (offset != rowInfo.numEntries) {
3005  if (atomic) {
3006  Kokkos::atomic_add (&rowVals[offset], newVals[j]);
3007  }
3008  else {
3009  rowVals[offset] += newVals[j];
3010  }
3011  hint = offset + 1;
3012  ++numValid;
3013  }
3014  }
3015  }
3016  }
3017  // NOTE (mfh 26 Jun 2014, 26 Nov 2015) In the current version of
3018  // CrsGraph and CrsMatrix, it's possible for a matrix (or graph)
3019  // to be neither locally nor globally indexed on a process.
3020  // This means that the graph or matrix has no entries on that
3021  // process. Epetra also works like this. It's related to lazy
3022  // allocation (on first insertion, not at graph / matrix
3023  // construction). Lazy allocation will go away because it is
3024  // not thread scalable.
3025 
3026  return numValid;
3027  }
3028 
3029  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3030  LocalOrdinal
3031  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3032  sumIntoLocalValues (const LocalOrdinal localRow,
3033  const Teuchos::ArrayView<const LocalOrdinal>& indices,
3034  const Teuchos::ArrayView<const Scalar>& values,
3035  const bool atomic)
3036  {
3037  using LO = local_ordinal_type;
3038  const LO numInputEnt = static_cast<LO>(indices.size());
3039  if (static_cast<LO>(values.size()) != numInputEnt) {
3040  return Teuchos::OrdinalTraits<LO>::invalid();
3041  }
3042  const LO* const inputInds = indices.getRawPtr();
3043  const scalar_type* const inputVals = values.getRawPtr();
3044  return this->sumIntoLocalValues(localRow, numInputEnt,
3045  inputVals, inputInds, atomic);
3046  }
3047 
3048  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3049  typename CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3050  local_ordinal_type
3051  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3052  sumIntoLocalValues(
3053  const local_ordinal_type localRow,
3054  const Kokkos::View<const local_ordinal_type*, Kokkos::AnonymousSpace>& inputInds,
3055  const Kokkos::View<const impl_scalar_type*, Kokkos::AnonymousSpace>& inputVals,
3056  const bool atomic)
3057  {
3058  using LO = local_ordinal_type;
3059  const LO numInputEnt = static_cast<LO>(inputInds.extent(0));
3060  if (static_cast<LO>(inputVals.extent(0)) != numInputEnt) {
3061  return Teuchos::OrdinalTraits<LO>::invalid();
3062  }
3063  const scalar_type* inVals =
3064  reinterpret_cast<const scalar_type*>(inputVals.data());
3065  return this->sumIntoLocalValues(localRow, numInputEnt, inVals,
3066  inputInds.data(), atomic);
3067  }
3068 
3069  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3070  LocalOrdinal
3071  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3072  sumIntoLocalValues (const LocalOrdinal localRow,
3073  const LocalOrdinal numEnt,
3074  const Scalar vals[],
3075  const LocalOrdinal cols[],
3076  const bool atomic)
3077  {
3078  typedef impl_scalar_type IST;
3079  typedef LocalOrdinal LO;
3080 
3081  if (! this->isFillActive () || this->staticGraph_.is_null ()) {
3082  // Fill must be active and the "nonconst" graph must exist.
3083  return Teuchos::OrdinalTraits<LO>::invalid ();
3084  }
3085  const crs_graph_type& graph = * (this->staticGraph_);
3086  const RowInfo rowInfo = graph.getRowInfo (localRow);
3087 
3088  if (rowInfo.localRow == Teuchos::OrdinalTraits<size_t>::invalid ()) {
3089  // The calling process does not own this row, so it is not
3090  // allowed to modify its values.
3091  return static_cast<LO> (0);
3092  }
3093  auto curRowVals = this->getValuesViewHostNonConst (rowInfo);
3094  const IST* const inputVals = reinterpret_cast<const IST*> (vals);
3095  return this->sumIntoLocalValuesImpl (curRowVals.data (), graph, rowInfo,
3096  cols, inputVals, numEnt, atomic);
3097  }
3098 
3099  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3100  typename CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3101  values_dualv_type::t_host::const_type
3102  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3103  getValuesViewHost (const RowInfo& rowinfo) const
3104  {
3105  if (rowinfo.allocSize == 0 || valuesUnpacked_wdv.extent(0) == 0)
3106  return typename values_dualv_type::t_host::const_type ();
3107  else
3108  return valuesUnpacked_wdv.getHostSubview(rowinfo.offset1D,
3109  rowinfo.allocSize,
3110  Access::ReadOnly);
3111  }
3112 
3113  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3114  typename CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3115  values_dualv_type::t_host
3116  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3117  getValuesViewHostNonConst (const RowInfo& rowinfo)
3118  {
3119  if (rowinfo.allocSize == 0 || valuesUnpacked_wdv.extent(0) == 0)
3120  return typename values_dualv_type::t_host ();
3121  else
3122  return valuesUnpacked_wdv.getHostSubview(rowinfo.offset1D,
3123  rowinfo.allocSize,
3124  Access::ReadWrite);
3125  }
3126 
3127  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3128  typename CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3129  values_dualv_type::t_dev::const_type
3130  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3131  getValuesViewDevice (const RowInfo& rowinfo) const
3132  {
3133  if (rowinfo.allocSize == 0 || valuesUnpacked_wdv.extent(0) == 0)
3134  return typename values_dualv_type::t_dev::const_type ();
3135  else
3136  return valuesUnpacked_wdv.getDeviceSubview(rowinfo.offset1D,
3137  rowinfo.allocSize,
3138  Access::ReadOnly);
3139  }
3140 
3141  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3142  typename CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3143  values_dualv_type::t_dev
3144  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3145  getValuesViewDeviceNonConst (const RowInfo& rowinfo)
3146  {
3147  if (rowinfo.allocSize == 0 || valuesUnpacked_wdv.extent(0) == 0)
3148  return typename values_dualv_type::t_dev ();
3149  else
3150  return valuesUnpacked_wdv.getDeviceSubview(rowinfo.offset1D,
3151  rowinfo.allocSize,
3152  Access::ReadWrite);
3153  }
3154 
3155 
3156  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3157  void
3158  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3159  getLocalRowCopy (local_ordinal_type localRow,
3160  nonconst_local_inds_host_view_type &indices,
3161  nonconst_values_host_view_type &values,
3162  size_t& numEntries) const
3163  {
3164  using Teuchos::ArrayView;
3165  using Teuchos::av_reinterpret_cast;
3166  const char tfecfFuncName[] = "getLocalRowCopy: ";
3167 
3168  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3169  (! this->hasColMap (), std::runtime_error,
3170  "The matrix does not have a column Map yet. This means we don't have "
3171  "local indices for columns yet, so it doesn't make sense to call this "
3172  "method. If the matrix doesn't have a column Map yet, you should call "
3173  "fillComplete on it first.");
3174 
3175  const RowInfo rowinfo = staticGraph_->getRowInfo (localRow);
3176  const size_t theNumEntries = rowinfo.numEntries;
3177  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3178  (static_cast<size_t> (indices.size ()) < theNumEntries ||
3179  static_cast<size_t> (values.size ()) < theNumEntries,
3180  std::runtime_error, "Row with local index " << localRow << " has " <<
3181  theNumEntries << " entry/ies, but indices.size() = " <<
3182  indices.size () << " and values.size() = " << values.size () << ".");
3183  numEntries = theNumEntries; // first side effect
3184 
3185  if (rowinfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid ()) {
3186  if (staticGraph_->isLocallyIndexed ()) {
3187  auto curLclInds = staticGraph_->getLocalIndsViewHost(rowinfo);
3188  auto curVals = getValuesViewHost(rowinfo);
3189 
3190  for (size_t j = 0; j < theNumEntries; ++j) {
3191  values[j] = curVals[j];
3192  indices[j] = curLclInds(j);
3193  }
3194  }
3195  else if (staticGraph_->isGloballyIndexed ()) {
3196  // Don't call getColMap(), because it touches RCP's reference count.
3197  const map_type& colMap = * (staticGraph_->colMap_);
3198  auto curGblInds = staticGraph_->getGlobalIndsViewHost(rowinfo);
3199  auto curVals = getValuesViewHost(rowinfo);
3200 
3201  for (size_t j = 0; j < theNumEntries; ++j) {
3202  values[j] = curVals[j];
3203  indices[j] = colMap.getLocalElement (curGblInds(j));
3204  }
3205  }
3206  }
3207  }
3208 
3209 
3210 template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3211 void
3212 CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3213  getGlobalRowCopy (global_ordinal_type globalRow,
3214  nonconst_global_inds_host_view_type &indices,
3215  nonconst_values_host_view_type &values,
3216  size_t& numEntries) const
3217  {
3218  using Teuchos::ArrayView;
3219  using Teuchos::av_reinterpret_cast;
3220  const char tfecfFuncName[] = "getGlobalRowCopy: ";
3221 
3222  const RowInfo rowinfo =
3223  staticGraph_->getRowInfoFromGlobalRowIndex (globalRow);
3224  const size_t theNumEntries = rowinfo.numEntries;
3225  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3226  static_cast<size_t> (indices.size ()) < theNumEntries ||
3227  static_cast<size_t> (values.size ()) < theNumEntries,
3228  std::runtime_error, "Row with global index " << globalRow << " has "
3229  << theNumEntries << " entry/ies, but indices.size() = " <<
3230  indices.size () << " and values.size() = " << values.size () << ".");
3231  numEntries = theNumEntries; // first side effect
3232 
3233  if (rowinfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid ()) {
3234  if (staticGraph_->isLocallyIndexed ()) {
3235  const map_type& colMap = * (staticGraph_->colMap_);
3236  auto curLclInds = staticGraph_->getLocalIndsViewHost(rowinfo);
3237  auto curVals = getValuesViewHost(rowinfo);
3238 
3239  for (size_t j = 0; j < theNumEntries; ++j) {
3240  values[j] = curVals[j];
3241  indices[j] = colMap.getGlobalElement (curLclInds(j));
3242  }
3243  }
3244  else if (staticGraph_->isGloballyIndexed ()) {
3245  auto curGblInds = staticGraph_->getGlobalIndsViewHost(rowinfo);
3246  auto curVals = getValuesViewHost(rowinfo);
3247 
3248  for (size_t j = 0; j < theNumEntries; ++j) {
3249  values[j] = curVals[j];
3250  indices[j] = curGblInds(j);
3251  }
3252  }
3253  }
3254  }
3255 
3256 
3257  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3258  void
3259  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3260  getLocalRowView(LocalOrdinal localRow,
3261  local_inds_host_view_type &indices,
3262  values_host_view_type &values) const
3263  {
3264  const char tfecfFuncName[] = "getLocalRowView: ";
3265 
3266  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3267  isGloballyIndexed (), std::runtime_error, "The matrix currently stores "
3268  "its indices as global indices, so you cannot get a view with local "
3269  "column indices. If the matrix has a column Map, you may call "
3270  "getLocalRowCopy() to get local column indices; otherwise, you may get "
3271  "a view with global column indices by calling getGlobalRowCopy().");
3272 
3273  const RowInfo rowInfo = staticGraph_->getRowInfo (localRow);
3274  if (rowInfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid () &&
3275  rowInfo.numEntries > 0) {
3276  indices = staticGraph_->lclIndsUnpacked_wdv.getHostSubview(
3277  rowInfo.offset1D,
3278  rowInfo.numEntries,
3279  Access::ReadOnly);
3280  values = valuesUnpacked_wdv.getHostSubview(rowInfo.offset1D,
3281  rowInfo.numEntries,
3282  Access::ReadOnly);
3283  }
3284  else {
3285  // This does the right thing (reports an empty row) if the input
3286  // row is invalid.
3287  indices = local_inds_host_view_type();
3288  values = values_host_view_type();
3289  }
3290 
3291 #ifdef HAVE_TPETRA_DEBUG
3292  const char suffix[] = ". This should never happen. Please report this "
3293  "bug to the Tpetra developers.";
3294  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3295  (static_cast<size_t> (indices.size ()) !=
3296  static_cast<size_t> (values.size ()), std::logic_error,
3297  "At the end of this method, for local row " << localRow << ", "
3298  "indices.size() = " << indices.size () << " != values.size () = "
3299  << values.size () << suffix);
3300  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3301  (static_cast<size_t> (indices.size ()) !=
3302  static_cast<size_t> (rowInfo.numEntries), std::logic_error,
3303  "At the end of this method, for local row " << localRow << ", "
3304  "indices.size() = " << indices.size () << " != rowInfo.numEntries = "
3305  << rowInfo.numEntries << suffix);
3306  const size_t expectedNumEntries = getNumEntriesInLocalRow (localRow);
3307  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3308  (rowInfo.numEntries != expectedNumEntries, std::logic_error, "At the end "
3309  "of this method, for local row " << localRow << ", rowInfo.numEntries = "
3310  << rowInfo.numEntries << " != getNumEntriesInLocalRow(localRow) = " <<
3311  expectedNumEntries << suffix);
3312 #endif // HAVE_TPETRA_DEBUG
3313  }
3314 
3315 
3316  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3317  void
3318  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3319  getGlobalRowView (GlobalOrdinal globalRow,
3320  global_inds_host_view_type &indices,
3321  values_host_view_type &values) const
3322  {
3323  const char tfecfFuncName[] = "getGlobalRowView: ";
3324 
3325  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3326  isLocallyIndexed (), std::runtime_error,
3327  "The matrix is locally indexed, so we cannot return a view of the row "
3328  "with global column indices. Use getGlobalRowCopy() instead.");
3329 
3330  // This does the right thing (reports an empty row) if the input
3331  // row is invalid.
3332  const RowInfo rowInfo =
3333  staticGraph_->getRowInfoFromGlobalRowIndex (globalRow);
3334  if (rowInfo.localRow != Teuchos::OrdinalTraits<size_t>::invalid () &&
3335  rowInfo.numEntries > 0) {
3336  indices = staticGraph_->gblInds_wdv.getHostSubview(rowInfo.offset1D,
3337  rowInfo.numEntries,
3338  Access::ReadOnly);
3339  values = valuesUnpacked_wdv.getHostSubview(rowInfo.offset1D,
3340  rowInfo.numEntries,
3341  Access::ReadOnly);
3342  }
3343  else {
3344  indices = global_inds_host_view_type();
3345  values = values_host_view_type();
3346  }
3347 
3348 #ifdef HAVE_TPETRA_DEBUG
3349  const char suffix[] = ". This should never happen. Please report this "
3350  "bug to the Tpetra developers.";
3351  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3352  (static_cast<size_t> (indices.size ()) !=
3353  static_cast<size_t> (values.size ()), std::logic_error,
3354  "At the end of this method, for global row " << globalRow << ", "
3355  "indices.size() = " << indices.size () << " != values.size () = "
3356  << values.size () << suffix);
3357  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3358  (static_cast<size_t> (indices.size ()) !=
3359  static_cast<size_t> (rowInfo.numEntries), std::logic_error,
3360  "At the end of this method, for global row " << globalRow << ", "
3361  "indices.size() = " << indices.size () << " != rowInfo.numEntries = "
3362  << rowInfo.numEntries << suffix);
3363  const size_t expectedNumEntries = getNumEntriesInGlobalRow (globalRow);
3364  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3365  (rowInfo.numEntries != expectedNumEntries, std::logic_error, "At the end "
3366  "of this method, for global row " << globalRow << ", rowInfo.numEntries "
3367  "= " << rowInfo.numEntries << " != getNumEntriesInGlobalRow(globalRow) ="
3368  " " << expectedNumEntries << suffix);
3369 #endif // HAVE_TPETRA_DEBUG
3370  }
3371 
3372 
3373  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3374  void
3375  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3376  scale (const Scalar& alpha)
3377  {
3378  const impl_scalar_type theAlpha = static_cast<impl_scalar_type> (alpha);
3379 
3380  const size_t nlrs = staticGraph_->getLocalNumRows ();
3381  const size_t numEntries = staticGraph_->getLocalNumEntries ();
3382  if (! staticGraph_->indicesAreAllocated () ||
3383  nlrs == 0 || numEntries == 0) {
3384  // do nothing
3385  }
3386  else {
3387 
3388  auto vals = valuesPacked_wdv.getDeviceView(Access::ReadWrite);
3389  KokkosBlas::scal(vals, theAlpha, vals);
3390 
3391  }
3392  }
3393 
3394  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3395  void
3396  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3397  setAllToScalar (const Scalar& alpha)
3398  {
3399  const impl_scalar_type theAlpha = static_cast<impl_scalar_type> (alpha);
3400 
3401  // replace all values in the matrix
3402  // it is easiest to replace all allocated values, instead of replacing only the ones with valid entries
3403  // however, if there are no valid entries, we can short-circuit
3404  // furthermore, if the values aren't allocated, we can short-circuit (no entry have been inserted so far)
3405  const size_t numEntries = staticGraph_->getLocalNumEntries();
3406  if (! staticGraph_->indicesAreAllocated () || numEntries == 0) {
3407  // do nothing
3408  }
3409  else {
3410  // DEEP_COPY REVIEW - VALUE-TO-DEVICE
3411  Kokkos::deep_copy (execution_space(), valuesUnpacked_wdv.getDeviceView(Access::OverwriteAll),
3412  theAlpha);
3413  // CAG: This fence was found to be required on Cuda with UVM=on.
3414  Kokkos::fence("CrsMatrix::setAllToScalar");
3415  }
3416  }
3417 
3418  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3419  void
3420  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3421  setAllValues (const typename local_graph_device_type::row_map_type& rowPointers,
3422  const typename local_graph_device_type::entries_type::non_const_type& columnIndices,
3423  const typename local_matrix_device_type::values_type& values)
3424  {
3425  using ProfilingRegion=Details::ProfilingRegion;
3426  ProfilingRegion region ("Tpetra::CrsMatrix::setAllValues");
3427  const char tfecfFuncName[] = "setAllValues: ";
3428  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3429  (columnIndices.size () != values.size (), std::invalid_argument,
3430  "columnIndices.size() = " << columnIndices.size () << " != values.size()"
3431  " = " << values.size () << ".");
3432  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3433  (myGraph_.is_null (), std::runtime_error, "myGraph_ must not be null.");
3434 
3435  try {
3436  myGraph_->setAllIndices (rowPointers, columnIndices);
3437  }
3438  catch (std::exception &e) {
3439  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3440  (true, std::runtime_error, "myGraph_->setAllIndices() threw an "
3441  "exception: " << e.what ());
3442  }
3443 
3444  // Make sure that myGraph_ now has a local graph. It may not be
3445  // fillComplete yet, so it's important to check. We don't care
3446  // whether setAllIndices() did a shallow copy or a deep copy, so a
3447  // good way to check is to compare dimensions.
3448  auto lclGraph = myGraph_->getLocalGraphDevice ();
3449  const size_t numEnt = lclGraph.entries.extent (0);
3450  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3451  (lclGraph.row_map.extent (0) != rowPointers.extent (0) ||
3452  numEnt != static_cast<size_t> (columnIndices.extent (0)),
3453  std::logic_error, "myGraph_->setAllIndices() did not correctly create "
3454  "local graph. Please report this bug to the Tpetra developers.");
3455 
3456  valuesPacked_wdv = values_wdv_type(values);
3457  valuesUnpacked_wdv = valuesPacked_wdv;
3458 
3459  // Storage MUST be packed, since the interface doesn't give any
3460  // way to indicate any extra space at the end of each row.
3461  this->storageStatus_ = Details::STORAGE_1D_PACKED;
3462 
3463  checkInternalState ();
3464  }
3465 
3466  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3467  void
3468  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3469  setAllValues ( const local_matrix_device_type& localDeviceMatrix)
3470  {
3471  using ProfilingRegion=Details::ProfilingRegion;
3472  ProfilingRegion region ("Tpetra::CrsMatrix::setAllValues from KokkosSparse::CrsMatrix");
3473 
3474  auto graph = localDeviceMatrix.graph;
3475  //FIXME how to check whether graph is allocated
3476 
3477  auto rows = graph.row_map;
3478  auto columns = graph.entries;
3479  auto values = localDeviceMatrix.values;
3480 
3481  setAllValues(rows,columns,values);
3482  }
3483 
3484  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3485  void
3486  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3487  setAllValues (const Teuchos::ArrayRCP<size_t>& ptr,
3488  const Teuchos::ArrayRCP<LocalOrdinal>& ind,
3489  const Teuchos::ArrayRCP<Scalar>& val)
3490  {
3491  using Kokkos::Compat::getKokkosViewDeepCopy;
3492  using Teuchos::ArrayRCP;
3493  using Teuchos::av_reinterpret_cast;
3494  typedef device_type DT;
3495  typedef impl_scalar_type IST;
3496  typedef typename local_graph_device_type::row_map_type row_map_type;
3497  //typedef typename row_map_type::non_const_value_type row_offset_type;
3498  const char tfecfFuncName[] = "setAllValues(ArrayRCP<size_t>, ArrayRCP<LO>, ArrayRCP<Scalar>): ";
3499 
3500  // The row offset type may depend on the execution space. It may
3501  // not necessarily be size_t. If it's not, we need to make a deep
3502  // copy. We need to make a deep copy anyway so that Kokkos can
3503  // own the memory. Regardless, ptrIn gets the copy.
3504  typename row_map_type::non_const_type ptrNative ("ptr", ptr.size ());
3505  Kokkos::View<const size_t*,
3506  typename row_map_type::array_layout,
3507  Kokkos::HostSpace,
3508  Kokkos::MemoryUnmanaged> ptrSizeT (ptr.getRawPtr (), ptr.size ());
3509  ::Tpetra::Details::copyOffsets (ptrNative, ptrSizeT);
3510 
3511  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3512  (ptrNative.extent (0) != ptrSizeT.extent (0),
3513  std::logic_error, "ptrNative.extent(0) = " <<
3514  ptrNative.extent (0) << " != ptrSizeT.extent(0) = "
3515  << ptrSizeT.extent (0) << ". Please report this bug to the "
3516  "Tpetra developers.");
3517 
3518  auto indIn = getKokkosViewDeepCopy<DT> (ind ());
3519  auto valIn = getKokkosViewDeepCopy<DT> (av_reinterpret_cast<IST> (val ()));
3520  this->setAllValues (ptrNative, indIn, valIn);
3521  }
3522 
3523  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3524  void
3525  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3526  getLocalDiagOffsets (Teuchos::ArrayRCP<size_t>& offsets) const
3527  {
3528  const char tfecfFuncName[] = "getLocalDiagOffsets: ";
3529  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3530  (staticGraph_.is_null (), std::runtime_error, "The matrix has no graph.");
3531 
3532  // mfh 11 May 2016: We plan to deprecate the ArrayRCP version of
3533  // this method in CrsGraph too, so don't call it (otherwise build
3534  // warnings will show up and annoy users). Instead, copy results
3535  // in and out, if the memory space requires it.
3536 
3537  const size_t lclNumRows = staticGraph_->getLocalNumRows ();
3538  if (static_cast<size_t> (offsets.size ()) < lclNumRows) {
3539  offsets.resize (lclNumRows);
3540  }
3541 
3542  // The input ArrayRCP must always be a host pointer. Thus, if
3543  // device_type::memory_space is Kokkos::HostSpace, it's OK for us
3544  // to write to that allocation directly as a Kokkos::View.
3545  if (std::is_same<memory_space, Kokkos::HostSpace>::value) {
3546  // It is always syntactically correct to assign a raw host
3547  // pointer to a device View, so this code will compile correctly
3548  // even if this branch never runs.
3549  typedef Kokkos::View<size_t*, device_type,
3550  Kokkos::MemoryUnmanaged> output_type;
3551  output_type offsetsOut (offsets.getRawPtr (), lclNumRows);
3552  staticGraph_->getLocalDiagOffsets (offsetsOut);
3553  }
3554  else {
3555  Kokkos::View<size_t*, device_type> offsetsTmp ("diagOffsets", lclNumRows);
3556  staticGraph_->getLocalDiagOffsets (offsetsTmp);
3557  typedef Kokkos::View<size_t*, Kokkos::HostSpace,
3558  Kokkos::MemoryUnmanaged> output_type;
3559  output_type offsetsOut (offsets.getRawPtr (), lclNumRows);
3560  // DEEP_COPY REVIEW - DEVICE-TO-HOST
3561  Kokkos::deep_copy (execution_space(), offsetsOut, offsetsTmp);
3562  }
3563  }
3564 
3565  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3566  void
3567  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3568  getLocalDiagCopy (Vector<Scalar, LocalOrdinal, GlobalOrdinal, Node>& diag) const
3569  {
3570  using Teuchos::ArrayRCP;
3571  using Teuchos::ArrayView;
3572  using Teuchos::av_reinterpret_cast;
3573  const char tfecfFuncName[] = "getLocalDiagCopy (1-arg): ";
3574  typedef local_ordinal_type LO;
3575 
3576 
3577  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3578  staticGraph_.is_null (), std::runtime_error,
3579  "This method requires that the matrix have a graph.");
3580  auto rowMapPtr = this->getRowMap ();
3581  if (rowMapPtr.is_null () || rowMapPtr->getComm ().is_null ()) {
3582  // Processes on which the row Map or its communicator is null
3583  // don't participate. Users shouldn't even call this method on
3584  // those processes.
3585  return;
3586  }
3587  auto colMapPtr = this->getColMap ();
3588  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3589  (! this->hasColMap () || colMapPtr.is_null (), std::runtime_error,
3590  "This method requires that the matrix have a column Map.");
3591  const map_type& rowMap = * rowMapPtr;
3592  const map_type& colMap = * colMapPtr;
3593  const LO myNumRows = static_cast<LO> (this->getLocalNumRows ());
3594 
3595 #ifdef HAVE_TPETRA_DEBUG
3596  // isCompatible() requires an all-reduce, and thus this check
3597  // should only be done in debug mode.
3598  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3599  ! diag.getMap ()->isCompatible (rowMap), std::runtime_error,
3600  "The input Vector's Map must be compatible with the CrsMatrix's row "
3601  "Map. You may check this by using Map's isCompatible method: "
3602  "diag.getMap ()->isCompatible (A.getRowMap ());");
3603 #endif // HAVE_TPETRA_DEBUG
3604 
3605  const auto D_lcl = diag.getLocalViewDevice(Access::OverwriteAll);
3606  // 1-D subview of the first (and only) column of D_lcl.
3607  const auto D_lcl_1d =
3608  Kokkos::subview (D_lcl, Kokkos::make_pair (LO (0), myNumRows), 0);
3609 
3610  const auto lclRowMap = rowMap.getLocalMap ();
3611  const auto lclColMap = colMap.getLocalMap ();
3612  using ::Tpetra::Details::getDiagCopyWithoutOffsets;
3613  (void) getDiagCopyWithoutOffsets (D_lcl_1d, lclRowMap,
3614  lclColMap,
3615  getLocalMatrixDevice ());
3616  }
3617 
3618  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3619  void
3620  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3621  getLocalDiagCopy (Vector<Scalar, LocalOrdinal, GlobalOrdinal, Node>& diag,
3622  const Kokkos::View<const size_t*, device_type,
3623  Kokkos::MemoryUnmanaged>& offsets) const
3624  {
3625  typedef LocalOrdinal LO;
3626 
3627 #ifdef HAVE_TPETRA_DEBUG
3628  const char tfecfFuncName[] = "getLocalDiagCopy: ";
3629  const map_type& rowMap = * (this->getRowMap ());
3630  // isCompatible() requires an all-reduce, and thus this check
3631  // should only be done in debug mode.
3632  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3633  ! diag.getMap ()->isCompatible (rowMap), std::runtime_error,
3634  "The input Vector's Map must be compatible with (in the sense of Map::"
3635  "isCompatible) the CrsMatrix's row Map.");
3636 #endif // HAVE_TPETRA_DEBUG
3637 
3638  // For now, we fill the Vector on the host and sync to device.
3639  // Later, we may write a parallel kernel that works entirely on
3640  // device.
3641  //
3642  // NOTE (mfh 21 Jan 2016): The host kernel here assumes UVM. Once
3643  // we write a device kernel, it will not need to assume UVM.
3644 
3645  auto D_lcl = diag.getLocalViewDevice (Access::OverwriteAll);
3646  const LO myNumRows = static_cast<LO> (this->getLocalNumRows ());
3647  // Get 1-D subview of the first (and only) column of D_lcl.
3648  auto D_lcl_1d =
3649  Kokkos::subview (D_lcl, Kokkos::make_pair (LO (0), myNumRows), 0);
3650 
3651  KokkosSparse::getDiagCopy (D_lcl_1d, offsets,
3652  getLocalMatrixDevice ());
3653  }
3654 
3655  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3656  void
3657  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3658  getLocalDiagCopy (Vector<Scalar, LocalOrdinal, GlobalOrdinal, Node>& diag,
3659  const Teuchos::ArrayView<const size_t>& offsets) const
3660  {
3661  using LO = LocalOrdinal;
3662  using host_execution_space = Kokkos::DefaultHostExecutionSpace;
3663  using IST = impl_scalar_type;
3664 
3665 #ifdef HAVE_TPETRA_DEBUG
3666  const char tfecfFuncName[] = "getLocalDiagCopy: ";
3667  const map_type& rowMap = * (this->getRowMap ());
3668  // isCompatible() requires an all-reduce, and thus this check
3669  // should only be done in debug mode.
3670  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3671  ! diag.getMap ()->isCompatible (rowMap), std::runtime_error,
3672  "The input Vector's Map must be compatible with (in the sense of Map::"
3673  "isCompatible) the CrsMatrix's row Map.");
3674 #endif // HAVE_TPETRA_DEBUG
3675 
3676  // See #1510. In case diag has already been marked modified on
3677  // device, we need to clear that flag, since the code below works
3678  // on host.
3679  //diag.clear_sync_state ();
3680 
3681  // For now, we fill the Vector on the host and sync to device.
3682  // Later, we may write a parallel kernel that works entirely on
3683  // device.
3684  auto lclVecHost = diag.getLocalViewHost(Access::OverwriteAll);
3685  // 1-D subview of the first (and only) column of lclVecHost.
3686  auto lclVecHost1d = Kokkos::subview (lclVecHost, Kokkos::ALL (), 0);
3687 
3688  using host_offsets_view_type =
3689  Kokkos::View<const size_t*, Kokkos::HostSpace,
3690  Kokkos::MemoryTraits<Kokkos::Unmanaged> >;
3691  host_offsets_view_type h_offsets (offsets.getRawPtr (), offsets.size ());
3692  // Find the diagonal entries and put them in lclVecHost1d.
3693  using range_type = Kokkos::RangePolicy<host_execution_space, LO>;
3694  const LO myNumRows = static_cast<LO> (this->getLocalNumRows ());
3695  const size_t INV = Tpetra::Details::OrdinalTraits<size_t>::invalid ();
3696 
3697  auto rowPtrsPackedHost = staticGraph_->getRowPtrsPackedHost();
3698  auto valuesPackedHost = valuesPacked_wdv.getHostView(Access::ReadOnly);
3699  Kokkos::parallel_for
3700  ("Tpetra::CrsMatrix::getLocalDiagCopy",
3701  range_type (0, myNumRows),
3702  [&, INV, h_offsets] (const LO lclRow) { // Value capture is a workaround for cuda + gcc-7.2 compiler bug w/c++14
3703  lclVecHost1d(lclRow) = STS::zero (); // default value if no diag entry
3704  if (h_offsets[lclRow] != INV) {
3705  auto curRowOffset = rowPtrsPackedHost (lclRow);
3706  lclVecHost1d(lclRow) =
3707  static_cast<IST> (valuesPackedHost(curRowOffset+h_offsets[lclRow]));
3708  }
3709  });
3710  //diag.sync_device ();
3711  }
3712 
3713 
3714  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3715  void
3716  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3717  leftScale (const Vector<Scalar, LocalOrdinal, GlobalOrdinal, Node>& x)
3718  {
3719  using ::Tpetra::Details::ProfilingRegion;
3720  using Teuchos::ArrayRCP;
3721  using Teuchos::ArrayView;
3722  using Teuchos::null;
3723  using Teuchos::RCP;
3724  using Teuchos::rcp;
3725  using Teuchos::rcpFromRef;
3726  using vec_type = Vector<Scalar, LocalOrdinal, GlobalOrdinal, Node>;
3727  const char tfecfFuncName[] = "leftScale: ";
3728 
3729  ProfilingRegion region ("Tpetra::CrsMatrix::leftScale");
3730 
3731  RCP<const vec_type> xp;
3732  if (this->getRangeMap ()->isSameAs (* (x.getMap ()))) {
3733  // Take from Epetra: If we have a non-trivial exporter, we must
3734  // import elements that are permuted or are on other processors.
3735  auto exporter = this->getCrsGraphRef ().getExporter ();
3736  if (exporter.get () != nullptr) {
3737  RCP<vec_type> tempVec (new vec_type (this->getRowMap ()));
3738  tempVec->doImport (x, *exporter, REPLACE); // reverse mode
3739  xp = tempVec;
3740  }
3741  else {
3742  xp = rcpFromRef (x);
3743  }
3744  }
3745  else if (this->getRowMap ()->isSameAs (* (x.getMap ()))) {
3746  xp = rcpFromRef (x);
3747  }
3748  else {
3749  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3750  (true, std::invalid_argument, "x's Map must be the same as "
3751  "either the row Map or the range Map of the CrsMatrix.");
3752  }
3753 
3754  if (this->isFillComplete()) {
3755  auto x_lcl = xp->getLocalViewDevice (Access::ReadOnly);
3756  auto x_lcl_1d = Kokkos::subview (x_lcl, Kokkos::ALL (), 0);
3757  using ::Tpetra::Details::leftScaleLocalCrsMatrix;
3758  leftScaleLocalCrsMatrix (getLocalMatrixDevice (),
3759  x_lcl_1d, false, false);
3760  }
3761  else {
3762  // 6/2020 Disallow leftScale of non-fillComplete matrices #7446
3763  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3764  (true, std::runtime_error, "CrsMatrix::leftScale requires matrix to be"
3765  " fillComplete");
3766  }
3767  }
3768 
3769  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3770  void
3771  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3772  rightScale (const Vector<Scalar, LocalOrdinal, GlobalOrdinal, Node>& x)
3773  {
3774  using ::Tpetra::Details::ProfilingRegion;
3775  using Teuchos::ArrayRCP;
3776  using Teuchos::ArrayView;
3777  using Teuchos::null;
3778  using Teuchos::RCP;
3779  using Teuchos::rcp;
3780  using Teuchos::rcpFromRef;
3781  typedef Vector<Scalar, LocalOrdinal, GlobalOrdinal, Node> vec_type;
3782  const char tfecfFuncName[] = "rightScale: ";
3783 
3784  ProfilingRegion region ("Tpetra::CrsMatrix::rightScale");
3785 
3786  RCP<const vec_type> xp;
3787  if (this->getDomainMap ()->isSameAs (* (x.getMap ()))) {
3788  // Take from Epetra: If we have a non-trivial exporter, we must
3789  // import elements that are permuted or are on other processors.
3790  auto importer = this->getCrsGraphRef ().getImporter ();
3791  if (importer.get () != nullptr) {
3792  RCP<vec_type> tempVec (new vec_type (this->getColMap ()));
3793  tempVec->doImport (x, *importer, REPLACE);
3794  xp = tempVec;
3795  }
3796  else {
3797  xp = rcpFromRef (x);
3798  }
3799  }
3800  else if (this->getColMap ()->isSameAs (* (x.getMap ()))) {
3801  xp = rcpFromRef (x);
3802  } else {
3803  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3804  (true, std::runtime_error, "x's Map must be the same as "
3805  "either the domain Map or the column Map of the CrsMatrix.");
3806  }
3807 
3808  if (this->isFillComplete()) {
3809  auto x_lcl = xp->getLocalViewDevice (Access::ReadOnly);
3810  auto x_lcl_1d = Kokkos::subview (x_lcl, Kokkos::ALL (), 0);
3811  using ::Tpetra::Details::rightScaleLocalCrsMatrix;
3812  rightScaleLocalCrsMatrix (getLocalMatrixDevice (),
3813  x_lcl_1d, false, false);
3814  }
3815  else {
3816  // 6/2020 Disallow rightScale of non-fillComplete matrices #7446
3817  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
3818  (true, std::runtime_error, "CrsMatrix::rightScale requires matrix to be"
3819  " fillComplete");
3820  }
3821  }
3822 
3823  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3824  typename CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::mag_type
3825  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3826  getFrobeniusNorm () const
3827  {
3828  using Teuchos::ArrayView;
3829  using Teuchos::outArg;
3830  using Teuchos::REDUCE_SUM;
3831  using Teuchos::reduceAll;
3832 
3833  // FIXME (mfh 05 Aug 2014) Write a thread-parallel kernel for the
3834  // local part of this computation. It could make sense to put
3835  // this operation in the Kokkos::CrsMatrix.
3836 
3837  // check the cache first
3838  mag_type mySum = STM::zero ();
3839  if (getLocalNumEntries() > 0) {
3840  if (isStorageOptimized ()) {
3841  // "Optimized" storage is packed storage. That means we can
3842  // iterate in one pass through the 1-D values array.
3843  const size_t numEntries = getLocalNumEntries ();
3844  auto values = valuesPacked_wdv.getHostView(Access::ReadOnly);
3845  for (size_t k = 0; k < numEntries; ++k) {
3846  auto val = values[k];
3847  // Note (etp 06 Jan 2015) We need abs() here for composite types
3848  // (in general, if mag_type is on the left-hand-side, we need
3849  // abs() on the right-hand-side)
3850  const mag_type val_abs = STS::abs (val);
3851  mySum += val_abs * val_abs;
3852  }
3853  }
3854  else {
3855  const LocalOrdinal numRows =
3856  static_cast<LocalOrdinal> (this->getLocalNumRows ());
3857  for (LocalOrdinal r = 0; r < numRows; ++r) {
3858  const RowInfo rowInfo = myGraph_->getRowInfo (r);
3859  const size_t numEntries = rowInfo.numEntries;
3860  auto A_r = this->getValuesViewHost(rowInfo);
3861  for (size_t k = 0; k < numEntries; ++k) {
3862  const impl_scalar_type val = A_r[k];
3863  const mag_type val_abs = STS::abs (val);
3864  mySum += val_abs * val_abs;
3865  }
3866  }
3867  }
3868  }
3869  mag_type totalSum = STM::zero ();
3870  reduceAll<int, mag_type> (* (getComm ()), REDUCE_SUM,
3871  mySum, outArg (totalSum));
3872  return STM::sqrt (totalSum);
3873  }
3874 
3875  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3876  void
3877  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3878  replaceColMap (const Teuchos::RCP<const map_type>& newColMap)
3879  {
3880  const char tfecfFuncName[] = "replaceColMap: ";
3881  // FIXME (mfh 06 Aug 2014) What if the graph is locally indexed?
3882  // Then replacing the column Map might mean that we need to
3883  // reindex the column indices.
3884  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3885  myGraph_.is_null (), std::runtime_error,
3886  "This method does not work if the matrix has a const graph. The whole "
3887  "idea of a const graph is that you are not allowed to change it, but "
3888  "this method necessarily must modify the graph, since the graph owns "
3889  "the matrix's column Map.");
3890  myGraph_->replaceColMap (newColMap);
3891  }
3892 
3893  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3894  void
3895  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3896  reindexColumns (crs_graph_type* const graph,
3897  const Teuchos::RCP<const map_type>& newColMap,
3898  const Teuchos::RCP<const import_type>& newImport,
3899  const bool sortEachRow)
3900  {
3901  const char tfecfFuncName[] = "reindexColumns: ";
3902  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3903  graph == nullptr && myGraph_.is_null (), std::invalid_argument,
3904  "The input graph is null, but the matrix does not own its graph.");
3905 
3906  crs_graph_type& theGraph = (graph == nullptr) ? *myGraph_ : *graph;
3907  const bool sortGraph = false; // we'll sort graph & matrix together below
3908 
3909  theGraph.reindexColumns (newColMap, newImport, sortGraph);
3910 
3911  if (sortEachRow && theGraph.isLocallyIndexed () && ! theGraph.isSorted ()) {
3912  const LocalOrdinal lclNumRows =
3913  static_cast<LocalOrdinal> (theGraph.getLocalNumRows ());
3914 
3915  for (LocalOrdinal row = 0; row < lclNumRows; ++row) {
3916 
3917  const RowInfo rowInfo = theGraph.getRowInfo (row);
3918  auto lclColInds = theGraph.getLocalIndsViewHostNonConst (rowInfo);
3919  auto vals = this->getValuesViewHostNonConst (rowInfo);
3920 
3921  sort2 (lclColInds.data (),
3922  lclColInds.data () + rowInfo.numEntries,
3923  vals.data ());
3924  }
3925  theGraph.indicesAreSorted_ = true;
3926  }
3927  }
3928 
3929  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3930  void
3931  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3932  replaceDomainMap (const Teuchos::RCP<const map_type>& newDomainMap)
3933  {
3934  const char tfecfFuncName[] = "replaceDomainMap: ";
3935  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3936  myGraph_.is_null (), std::runtime_error,
3937  "This method does not work if the matrix has a const graph. The whole "
3938  "idea of a const graph is that you are not allowed to change it, but this"
3939  " method necessarily must modify the graph, since the graph owns the "
3940  "matrix's domain Map and Import objects.");
3941  myGraph_->replaceDomainMap (newDomainMap);
3942  }
3943 
3944  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3945  void
3946  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3947  replaceDomainMapAndImporter (const Teuchos::RCP<const map_type>& newDomainMap,
3948  Teuchos::RCP<const import_type>& newImporter)
3949  {
3950  const char tfecfFuncName[] = "replaceDomainMapAndImporter: ";
3951  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3952  myGraph_.is_null (), std::runtime_error,
3953  "This method does not work if the matrix has a const graph. The whole "
3954  "idea of a const graph is that you are not allowed to change it, but this"
3955  " method necessarily must modify the graph, since the graph owns the "
3956  "matrix's domain Map and Import objects.");
3957  myGraph_->replaceDomainMapAndImporter (newDomainMap, newImporter);
3958  }
3959 
3960  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3961  void
3962  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3963  replaceRangeMap (const Teuchos::RCP<const map_type>& newRangeMap)
3964  {
3965  const char tfecfFuncName[] = "replaceRangeMap: ";
3966  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3967  myGraph_.is_null (), std::runtime_error,
3968  "This method does not work if the matrix has a const graph. The whole "
3969  "idea of a const graph is that you are not allowed to change it, but this"
3970  " method necessarily must modify the graph, since the graph owns the "
3971  "matrix's domain Map and Import objects.");
3972  myGraph_->replaceRangeMap (newRangeMap);
3973  }
3974 
3975  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3976  void
3977  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3978  replaceRangeMapAndExporter (const Teuchos::RCP<const map_type>& newRangeMap,
3979  Teuchos::RCP<const export_type>& newExporter)
3980  {
3981  const char tfecfFuncName[] = "replaceRangeMapAndExporter: ";
3982  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
3983  myGraph_.is_null (), std::runtime_error,
3984  "This method does not work if the matrix has a const graph. The whole "
3985  "idea of a const graph is that you are not allowed to change it, but this"
3986  " method necessarily must modify the graph, since the graph owns the "
3987  "matrix's domain Map and Import objects.");
3988  myGraph_->replaceRangeMapAndExporter (newRangeMap, newExporter);
3989  }
3990 
3991  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
3992  void
3993  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
3994  insertNonownedGlobalValues (const GlobalOrdinal globalRow,
3995  const Teuchos::ArrayView<const GlobalOrdinal>& indices,
3996  const Teuchos::ArrayView<const Scalar>& values)
3997  {
3998  using Teuchos::Array;
3999  typedef GlobalOrdinal GO;
4000  typedef typename Array<GO>::size_type size_type;
4001 
4002  const size_type numToInsert = indices.size ();
4003  // Add the new data to the list of nonlocals.
4004  // This creates the arrays if they don't exist yet.
4005  std::pair<Array<GO>, Array<Scalar> >& curRow = nonlocals_[globalRow];
4006  Array<GO>& curRowInds = curRow.first;
4007  Array<Scalar>& curRowVals = curRow.second;
4008  const size_type newCapacity = curRowInds.size () + numToInsert;
4009  curRowInds.reserve (newCapacity);
4010  curRowVals.reserve (newCapacity);
4011  for (size_type k = 0; k < numToInsert; ++k) {
4012  curRowInds.push_back (indices[k]);
4013  curRowVals.push_back (values[k]);
4014  }
4015  }
4016 
4017  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
4018  void
4019  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
4020  globalAssemble ()
4021  {
4022  using Details::Behavior;
4023  using Details::ProfilingRegion;
4024  using Teuchos::Comm;
4025  using Teuchos::outArg;
4026  using Teuchos::RCP;
4027  using Teuchos::rcp;
4028  using Teuchos::REDUCE_MAX;
4029  using Teuchos::REDUCE_MIN;
4030  using Teuchos::reduceAll;
4031  using std::endl;
4032  typedef CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> crs_matrix_type;
4033  //typedef LocalOrdinal LO;
4034  typedef GlobalOrdinal GO;
4035  typedef typename Teuchos::Array<GO>::size_type size_type;
4036  const char tfecfFuncName[] = "globalAssemble: "; // for exception macro
4037  ProfilingRegion regionGlobalAssemble ("Tpetra::CrsMatrix::globalAssemble");
4038 
4039  const bool verbose = Behavior::verbose("CrsMatrix");
4040  std::unique_ptr<std::string> prefix;
4041  if (verbose) {
4042  prefix = this->createPrefix("CrsMatrix", "globalAssemble");
4043  std::ostringstream os;
4044  os << *prefix << "nonlocals_.size()=" << nonlocals_.size()
4045  << endl;
4046  std::cerr << os.str();
4047  }
4048  RCP<const Comm<int> > comm = getComm ();
4049 
4050  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4051  (! isFillActive (), std::runtime_error, "Fill must be active before "
4052  "you may call this method.");
4053 
4054  const size_t myNumNonlocalRows = nonlocals_.size ();
4055 
4056  // If no processes have nonlocal rows, then we don't have to do
4057  // anything. Checking this is probably cheaper than constructing
4058  // the Map of nonlocal rows (see below) and noticing that it has
4059  // zero global entries.
4060  {
4061  const int iHaveNonlocalRows = (myNumNonlocalRows == 0) ? 0 : 1;
4062  int someoneHasNonlocalRows = 0;
4063  reduceAll<int, int> (*comm, REDUCE_MAX, iHaveNonlocalRows,
4064  outArg (someoneHasNonlocalRows));
4065  if (someoneHasNonlocalRows == 0) {
4066  return; // no process has nonlocal rows, so nothing to do
4067  }
4068  }
4069 
4070  // 1. Create a list of the "nonlocal" rows on each process. this
4071  // requires iterating over nonlocals_, so while we do this,
4072  // deduplicate the entries and get a count for each nonlocal
4073  // row on this process.
4074  // 2. Construct a new row Map corresponding to those rows. This
4075  // Map is likely overlapping. We know that the Map is not
4076  // empty on all processes, because the above all-reduce and
4077  // return exclude that case.
4078 
4079  RCP<const map_type> nonlocalRowMap;
4080  Teuchos::Array<size_t> numEntPerNonlocalRow (myNumNonlocalRows);
4081  {
4082  Teuchos::Array<GO> myNonlocalGblRows (myNumNonlocalRows);
4083  size_type curPos = 0;
4084  for (auto mapIter = nonlocals_.begin (); mapIter != nonlocals_.end ();
4085  ++mapIter, ++curPos) {
4086  myNonlocalGblRows[curPos] = mapIter->first;
4087  // Get the values and column indices by reference, since we
4088  // intend to change them in place (that's what "erase" does).
4089  Teuchos::Array<GO>& gblCols = (mapIter->second).first;
4090  Teuchos::Array<Scalar>& vals = (mapIter->second).second;
4091 
4092  // Sort both arrays jointly, using the column indices as keys,
4093  // then merge them jointly. "Merge" here adds values
4094  // corresponding to the same column indices. The first 2 args
4095  // of merge2 are output arguments that work just like the
4096  // return value of std::unique.
4097  sort2 (gblCols.begin (), gblCols.end (), vals.begin ());
4098  typename Teuchos::Array<GO>::iterator gblCols_newEnd;
4099  typename Teuchos::Array<Scalar>::iterator vals_newEnd;
4100  merge2 (gblCols_newEnd, vals_newEnd,
4101  gblCols.begin (), gblCols.end (),
4102  vals.begin (), vals.end ());
4103  gblCols.erase (gblCols_newEnd, gblCols.end ());
4104  vals.erase (vals_newEnd, vals.end ());
4105  numEntPerNonlocalRow[curPos] = gblCols.size ();
4106  }
4107 
4108  // Currently, Map requires that its indexBase be the global min
4109  // of all its global indices. Map won't compute this for us, so
4110  // we must do it. If our process has no nonlocal rows, set the
4111  // "min" to the max possible GO value. This ensures that if
4112  // some process has at least one nonlocal row, then it will pick
4113  // that up as the min. We know that at least one process has a
4114  // nonlocal row, since the all-reduce and return at the top of
4115  // this method excluded that case.
4116  GO myMinNonlocalGblRow = std::numeric_limits<GO>::max ();
4117  {
4118  auto iter = std::min_element (myNonlocalGblRows.begin (),
4119  myNonlocalGblRows.end ());
4120  if (iter != myNonlocalGblRows.end ()) {
4121  myMinNonlocalGblRow = *iter;
4122  }
4123  }
4124  GO gblMinNonlocalGblRow = 0;
4125  reduceAll<int, GO> (*comm, REDUCE_MIN, myMinNonlocalGblRow,
4126  outArg (gblMinNonlocalGblRow));
4127  const GO indexBase = gblMinNonlocalGblRow;
4128  const global_size_t INV = Teuchos::OrdinalTraits<global_size_t>::invalid ();
4129  nonlocalRowMap = rcp (new map_type (INV, myNonlocalGblRows (), indexBase, comm));
4130  }
4131 
4132  // 3. Use the values and column indices for each nonlocal row, as
4133  // stored in nonlocals_, to construct a CrsMatrix corresponding
4134  // to nonlocal rows. We have
4135  // exact counts of the number of entries in each nonlocal row.
4136 
4137  if (verbose) {
4138  std::ostringstream os;
4139  os << *prefix << "Create nonlocal matrix" << endl;
4140  std::cerr << os.str();
4141  }
4142  RCP<crs_matrix_type> nonlocalMatrix =
4143  rcp (new crs_matrix_type (nonlocalRowMap, numEntPerNonlocalRow ()));
4144  {
4145  size_type curPos = 0;
4146  for (auto mapIter = nonlocals_.begin (); mapIter != nonlocals_.end ();
4147  ++mapIter, ++curPos) {
4148  const GO gblRow = mapIter->first;
4149  // Get values & column indices by ref, just to avoid copy.
4150  Teuchos::Array<GO>& gblCols = (mapIter->second).first;
4151  Teuchos::Array<Scalar>& vals = (mapIter->second).second;
4152  //const LO numEnt = static_cast<LO> (numEntPerNonlocalRow[curPos]);
4153  nonlocalMatrix->insertGlobalValues (gblRow, gblCols (), vals ());
4154  }
4155  }
4156  // There's no need to fill-complete the nonlocals matrix.
4157  // We just use it as a temporary container for the Export.
4158 
4159  // 4. If the original row Map is one to one, then we can Export
4160  // directly from nonlocalMatrix into this. Otherwise, we have
4161  // to create a temporary matrix with a one-to-one row Map,
4162  // Export into that, then Import from the temporary matrix into
4163  // *this.
4164 
4165  auto origRowMap = this->getRowMap ();
4166  const bool origRowMapIsOneToOne = origRowMap->isOneToOne ();
4167 
4168  int isLocallyComplete = 1; // true by default
4169 
4170  if (origRowMapIsOneToOne) {
4171  if (verbose) {
4172  std::ostringstream os;
4173  os << *prefix << "Original row Map is 1-to-1" << endl;
4174  std::cerr << os.str();
4175  }
4176  export_type exportToOrig (nonlocalRowMap, origRowMap);
4177  if (! exportToOrig.isLocallyComplete ()) {
4178  isLocallyComplete = 0;
4179  }
4180  if (verbose) {
4181  std::ostringstream os;
4182  os << *prefix << "doExport from nonlocalMatrix" << endl;
4183  std::cerr << os.str();
4184  }
4185  this->doExport (*nonlocalMatrix, exportToOrig, Tpetra::ADD);
4186  // We're done at this point!
4187  }
4188  else {
4189  if (verbose) {
4190  std::ostringstream os;
4191  os << *prefix << "Original row Map is NOT 1-to-1" << endl;
4192  std::cerr << os.str();
4193  }
4194  // If you ask a Map whether it is one to one, it does some
4195  // communication and stashes intermediate results for later use
4196  // by createOneToOne. Thus, calling createOneToOne doesn't cost
4197  // much more then the original cost of calling isOneToOne.
4198  auto oneToOneRowMap = Tpetra::createOneToOne (origRowMap);
4199  export_type exportToOneToOne (nonlocalRowMap, oneToOneRowMap);
4200  if (! exportToOneToOne.isLocallyComplete ()) {
4201  isLocallyComplete = 0;
4202  }
4203 
4204  // Create a temporary matrix with the one-to-one row Map.
4205  //
4206  // TODO (mfh 09 Sep 2016, 12 Sep 2016) Estimate # entries in
4207  // each row, to avoid reallocation during the Export operation.
4208  if (verbose) {
4209  std::ostringstream os;
4210  os << *prefix << "Create & doExport into 1-to-1 matrix"
4211  << endl;
4212  std::cerr << os.str();
4213  }
4214  crs_matrix_type oneToOneMatrix (oneToOneRowMap, 0);
4215  // Export from matrix of nonlocals into the temp one-to-one matrix.
4216  oneToOneMatrix.doExport(*nonlocalMatrix, exportToOneToOne,
4217  Tpetra::ADD);
4218 
4219  // We don't need the matrix of nonlocals anymore, so get rid of
4220  // it, to keep the memory high-water mark down.
4221  if (verbose) {
4222  std::ostringstream os;
4223  os << *prefix << "Free nonlocalMatrix" << endl;
4224  std::cerr << os.str();
4225  }
4226  nonlocalMatrix = Teuchos::null;
4227 
4228  // Import from the one-to-one matrix to the original matrix.
4229  if (verbose) {
4230  std::ostringstream os;
4231  os << *prefix << "doImport from 1-to-1 matrix" << endl;
4232  std::cerr << os.str();
4233  }
4234  import_type importToOrig (oneToOneRowMap, origRowMap);
4235  this->doImport (oneToOneMatrix, importToOrig, Tpetra::ADD);
4236  }
4237 
4238  // It's safe now to clear out nonlocals_, since we've already
4239  // committed side effects to *this. The standard idiom for
4240  // clearing a Container like std::map, is to swap it with an empty
4241  // Container and let the swapped Container fall out of scope.
4242  if (verbose) {
4243  std::ostringstream os;
4244  os << *prefix << "Free nonlocals_ (std::map)" << endl;
4245  std::cerr << os.str();
4246  }
4247  decltype (nonlocals_) newNonlocals;
4248  std::swap (nonlocals_, newNonlocals);
4249 
4250  // FIXME (mfh 12 Sep 2016) I don't like this all-reduce, and I
4251  // don't like throwing an exception here. A local return value
4252  // would likely be more useful to users. However, if users find
4253  // themselves exercising nonlocal inserts often, then they are
4254  // probably novice users who need the help. See Gibhub Issues
4255  // #603 and #601 (esp. the latter) for discussion.
4256 
4257  int isGloballyComplete = 0; // output argument of reduceAll
4258  reduceAll<int, int> (*comm, REDUCE_MIN, isLocallyComplete,
4259  outArg (isGloballyComplete));
4260  TEUCHOS_TEST_FOR_EXCEPTION
4261  (isGloballyComplete != 1, std::runtime_error, "On at least one process, "
4262  "you called insertGlobalValues with a global row index which is not in "
4263  "the matrix's row Map on any process in its communicator.");
4264  }
4265 
4266  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
4267  void
4268  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
4269  resumeFill (const Teuchos::RCP<Teuchos::ParameterList>& params)
4270  {
4271  if (! isStaticGraph ()) { // Don't resume fill of a nonowned graph.
4272  myGraph_->resumeFill (params);
4273  }
4274 #if KOKKOSKERNELS_VERSION >= 40299
4275  // Delete the apply helper (if it exists)
4276  applyHelper.reset();
4277 #endif
4278  fillComplete_ = false;
4279  }
4280 
4281  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
4282  bool
4283  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
4284  haveGlobalConstants() const {
4285  return getCrsGraphRef ().haveGlobalConstants ();
4286  }
4287 
4288  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
4289  void
4290  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
4291  fillComplete (const Teuchos::RCP<Teuchos::ParameterList>& params)
4292  {
4293  const char tfecfFuncName[] = "fillComplete(params): ";
4294 
4295  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4296  (this->getCrsGraph ().is_null (), std::logic_error,
4297  "getCrsGraph() returns null. This should not happen at this point. "
4298  "Please report this bug to the Tpetra developers.");
4299 
4300  const crs_graph_type& graph = this->getCrsGraphRef ();
4301  if (this->isStaticGraph () && graph.isFillComplete ()) {
4302  // If this matrix's graph is fill complete and the user did not
4303  // supply a domain or range Map, use the graph's domain and
4304  // range Maps.
4305  this->fillComplete (graph.getDomainMap (), graph.getRangeMap (), params);
4306  }
4307  else { // assume that user's row Map is the domain and range Map
4308  Teuchos::RCP<const map_type> rangeMap = graph.getRowMap ();
4309  Teuchos::RCP<const map_type> domainMap = rangeMap;
4310  this->fillComplete (domainMap, rangeMap, params);
4311  }
4312  }
4313 
4314  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
4315  void
4316  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
4317  fillComplete (const Teuchos::RCP<const map_type>& domainMap,
4318  const Teuchos::RCP<const map_type>& rangeMap,
4319  const Teuchos::RCP<Teuchos::ParameterList>& params)
4320  {
4321  using Details::Behavior;
4322  using Details::ProfilingRegion;
4323  using Teuchos::ArrayRCP;
4324  using Teuchos::RCP;
4325  using Teuchos::rcp;
4326  using std::endl;
4327  const char tfecfFuncName[] = "fillComplete: ";
4328  ProfilingRegion regionFillComplete
4329  ("Tpetra::CrsMatrix::fillComplete");
4330  const bool verbose = Behavior::verbose("CrsMatrix");
4331  std::unique_ptr<std::string> prefix;
4332  if (verbose) {
4333  prefix = this->createPrefix("CrsMatrix", "fillComplete(dom,ran,p)");
4334  std::ostringstream os;
4335  os << *prefix << endl;
4336  std::cerr << os.str ();
4337  }
4338  Details::ProfilingRegion region(
4339  "Tpetra::CrsMatrix::fillCompete",
4340  "fillCompete");
4341 
4342  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4343  (! this->isFillActive () || this->isFillComplete (), std::runtime_error,
4344  "Matrix fill state must be active (isFillActive() "
4345  "must be true) before you may call fillComplete().");
4346  const int numProcs = this->getComm ()->getSize ();
4347 
4348  //
4349  // Read parameters from the input ParameterList.
4350  //
4351  {
4352  Details::ProfilingRegion region_fc("Tpetra::CrsMatrix::fillCompete", "ParameterList");
4353 
4354  // If true, the caller promises that no process did nonlocal
4355  // changes since the last call to fillComplete.
4356  bool assertNoNonlocalInserts = false;
4357  // If true, makeColMap sorts remote GIDs (within each remote
4358  // process' group).
4359  bool sortGhosts = true;
4360 
4361  if (! params.is_null ()) {
4362  assertNoNonlocalInserts = params->get ("No Nonlocal Changes",
4363  assertNoNonlocalInserts);
4364  if (params->isParameter ("sort column map ghost gids")) {
4365  sortGhosts = params->get ("sort column map ghost gids", sortGhosts);
4366  }
4367  else if (params->isParameter ("Sort column Map ghost GIDs")) {
4368  sortGhosts = params->get ("Sort column Map ghost GIDs", sortGhosts);
4369  }
4370  }
4371  // We also don't need to do global assembly if there is only one
4372  // process in the communicator.
4373  const bool needGlobalAssemble = ! assertNoNonlocalInserts && numProcs > 1;
4374  // This parameter only matters if this matrix owns its graph.
4375  if (! this->myGraph_.is_null ()) {
4376  this->myGraph_->sortGhostsAssociatedWithEachProcessor_ = sortGhosts;
4377  }
4378 
4379  if (! this->getCrsGraphRef ().indicesAreAllocated ()) {
4380  if (this->hasColMap ()) { // use local indices
4381  allocateValues(LocalIndices, GraphNotYetAllocated, verbose);
4382  }
4383  else { // no column Map, so use global indices
4384  allocateValues(GlobalIndices, GraphNotYetAllocated, verbose);
4385  }
4386  }
4387  // Global assemble, if we need to. This call only costs a single
4388  // all-reduce if we didn't need global assembly after all.
4389  if (needGlobalAssemble) {
4390  this->globalAssemble ();
4391  }
4392  else {
4393  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4394  (numProcs == 1 && nonlocals_.size() > 0,
4395  std::runtime_error, "Cannot have nonlocal entries on a serial run. "
4396  "An invalid entry (i.e., with row index not in the row Map) must have "
4397  "been submitted to the CrsMatrix.");
4398  }
4399  }
4400  if (this->isStaticGraph ()) {
4401  Details::ProfilingRegion region_isg("Tpetra::CrsMatrix::fillCompete", "isStaticGraph");
4402  // FIXME (mfh 14 Nov 2016) In order to fix #843, I enable the
4403  // checks below only in debug mode. It would be nicer to do a
4404  // local check, then propagate the error state in a deferred
4405  // way, whenever communication happens. That would reduce the
4406  // cost of checking, to the point where it may make sense to
4407  // enable it even in release mode.
4408 #ifdef HAVE_TPETRA_DEBUG
4409  // FIXME (mfh 18 Jun 2014) This check for correctness of the
4410  // input Maps incurs a penalty of two all-reduces for the
4411  // otherwise optimal const graph case.
4412  //
4413  // We could turn these (max) 2 all-reduces into (max) 1, by
4414  // fusing them. We could do this by adding a "locallySameAs"
4415  // method to Map, which would return one of four states:
4416  //
4417  // a. Certainly globally the same
4418  // b. Certainly globally not the same
4419  // c. Locally the same
4420  // d. Locally not the same
4421  //
4422  // The first two states don't require further communication.
4423  // The latter two states require an all-reduce to communicate
4424  // globally, but we only need one all-reduce, since we only need
4425  // to check whether at least one of the Maps is wrong.
4426  const bool domainMapsMatch =
4427  this->staticGraph_->getDomainMap ()->isSameAs (*domainMap);
4428  const bool rangeMapsMatch =
4429  this->staticGraph_->getRangeMap ()->isSameAs (*rangeMap);
4430 
4431  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4432  (! domainMapsMatch, std::runtime_error,
4433  "The CrsMatrix's domain Map does not match the graph's domain Map. "
4434  "The graph cannot be changed because it was given to the CrsMatrix "
4435  "constructor as const. You can fix this by passing in the graph's "
4436  "domain Map and range Map to the matrix's fillComplete call.");
4437 
4438  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4439  (! rangeMapsMatch, std::runtime_error,
4440  "The CrsMatrix's range Map does not match the graph's range Map. "
4441  "The graph cannot be changed because it was given to the CrsMatrix "
4442  "constructor as const. You can fix this by passing in the graph's "
4443  "domain Map and range Map to the matrix's fillComplete call.");
4444 #endif // HAVE_TPETRA_DEBUG
4445 
4446  // The matrix does _not_ own the graph, and the graph's
4447  // structure is already fixed, so just fill the local matrix.
4448  this->fillLocalMatrix (params);
4449  }
4450  else {
4451  Details::ProfilingRegion region_insg("Tpetra::CrsMatrix::fillCompete", "isNotStaticGraph");
4452  // Set the graph's domain and range Maps. This will clear the
4453  // Import if the domain Map has changed (is a different
4454  // pointer), and the Export if the range Map has changed (is a
4455  // different pointer).
4456  this->myGraph_->setDomainRangeMaps (domainMap, rangeMap);
4457 
4458  // Make the graph's column Map, if necessary.
4459  Teuchos::Array<int> remotePIDs (0);
4460  const bool mustBuildColMap = ! this->hasColMap ();
4461  if (mustBuildColMap) {
4462  this->myGraph_->makeColMap (remotePIDs);
4463  }
4464 
4465  // Make indices local, if necessary. The method won't do
4466  // anything if the graph is already locally indexed.
4467  const std::pair<size_t, std::string> makeIndicesLocalResult =
4468  this->myGraph_->makeIndicesLocal(verbose);
4469  // TODO (mfh 20 Jul 2017) Instead of throwing here, pass along
4470  // the error state to makeImportExport
4471  // which may do all-reduces and thus may
4472  // have the opportunity to communicate that error state.
4473  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4474  (makeIndicesLocalResult.first != 0, std::runtime_error,
4475  makeIndicesLocalResult.second);
4476 
4477  const bool sorted = this->myGraph_->isSorted ();
4478  const bool merged = this->myGraph_->isMerged ();
4479  this->sortAndMergeIndicesAndValues (sorted, merged);
4480 
4481  // Make Import and Export objects, if they haven't been made
4482  // already. If we made a column Map above, reuse information
4483  // from that process to avoid communiation in the Import setup.
4484  this->myGraph_->makeImportExport (remotePIDs, mustBuildColMap);
4485 
4486  // The matrix _does_ own the graph, so fill the local graph at
4487  // the same time as the local matrix.
4488  this->fillLocalGraphAndMatrix (params);
4489 
4490  const bool callGraphComputeGlobalConstants = params.get () == nullptr ||
4491  params->get ("compute global constants", true);
4492  if (callGraphComputeGlobalConstants) {
4493  this->myGraph_->computeGlobalConstants ();
4494  }
4495  else {
4496  this->myGraph_->computeLocalConstants ();
4497  }
4498  this->myGraph_->fillComplete_ = true;
4499  this->myGraph_->checkInternalState ();
4500  }
4501 
4502  // FIXME (mfh 28 Aug 2014) "Preserve Local Graph" bool parameter no longer used.
4503 
4504  this->fillComplete_ = true; // Now we're fill complete!
4505  {
4506  Details::ProfilingRegion region_cis(
4507  "Tpetra::CrsMatrix::fillCompete", "checkInternalState"
4508  );
4509  this->checkInternalState ();
4510  }
4511  } //fillComplete(domainMap, rangeMap, params)
4512 
4513  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
4514  void
4515  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
4516  expertStaticFillComplete (const Teuchos::RCP<const map_type> & domainMap,
4517  const Teuchos::RCP<const map_type> & rangeMap,
4518  const Teuchos::RCP<const import_type>& importer,
4519  const Teuchos::RCP<const export_type>& exporter,
4520  const Teuchos::RCP<Teuchos::ParameterList> &params)
4521  {
4522 #ifdef HAVE_TPETRA_MMM_TIMINGS
4523  std::string label;
4524  if(!params.is_null())
4525  label = params->get("Timer Label",label);
4526  std::string prefix = std::string("Tpetra ")+ label + std::string(": ");
4527  using Teuchos::TimeMonitor;
4528 
4529  Teuchos::TimeMonitor all(*TimeMonitor::getNewTimer(prefix + std::string("ESFC-all")));
4530 #endif
4531 
4532  const char tfecfFuncName[] = "expertStaticFillComplete: ";
4533  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC( ! isFillActive() || isFillComplete(),
4534  std::runtime_error, "Matrix fill state must be active (isFillActive() "
4535  "must be true) before calling fillComplete().");
4536  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
4537  myGraph_.is_null (), std::logic_error, "myGraph_ is null. This is not allowed.");
4538 
4539  {
4540 #ifdef HAVE_TPETRA_MMM_TIMINGS
4541  Teuchos::TimeMonitor graph(*TimeMonitor::getNewTimer(prefix + std::string("eSFC-M-Graph")));
4542 #endif
4543  // We will presume globalAssemble is not needed, so we do the ESFC on the graph
4544  myGraph_->expertStaticFillComplete (domainMap, rangeMap, importer, exporter,params);
4545  }
4546 
4547  {
4548 #ifdef HAVE_TPETRA_MMM_TIMINGS
4549  TimeMonitor fLGAM(*TimeMonitor::getNewTimer(prefix + std::string("eSFC-M-fLGAM")));
4550 #endif
4551  // Fill the local graph and matrix
4552  fillLocalGraphAndMatrix (params);
4553  }
4554  // FIXME (mfh 28 Aug 2014) "Preserve Local Graph" bool parameter no longer used.
4555 
4556  // Now we're fill complete!
4557  fillComplete_ = true;
4558 
4559  // Sanity checks at the end.
4560 #ifdef HAVE_TPETRA_DEBUG
4561  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(isFillActive(), std::logic_error,
4562  ": We're at the end of fillComplete(), but isFillActive() is true. "
4563  "Please report this bug to the Tpetra developers.");
4564  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(! isFillComplete(), std::logic_error,
4565  ": We're at the end of fillComplete(), but isFillActive() is true. "
4566  "Please report this bug to the Tpetra developers.");
4567 #endif // HAVE_TPETRA_DEBUG
4568  {
4569 #ifdef HAVE_TPETRA_MMM_TIMINGS
4570  Teuchos::TimeMonitor cIS(*TimeMonitor::getNewTimer(prefix + std::string("ESFC-M-cIS")));
4571 #endif
4572 
4573  checkInternalState();
4574  }
4575  }
4576 
4577  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
4578  size_t CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
4579  mergeRowIndicesAndValues (size_t rowLen, LocalOrdinal* cols, impl_scalar_type* vals)
4580  {
4581  impl_scalar_type* rowValueIter = vals;
4582  // beg,end define a half-exclusive interval over which to iterate.
4583  LocalOrdinal* beg = cols;
4584  LocalOrdinal* end = cols + rowLen;
4585  LocalOrdinal* newend = beg;
4586  if (beg != end) {
4587  LocalOrdinal* cur = beg + 1;
4588  impl_scalar_type* vcur = rowValueIter + 1;
4589  impl_scalar_type* vend = rowValueIter;
4590  cur = beg+1;
4591  while (cur != end) {
4592  if (*cur != *newend) {
4593  // new entry; save it
4594  ++newend;
4595  ++vend;
4596  (*newend) = (*cur);
4597  (*vend) = (*vcur);
4598  }
4599  else {
4600  // old entry; merge it
4601  //(*vend) = f (*vend, *vcur);
4602  (*vend) += *vcur;
4603  }
4604  ++cur;
4605  ++vcur;
4606  }
4607  ++newend; // one past the last entry, per typical [beg,end) semantics
4608  }
4609  return newend - beg;
4610  }
4611 
4612  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
4613  void
4614  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
4615  sortAndMergeIndicesAndValues (const bool sorted, const bool merged)
4616  {
4617  using ::Tpetra::Details::ProfilingRegion;
4618  typedef LocalOrdinal LO;
4619  typedef typename Kokkos::View<LO*, device_type>::HostMirror::execution_space
4620  host_execution_space;
4621  typedef Kokkos::RangePolicy<host_execution_space, LO> range_type;
4622  const char tfecfFuncName[] = "sortAndMergeIndicesAndValues: ";
4623  ProfilingRegion regionSAM ("Tpetra::CrsMatrix::sortAndMergeIndicesAndValues");
4624 
4625  if (! sorted || ! merged) {
4626  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4627  (this->isStaticGraph (), std::runtime_error, "Cannot sort or merge with "
4628  "\"static\" (const) graph, since the matrix does not own the graph.");
4629  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4630  (this->myGraph_.is_null (), std::logic_error, "myGraph_ is null, but "
4631  "this matrix claims ! isStaticGraph(). "
4632  "Please report this bug to the Tpetra developers.");
4633  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
4634  (this->isStorageOptimized (), std::logic_error, "It is invalid to call "
4635  "this method if the graph's storage has already been optimized. "
4636  "Please report this bug to the Tpetra developers.");
4637 
4638  crs_graph_type& graph = * (this->myGraph_);
4639  const LO lclNumRows = static_cast<LO> (this->getLocalNumRows ());
4640  size_t totalNumDups = 0;
4641  {
4642  //Accessing host unpacked (4-array CRS) local matrix.
4643  auto rowBegins_ = graph.getRowPtrsUnpackedHost();
4644  auto rowLengths_ = graph.k_numRowEntries_;
4645  auto vals_ = this->valuesUnpacked_wdv.getHostView(Access::ReadWrite);
4646  auto cols_ = graph.lclIndsUnpacked_wdv.getHostView(Access::ReadWrite);
4647  Kokkos::parallel_reduce ("sortAndMergeIndicesAndValues", range_type (0, lclNumRows),
4648  [=] (const LO lclRow, size_t& numDups) {
4649  size_t rowBegin = rowBegins_(lclRow);
4650  size_t rowLen = rowLengths_(lclRow);
4651  LO* cols = cols_.data() + rowBegin;
4652  impl_scalar_type* vals = vals_.data() + rowBegin;
4653  if (! sorted) {
4654  sort2 (cols, cols + rowLen, vals);
4655  }
4656  if (! merged) {
4657  size_t newRowLength = mergeRowIndicesAndValues (rowLen, cols, vals);
4658  rowLengths_(lclRow) = newRowLength;
4659  numDups += rowLen - newRowLength;
4660  }
4661  }, totalNumDups);
4662  }
4663  if (! sorted) {
4664  graph.indicesAreSorted_ = true; // we just sorted every row
4665  }
4666  if (! merged) {
4667  graph.noRedundancies_ = true; // we just merged every row
4668  }
4669  }
4670  }
4671 
4672  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
4673  void
4674  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
4675  applyNonTranspose (const MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> & X_in,
4676  MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> & Y_in,
4677  Scalar alpha,
4678  Scalar beta) const
4679  {
4680  using Tpetra::Details::ProfilingRegion;
4681  using Teuchos::RCP;
4682  using Teuchos::rcp;
4683  using Teuchos::rcp_const_cast;
4684  using Teuchos::rcpFromRef;
4685  const Scalar ZERO = Teuchos::ScalarTraits<Scalar>::zero ();
4686  const Scalar ONE = Teuchos::ScalarTraits<Scalar>::one ();
4687 
4688  // mfh 05 Jun 2014: Special case for alpha == 0. I added this to
4689  // fix an Ifpack2 test (RILUKSingleProcessUnitTests), which was
4690  // failing only for the Kokkos refactor version of Tpetra. It's a
4691  // good idea regardless to have the bypass.
4692  if (alpha == ZERO) {
4693  if (beta == ZERO) {
4694  Y_in.putScalar (ZERO);
4695  } else if (beta != ONE) {
4696  Y_in.scale (beta);
4697  }
4698  return;
4699  }
4700 
4701  // It's possible that X is a view of Y or vice versa. We don't
4702  // allow this (apply() requires that X and Y not alias one
4703  // another), but it's helpful to detect and work around this case.
4704  // We don't try to to detect the more subtle cases (e.g., one is a
4705  // subview of the other, but their initial pointers differ). We
4706  // only need to do this if this matrix's Import is trivial;
4707  // otherwise, we don't actually apply the operator from X into Y.
4708 
4709  RCP<const import_type> importer = this->getGraph ()->getImporter ();
4710  RCP<const export_type> exporter = this->getGraph ()->getExporter ();
4711 
4712  // If beta == 0, then the output MV will be overwritten; none of
4713  // its entries should be read. (Sparse BLAS semantics say that we
4714  // must ignore any Inf or NaN entries in Y_in, if beta is zero.)
4715  // This matters if we need to do an Export operation; see below.
4716  const bool Y_is_overwritten = (beta == ZERO);
4717 
4718  // We treat the case of a replicated MV output specially.
4719  const bool Y_is_replicated =
4720  (! Y_in.isDistributed () && this->getComm ()->getSize () != 1);
4721 
4722  // This is part of the special case for replicated MV output.
4723  // We'll let each process do its thing, but do an all-reduce at
4724  // the end to sum up the results. Setting beta=0 on all processes
4725  // but Proc 0 makes the math work out for the all-reduce. (This
4726  // assumes that the replicated data is correctly replicated, so
4727  // that the data are the same on all processes.)
4728  if (Y_is_replicated && this->getComm ()->getRank () > 0) {
4729  beta = ZERO;
4730  }
4731 
4732  // Temporary MV for Import operation. After the block of code
4733  // below, this will be an (Imported if necessary) column Map MV
4734  // ready to give to localApply(...).
4735  RCP<const MV> X_colMap;
4736  if (importer.is_null ()) {
4737  if (! X_in.isConstantStride ()) {
4738  // Not all sparse mat-vec kernels can handle an input MV with
4739  // nonconstant stride correctly, so we have to copy it in that
4740  // case into a constant stride MV. To make a constant stride
4741  // copy of X_in, we force creation of the column (== domain)
4742  // Map MV (if it hasn't already been created, else fetch the
4743  // cached copy). This avoids creating a new MV each time.
4744  RCP<MV> X_colMapNonConst = getColumnMapMultiVector (X_in, true);
4745  Tpetra::deep_copy (*X_colMapNonConst, X_in);
4746  X_colMap = rcp_const_cast<const MV> (X_colMapNonConst);
4747  }
4748  else {
4749  // The domain and column Maps are the same, so do the local
4750  // multiply using the domain Map input MV X_in.
4751  X_colMap = rcpFromRef (X_in);
4752  }
4753  }
4754  else { // need to Import source (multi)vector
4755  ProfilingRegion regionImport ("Tpetra::CrsMatrix::apply: Import");
4756 
4757  // We're doing an Import anyway, which will copy the relevant
4758  // elements of the domain Map MV X_in into a separate column Map
4759  // MV. Thus, we don't have to worry whether X_in is constant
4760  // stride.
4761  RCP<MV> X_colMapNonConst = getColumnMapMultiVector (X_in);
4762 
4763  // Import from the domain Map MV to the column Map MV.
4764  X_colMapNonConst->doImport (X_in, *importer, INSERT);
4765  X_colMap = rcp_const_cast<const MV> (X_colMapNonConst);
4766  }
4767 
4768  // Temporary MV for doExport (if needed), or for copying a
4769  // nonconstant stride output MV into a constant stride MV. This
4770  // is null if we don't need the temporary MV, that is, if the
4771  // Export is trivial (null).
4772  RCP<MV> Y_rowMap = getRowMapMultiVector (Y_in);
4773 
4774  // If we have a nontrivial Export object, we must perform an
4775  // Export. In that case, the local multiply result will go into
4776  // the row Map multivector. We don't have to make a
4777  // constant-stride version of Y_in in this case, because we had to
4778  // make a constant stride Y_rowMap MV and do an Export anyway.
4779  if (! exporter.is_null ()) {
4780  this->localApply (*X_colMap, *Y_rowMap, Teuchos::NO_TRANS, alpha, ZERO);
4781  {
4782  ProfilingRegion regionExport ("Tpetra::CrsMatrix::apply: Export");
4783 
4784  // If we're overwriting the output MV Y_in completely (beta ==
4785  // 0), then make sure that it is filled with zeros before we
4786  // do the Export. Otherwise, the ADD combine mode will use
4787  // data in Y_in, which is supposed to be zero.
4788  if (Y_is_overwritten) {
4789  Y_in.putScalar (ZERO);
4790  }
4791  else {
4792  // Scale output MV by beta, so that doExport sums in the
4793  // mat-vec contribution: Y_in = beta*Y_in + alpha*A*X_in.
4794  Y_in.scale (beta);
4795  }
4796  // Do the Export operation.
4797  Y_in.doExport (*Y_rowMap, *exporter, ADD_ASSIGN);
4798  }
4799  }
4800  else { // Don't do an Export: row Map and range Map are the same.
4801  //
4802  // If Y_in does not have constant stride, or if the column Map
4803  // MV aliases Y_in, then we can't let the kernel write directly
4804  // to Y_in. Instead, we have to use the cached row (== range)
4805  // Map MV as temporary storage.
4806  //
4807  // FIXME (mfh 05 Jun 2014) This test for aliasing only tests if
4808  // the user passed in the same MultiVector for both X and Y. It
4809  // won't detect whether one MultiVector views the other. We
4810  // should also check the MultiVectors' raw data pointers.
4811  if (! Y_in.isConstantStride () || X_colMap.getRawPtr () == &Y_in) {
4812  // Force creating the MV if it hasn't been created already.
4813  // This will reuse a previously created cached MV.
4814  Y_rowMap = getRowMapMultiVector (Y_in, true);
4815 
4816  // If beta == 0, we don't need to copy Y_in into Y_rowMap,
4817  // since we're overwriting it anyway.
4818  if (beta != ZERO) {
4819  Tpetra::deep_copy (*Y_rowMap, Y_in);
4820  }
4821  this->localApply (*X_colMap, *Y_rowMap, Teuchos::NO_TRANS, alpha, beta);
4822  Tpetra::deep_copy (Y_in, *Y_rowMap);
4823  }
4824  else {
4825  this->localApply (*X_colMap, Y_in, Teuchos::NO_TRANS, alpha, beta);
4826  }
4827  }
4828 
4829  // If the range Map is a locally replicated Map, sum up
4830  // contributions from each process. We set beta = 0 on all
4831  // processes but Proc 0 initially, so this will handle the scaling
4832  // factor beta correctly.
4833  if (Y_is_replicated) {
4834  ProfilingRegion regionReduce ("Tpetra::CrsMatrix::apply: Reduce Y");
4835  Y_in.reduce ();
4836  }
4837  }
4838 
4839  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
4840  void
4841  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
4842  applyTranspose (const MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node>& X_in,
4843  MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node>& Y_in,
4844  const Teuchos::ETransp mode,
4845  Scalar alpha,
4846  Scalar beta) const
4847  {
4848  using Tpetra::Details::ProfilingRegion;
4849  using Teuchos::null;
4850  using Teuchos::RCP;
4851  using Teuchos::rcp;
4852  using Teuchos::rcp_const_cast;
4853  using Teuchos::rcpFromRef;
4854  const Scalar ZERO = Teuchos::ScalarTraits<Scalar>::zero ();
4855 
4856  // Take shortcuts for alpha == 0.
4857  if (alpha == ZERO) {
4858  // Follow the Sparse BLAS convention by ignoring both the matrix
4859  // and X_in, in this case.
4860  if (beta == ZERO) {
4861  // Follow the Sparse BLAS convention by overwriting any Inf or
4862  // NaN values in Y_in, in this case.
4863  Y_in.putScalar (ZERO);
4864  }
4865  else {
4866  Y_in.scale (beta);
4867  }
4868  return;
4869  }
4870  else if (beta == ZERO) {
4871  //Thyra was implicitly assuming that Y gets set to zero / or is overwritten
4872  //when bets==0. This was not the case with transpose in a multithreaded
4873  //environment where a multiplication with subsequent atomic_adds is used
4874  //since 0 is effectively not special cased. Doing the explicit set to zero here
4875  //This catches cases where Y is nan or inf.
4876  Y_in.putScalar (ZERO);
4877  }
4878 
4879  const size_t numVectors = X_in.getNumVectors ();
4880 
4881  // We don't allow X_in and Y_in to alias one another. It's hard
4882  // to check this, because advanced users could create views from
4883  // raw pointers. However, if X_in and Y_in reference the same
4884  // object, we will do the user a favor by copying X into new
4885  // storage (with a warning). We only need to do this if we have
4886  // trivial importers; otherwise, we don't actually apply the
4887  // operator from X into Y.
4888  RCP<const import_type> importer = this->getGraph ()->getImporter ();
4889  RCP<const export_type> exporter = this->getGraph ()->getExporter ();
4890  // access X indirectly, in case we need to create temporary storage
4891  RCP<const MV> X;
4892 
4893  // some parameters for below
4894  const bool Y_is_replicated = (! Y_in.isDistributed () && this->getComm ()->getSize () != 1);
4895  const bool Y_is_overwritten = (beta == ZERO);
4896  if (Y_is_replicated && this->getComm ()->getRank () > 0) {
4897  beta = ZERO;
4898  }
4899 
4900  // The kernels do not allow input or output with nonconstant stride.
4901  if (! X_in.isConstantStride () && importer.is_null ()) {
4902  X = rcp (new MV (X_in, Teuchos::Copy)); // Constant-stride copy of X_in
4903  } else {
4904  X = rcpFromRef (X_in); // Reference to X_in
4905  }
4906 
4907  // Set up temporary multivectors for Import and/or Export.
4908  if (importer != Teuchos::null) {
4909  if (importMV_ != Teuchos::null && importMV_->getNumVectors() != numVectors) {
4910  importMV_ = null;
4911  }
4912  if (importMV_ == null) {
4913  importMV_ = rcp (new MV (this->getColMap (), numVectors));
4914  }
4915  }
4916  if (exporter != Teuchos::null) {
4917  if (exportMV_ != Teuchos::null && exportMV_->getNumVectors() != numVectors) {
4918  exportMV_ = null;
4919  }
4920  if (exportMV_ == null) {
4921  exportMV_ = rcp (new MV (this->getRowMap (), numVectors));
4922  }
4923  }
4924 
4925  // If we have a non-trivial exporter, we must import elements that
4926  // are permuted or are on other processors.
4927  if (! exporter.is_null ()) {
4928  ProfilingRegion regionImport ("Tpetra::CrsMatrix::apply (transpose): Import");
4929  exportMV_->doImport (X_in, *exporter, INSERT);
4930  X = exportMV_; // multiply out of exportMV_
4931  }
4932 
4933  // If we have a non-trivial importer, we must export elements that
4934  // are permuted or belong to other processors. We will compute
4935  // solution into the to-be-exported MV; get a view.
4936  if (importer != Teuchos::null) {
4937  ProfilingRegion regionExport ("Tpetra::CrsMatrix::apply (transpose): Export");
4938 
4939  // FIXME (mfh 18 Apr 2015) Temporary fix suggested by Clark
4940  // Dohrmann on Fri 17 Apr 2015. At some point, we need to go
4941  // back and figure out why this helps. importMV_ SHOULD be
4942  // completely overwritten in the localApply(...) call
4943  // below, because beta == ZERO there.
4944  importMV_->putScalar (ZERO);
4945  // Do the local computation.
4946  this->localApply (*X, *importMV_, mode, alpha, ZERO);
4947 
4948  if (Y_is_overwritten) {
4949  Y_in.putScalar (ZERO);
4950  } else {
4951  Y_in.scale (beta);
4952  }
4953  Y_in.doExport (*importMV_, *importer, ADD_ASSIGN);
4954  }
4955  // otherwise, multiply into Y
4956  else {
4957  // can't multiply in-situ; can't multiply into non-strided multivector
4958  //
4959  // FIXME (mfh 05 Jun 2014) This test for aliasing only tests if
4960  // the user passed in the same MultiVector for both X and Y. It
4961  // won't detect whether one MultiVector views the other. We
4962  // should also check the MultiVectors' raw data pointers.
4963  if (! Y_in.isConstantStride () || X.getRawPtr () == &Y_in) {
4964  // Make a deep copy of Y_in, into which to write the multiply result.
4965  MV Y (Y_in, Teuchos::Copy);
4966  this->localApply (*X, Y, mode, alpha, beta);
4967  Tpetra::deep_copy (Y_in, Y);
4968  } else {
4969  this->localApply (*X, Y_in, mode, alpha, beta);
4970  }
4971  }
4972 
4973  // If the range Map is a locally replicated map, sum the
4974  // contributions from each process. (That's why we set beta=0
4975  // above for all processes but Proc 0.)
4976  if (Y_is_replicated) {
4977  ProfilingRegion regionReduce ("Tpetra::CrsMatrix::apply (transpose): Reduce Y");
4978  Y_in.reduce ();
4979  }
4980  }
4981 
4982  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
4983  void
4984  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
4985  localApply (const MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node>& X,
4986  MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node>& Y,
4987  const Teuchos::ETransp mode,
4988  const Scalar& alpha,
4989  const Scalar& beta) const
4990  {
4991  using Tpetra::Details::ProfilingRegion;
4992  using Teuchos::NO_TRANS;
4993  ProfilingRegion regionLocalApply ("Tpetra::CrsMatrix::localApply");
4994 
4995  auto X_lcl = X.getLocalViewDevice(Access::ReadOnly);
4996  auto Y_lcl = Y.getLocalViewDevice(Access::ReadWrite);
4997 
4998  const bool debug = ::Tpetra::Details::Behavior::debug ();
4999  if (debug) {
5000  const char tfecfFuncName[] = "localApply: ";
5001  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5002  (X.getNumVectors () != Y.getNumVectors (), std::runtime_error,
5003  "X.getNumVectors() = " << X.getNumVectors () << " != "
5004  "Y.getNumVectors() = " << Y.getNumVectors () << ".");
5005  const bool transpose = (mode != Teuchos::NO_TRANS);
5006  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5007  (! transpose && X.getLocalLength () !=
5008  getColMap ()->getLocalNumElements (), std::runtime_error,
5009  "NO_TRANS case: X has the wrong number of local rows. "
5010  "X.getLocalLength() = " << X.getLocalLength () << " != "
5011  "getColMap()->getLocalNumElements() = " <<
5012  getColMap ()->getLocalNumElements () << ".");
5013  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5014  (! transpose && Y.getLocalLength () !=
5015  getRowMap ()->getLocalNumElements (), std::runtime_error,
5016  "NO_TRANS case: Y has the wrong number of local rows. "
5017  "Y.getLocalLength() = " << Y.getLocalLength () << " != "
5018  "getRowMap()->getLocalNumElements() = " <<
5019  getRowMap ()->getLocalNumElements () << ".");
5020  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5021  (transpose && X.getLocalLength () !=
5022  getRowMap ()->getLocalNumElements (), std::runtime_error,
5023  "TRANS or CONJ_TRANS case: X has the wrong number of local "
5024  "rows. X.getLocalLength() = " << X.getLocalLength ()
5025  << " != getRowMap()->getLocalNumElements() = "
5026  << getRowMap ()->getLocalNumElements () << ".");
5027  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5028  (transpose && Y.getLocalLength () !=
5029  getColMap ()->getLocalNumElements (), std::runtime_error,
5030  "TRANS or CONJ_TRANS case: X has the wrong number of local "
5031  "rows. Y.getLocalLength() = " << Y.getLocalLength ()
5032  << " != getColMap()->getLocalNumElements() = "
5033  << getColMap ()->getLocalNumElements () << ".");
5034  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5035  (! isFillComplete (), std::runtime_error, "The matrix is not "
5036  "fill complete. You must call fillComplete() (possibly with "
5037  "domain and range Map arguments) without an intervening "
5038  "resumeFill() call before you may call this method.");
5039  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5040  (! X.isConstantStride () || ! Y.isConstantStride (),
5041  std::runtime_error, "X and Y must be constant stride.");
5042  // If the two pointers are null, then they don't alias one
5043  // another, even though they are equal.
5044  // Kokkos does not guarantee that zero row-extent vectors
5045  // point to different places, so we have to check that too.
5046  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5047  (X_lcl.data () == Y_lcl.data () && X_lcl.data () != nullptr
5048  && X_lcl.extent(0) != 0,
5049  std::runtime_error, "X and Y may not alias one another.");
5050  }
5051 
5052 #if KOKKOSKERNELS_VERSION >= 40299
5053  auto A_lcl = getLocalMatrixDevice();
5054 
5055  if(!applyHelper.get()) {
5056  // The apply helper does not exist, so create it.
5057  // Decide now whether to use the imbalanced row path, or the default.
5058  bool useMergePath = false;
5059 #ifdef KOKKOSKERNELS_ENABLE_TPL_CUSPARSE
5060  //TODO: when https://github.com/kokkos/kokkos-kernels/issues/2166 is fixed and,
5061  //we can use SPMV_MERGE_PATH for the native spmv as well.
5062  //Take out this ifdef to enable that.
5063  //
5064  //Until then, only use SPMV_MERGE_PATH when calling cuSPARSE.
5065  if constexpr(std::is_same_v<execution_space, Kokkos::Cuda>) {
5066  LocalOrdinal nrows = getLocalNumRows();
5067  LocalOrdinal maxRowImbalance = 0;
5068  if(nrows != 0)
5069  maxRowImbalance = getLocalMaxNumRowEntries() - (getLocalNumEntries() / nrows);
5070 
5071  if(size_t(maxRowImbalance) >= Tpetra::Details::Behavior::rowImbalanceThreshold())
5072  useMergePath = true;
5073  }
5074 #endif
5075  applyHelper = std::make_shared<ApplyHelper>(A_lcl.nnz(), A_lcl.graph.row_map,
5076  useMergePath ? KokkosSparse::SPMV_MERGE_PATH : KokkosSparse::SPMV_DEFAULT);
5077  }
5078 
5079  // Translate mode (Teuchos enum) to KokkosKernels (1-character string)
5080  const char* modeKK = nullptr;
5081  switch(mode)
5082  {
5083  case Teuchos::NO_TRANS:
5084  modeKK = KokkosSparse::NoTranspose; break;
5085  case Teuchos::TRANS:
5086  modeKK = KokkosSparse::Transpose; break;
5087  case Teuchos::CONJ_TRANS:
5088  modeKK = KokkosSparse::ConjugateTranspose; break;
5089  default:
5090  throw std::invalid_argument("Tpetra::CrsMatrix::localApply: invalid mode");
5091  }
5092 
5093  if(applyHelper->shouldUseIntRowptrs())
5094  {
5095  auto A_lcl_int_rowptrs = applyHelper->getIntRowptrMatrix(A_lcl);
5096  KokkosSparse::spmv(
5097  &applyHelper->handle_int, modeKK,
5098  impl_scalar_type(alpha), A_lcl_int_rowptrs, X_lcl, impl_scalar_type(beta), Y_lcl);
5099  }
5100  else
5101  {
5102  KokkosSparse::spmv(
5103  &applyHelper->handle, modeKK,
5104  impl_scalar_type(alpha), A_lcl, X_lcl, impl_scalar_type(beta), Y_lcl);
5105  }
5106 #else
5107  LocalOrdinal nrows = getLocalNumRows();
5108  LocalOrdinal maxRowImbalance = 0;
5109  if(nrows != 0)
5110  maxRowImbalance = getLocalMaxNumRowEntries() - (getLocalNumEntries() / nrows);
5111 
5112  auto matrix_lcl = getLocalMultiplyOperator();
5113  if(size_t(maxRowImbalance) >= Tpetra::Details::Behavior::rowImbalanceThreshold())
5114  matrix_lcl->applyImbalancedRows (X_lcl, Y_lcl, mode, alpha, beta);
5115  else
5116  matrix_lcl->apply (X_lcl, Y_lcl, mode, alpha, beta);
5117 #endif
5118  }
5119 
5120  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
5121  void
5122  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
5123  apply (const MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> &X,
5124  MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> &Y,
5125  Teuchos::ETransp mode,
5126  Scalar alpha,
5127  Scalar beta) const
5128  {
5129  using Tpetra::Details::ProfilingRegion;
5130  const char fnName[] = "Tpetra::CrsMatrix::apply";
5131 
5132  TEUCHOS_TEST_FOR_EXCEPTION
5133  (! isFillComplete (), std::runtime_error,
5134  fnName << ": Cannot call apply() until fillComplete() "
5135  "has been called.");
5136 
5137  if (mode == Teuchos::NO_TRANS) {
5138  ProfilingRegion regionNonTranspose (fnName);
5139  this->applyNonTranspose (X, Y, alpha, beta);
5140  }
5141  else {
5142  ProfilingRegion regionTranspose ("Tpetra::CrsMatrix::apply (transpose)");
5143  this->applyTranspose (X, Y, mode, alpha, beta);
5144  }
5145  }
5146 
5147 
5148  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
5149  template<class T>
5150  Teuchos::RCP<CrsMatrix<T, LocalOrdinal, GlobalOrdinal, Node> >
5151  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
5152  convert () const
5153  {
5154  using Teuchos::RCP;
5155  typedef CrsMatrix<T, LocalOrdinal, GlobalOrdinal, Node> output_matrix_type;
5156  const char tfecfFuncName[] = "convert: ";
5157 
5158  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5159  (! this->isFillComplete (), std::runtime_error, "This matrix (the source "
5160  "of the conversion) is not fill complete. You must first call "
5161  "fillComplete() (possibly with the domain and range Map) without an "
5162  "intervening call to resumeFill(), before you may call this method.");
5163 
5164  RCP<output_matrix_type> newMatrix
5165  (new output_matrix_type (this->getCrsGraph ()));
5166  // Copy old values into new values. impl_scalar_type and T may
5167  // differ, so we can't use Kokkos::deep_copy.
5168  using ::Tpetra::Details::copyConvert;
5169  copyConvert (newMatrix->getLocalMatrixDevice ().values,
5170  this->getLocalMatrixDevice ().values);
5171  // Since newmat has a static (const) graph, the graph already has
5172  // a column Map, and Import and Export objects already exist (if
5173  // applicable). Thus, calling fillComplete is cheap.
5174  newMatrix->fillComplete (this->getDomainMap (), this->getRangeMap ());
5175 
5176  return newMatrix;
5177  }
5178 
5179 
5180  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
5181  void
5182  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
5183  checkInternalState () const
5184  {
5185  const bool debug = ::Tpetra::Details::Behavior::debug ("CrsGraph");
5186  if (debug) {
5187  const char tfecfFuncName[] = "checkInternalState: ";
5188  const char err[] = "Internal state is not consistent. "
5189  "Please report this bug to the Tpetra developers.";
5190 
5191  // This version of the graph (RCP<const crs_graph_type>) must
5192  // always be nonnull.
5193  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5194  (staticGraph_.is_null (), std::logic_error, err);
5195  // myGraph == null means that the matrix has a const ("static")
5196  // graph. Otherwise, the matrix has a dynamic graph (it owns its
5197  // graph).
5198  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5199  (! myGraph_.is_null () && myGraph_ != staticGraph_,
5200  std::logic_error, err);
5201  // if matrix is fill complete, then graph must be fill complete
5202  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5203  (isFillComplete () && ! staticGraph_->isFillComplete (),
5204  std::logic_error, err << " Specifically, the matrix is fill complete, "
5205  "but its graph is NOT fill complete.");
5206  // if values are allocated and they are non-zero in number, then
5207  // one of the allocations should be present
5208  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5209  (staticGraph_->indicesAreAllocated () &&
5210  staticGraph_->getLocalAllocationSize() > 0 &&
5211  staticGraph_->getLocalNumRows() > 0 &&
5212  valuesUnpacked_wdv.extent (0) == 0,
5213  std::logic_error, err);
5214  }
5215  }
5216 
5217  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
5218  std::string
5219  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
5220  description () const
5221  {
5222  std::ostringstream os;
5223 
5224  os << "Tpetra::CrsMatrix (Kokkos refactor): {";
5225  if (this->getObjectLabel () != "") {
5226  os << "Label: \"" << this->getObjectLabel () << "\", ";
5227  }
5228  if (isFillComplete ()) {
5229  os << "isFillComplete: true"
5230  << ", global dimensions: [" << getGlobalNumRows () << ", "
5231  << getGlobalNumCols () << "]"
5232  << ", global number of entries: " << getGlobalNumEntries ()
5233  << "}";
5234  }
5235  else {
5236  os << "isFillComplete: false"
5237  << ", global dimensions: [" << getGlobalNumRows () << ", "
5238  << getGlobalNumCols () << "]}";
5239  }
5240  return os.str ();
5241  }
5242 
5243  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
5244  void
5245  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
5246  describe (Teuchos::FancyOStream &out,
5247  const Teuchos::EVerbosityLevel verbLevel) const
5248  {
5249  using std::endl;
5250  using std::setw;
5251  using Teuchos::ArrayView;
5252  using Teuchos::Comm;
5253  using Teuchos::RCP;
5254  using Teuchos::TypeNameTraits;
5255  using Teuchos::VERB_DEFAULT;
5256  using Teuchos::VERB_NONE;
5257  using Teuchos::VERB_LOW;
5258  using Teuchos::VERB_MEDIUM;
5259  using Teuchos::VERB_HIGH;
5260  using Teuchos::VERB_EXTREME;
5261 
5262  const Teuchos::EVerbosityLevel vl = (verbLevel == VERB_DEFAULT) ? VERB_LOW : verbLevel;
5263 
5264  if (vl == VERB_NONE) {
5265  return; // Don't print anything at all
5266  }
5267 
5268  // By convention, describe() always begins with a tab.
5269  Teuchos::OSTab tab0 (out);
5270 
5271  RCP<const Comm<int> > comm = this->getComm();
5272  const int myRank = comm->getRank();
5273  const int numProcs = comm->getSize();
5274  size_t width = 1;
5275  for (size_t dec=10; dec<getGlobalNumRows(); dec *= 10) {
5276  ++width;
5277  }
5278  width = std::max<size_t> (width, static_cast<size_t> (11)) + 2;
5279 
5280  // none: print nothing
5281  // low: print O(1) info from node 0
5282  // medium: print O(P) info, num entries per process
5283  // high: print O(N) info, num entries per row
5284  // extreme: print O(NNZ) info: print indices and values
5285  //
5286  // for medium and higher, print constituent objects at specified verbLevel
5287  if (myRank == 0) {
5288  out << "Tpetra::CrsMatrix (Kokkos refactor):" << endl;
5289  }
5290  Teuchos::OSTab tab1 (out);
5291 
5292  if (myRank == 0) {
5293  if (this->getObjectLabel () != "") {
5294  out << "Label: \"" << this->getObjectLabel () << "\", ";
5295  }
5296  {
5297  out << "Template parameters:" << endl;
5298  Teuchos::OSTab tab2 (out);
5299  out << "Scalar: " << TypeNameTraits<Scalar>::name () << endl
5300  << "LocalOrdinal: " << TypeNameTraits<LocalOrdinal>::name () << endl
5301  << "GlobalOrdinal: " << TypeNameTraits<GlobalOrdinal>::name () << endl
5302  << "Node: " << TypeNameTraits<Node>::name () << endl;
5303  }
5304  if (isFillComplete()) {
5305  out << "isFillComplete: true" << endl
5306  << "Global dimensions: [" << getGlobalNumRows () << ", "
5307  << getGlobalNumCols () << "]" << endl
5308  << "Global number of entries: " << getGlobalNumEntries () << endl
5309  << endl << "Global max number of entries in a row: "
5310  << getGlobalMaxNumRowEntries () << endl;
5311  }
5312  else {
5313  out << "isFillComplete: false" << endl
5314  << "Global dimensions: [" << getGlobalNumRows () << ", "
5315  << getGlobalNumCols () << "]" << endl;
5316  }
5317  }
5318 
5319  if (vl < VERB_MEDIUM) {
5320  return; // all done!
5321  }
5322 
5323  // Describe the row Map.
5324  if (myRank == 0) {
5325  out << endl << "Row Map:" << endl;
5326  }
5327  if (getRowMap ().is_null ()) {
5328  if (myRank == 0) {
5329  out << "null" << endl;
5330  }
5331  }
5332  else {
5333  if (myRank == 0) {
5334  out << endl;
5335  }
5336  getRowMap ()->describe (out, vl);
5337  }
5338 
5339  // Describe the column Map.
5340  if (myRank == 0) {
5341  out << "Column Map: ";
5342  }
5343  if (getColMap ().is_null ()) {
5344  if (myRank == 0) {
5345  out << "null" << endl;
5346  }
5347  } else if (getColMap () == getRowMap ()) {
5348  if (myRank == 0) {
5349  out << "same as row Map" << endl;
5350  }
5351  } else {
5352  if (myRank == 0) {
5353  out << endl;
5354  }
5355  getColMap ()->describe (out, vl);
5356  }
5357 
5358  // Describe the domain Map.
5359  if (myRank == 0) {
5360  out << "Domain Map: ";
5361  }
5362  if (getDomainMap ().is_null ()) {
5363  if (myRank == 0) {
5364  out << "null" << endl;
5365  }
5366  } else if (getDomainMap () == getRowMap ()) {
5367  if (myRank == 0) {
5368  out << "same as row Map" << endl;
5369  }
5370  } else if (getDomainMap () == getColMap ()) {
5371  if (myRank == 0) {
5372  out << "same as column Map" << endl;
5373  }
5374  } else {
5375  if (myRank == 0) {
5376  out << endl;
5377  }
5378  getDomainMap ()->describe (out, vl);
5379  }
5380 
5381  // Describe the range Map.
5382  if (myRank == 0) {
5383  out << "Range Map: ";
5384  }
5385  if (getRangeMap ().is_null ()) {
5386  if (myRank == 0) {
5387  out << "null" << endl;
5388  }
5389  } else if (getRangeMap () == getDomainMap ()) {
5390  if (myRank == 0) {
5391  out << "same as domain Map" << endl;
5392  }
5393  } else if (getRangeMap () == getRowMap ()) {
5394  if (myRank == 0) {
5395  out << "same as row Map" << endl;
5396  }
5397  } else {
5398  if (myRank == 0) {
5399  out << endl;
5400  }
5401  getRangeMap ()->describe (out, vl);
5402  }
5403 
5404  // O(P) data
5405  for (int curRank = 0; curRank < numProcs; ++curRank) {
5406  if (myRank == curRank) {
5407  out << "Process rank: " << curRank << endl;
5408  Teuchos::OSTab tab2 (out);
5409  if (! staticGraph_->indicesAreAllocated ()) {
5410  out << "Graph indices not allocated" << endl;
5411  }
5412  else {
5413  out << "Number of allocated entries: "
5414  << staticGraph_->getLocalAllocationSize () << endl;
5415  }
5416  out << "Number of entries: " << getLocalNumEntries () << endl
5417  << "Max number of entries per row: " << getLocalMaxNumRowEntries ()
5418  << endl;
5419  }
5420  // Give output time to complete by executing some barriers.
5421  comm->barrier ();
5422  comm->barrier ();
5423  comm->barrier ();
5424  }
5425 
5426  if (vl < VERB_HIGH) {
5427  return; // all done!
5428  }
5429 
5430  // O(N) and O(NNZ) data
5431  for (int curRank = 0; curRank < numProcs; ++curRank) {
5432  if (myRank == curRank) {
5433  out << std::setw(width) << "Proc Rank"
5434  << std::setw(width) << "Global Row"
5435  << std::setw(width) << "Num Entries";
5436  if (vl == VERB_EXTREME) {
5437  out << std::setw(width) << "(Index,Value)";
5438  }
5439  out << endl;
5440  for (size_t r = 0; r < getLocalNumRows (); ++r) {
5441  const size_t nE = getNumEntriesInLocalRow(r);
5442  GlobalOrdinal gid = getRowMap()->getGlobalElement(r);
5443  out << std::setw(width) << myRank
5444  << std::setw(width) << gid
5445  << std::setw(width) << nE;
5446  if (vl == VERB_EXTREME) {
5447  if (isGloballyIndexed()) {
5448  global_inds_host_view_type rowinds;
5449  values_host_view_type rowvals;
5450  getGlobalRowView (gid, rowinds, rowvals);
5451  for (size_t j = 0; j < nE; ++j) {
5452  out << " (" << rowinds[j]
5453  << ", " << rowvals[j]
5454  << ") ";
5455  }
5456  }
5457  else if (isLocallyIndexed()) {
5458  local_inds_host_view_type rowinds;
5459  values_host_view_type rowvals;
5460  getLocalRowView (r, rowinds, rowvals);
5461  for (size_t j=0; j < nE; ++j) {
5462  out << " (" << getColMap()->getGlobalElement(rowinds[j])
5463  << ", " << rowvals[j]
5464  << ") ";
5465  }
5466  } // globally or locally indexed
5467  } // vl == VERB_EXTREME
5468  out << endl;
5469  } // for each row r on this process
5470  } // if (myRank == curRank)
5471 
5472  // Give output time to complete
5473  comm->barrier ();
5474  comm->barrier ();
5475  comm->barrier ();
5476  } // for each process p
5477  }
5478 
5479  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
5480  bool
5481  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
5482  checkSizes (const SrcDistObject& source)
5483  {
5484  // It's not clear what kind of compatibility checks on sizes can
5485  // be performed here. Epetra_CrsGraph doesn't check any sizes for
5486  // compatibility.
5487 
5488  // Currently, the source object must be a RowMatrix with the same
5489  // four template parameters as the target CrsMatrix. We might
5490  // relax this requirement later.
5491  const row_matrix_type* srcRowMat =
5492  dynamic_cast<const row_matrix_type*> (&source);
5493  return (srcRowMat != nullptr);
5494  }
5495 
5496  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
5497  void
5498  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
5499  applyCrsPadding(
5500  const typename crs_graph_type::padding_type& padding,
5501  const bool verbose)
5502  {
5503  using Details::ProfilingRegion;
5504  using Details::padCrsArrays;
5505  using std::endl;
5506  using LO = local_ordinal_type;
5507  using row_ptrs_type =
5508  typename local_graph_device_type::row_map_type::non_const_type;
5509  using range_policy =
5510  Kokkos::RangePolicy<execution_space, Kokkos::IndexType<LO>>;
5511  const char tfecfFuncName[] = "applyCrsPadding";
5512  const char suffix[] =
5513  ". Please report this bug to the Tpetra developers.";
5514  ProfilingRegion regionCAP("Tpetra::CrsMatrix::applyCrsPadding");
5515 
5516  std::unique_ptr<std::string> prefix;
5517  if (verbose) {
5518  prefix = this->createPrefix("CrsMatrix", tfecfFuncName);
5519  std::ostringstream os;
5520  os << *prefix << "padding: ";
5521  padding.print(os);
5522  os << endl;
5523  std::cerr << os.str();
5524  }
5525  const int myRank = ! verbose ? -1 : [&] () {
5526  auto map = this->getMap();
5527  if (map.is_null()) {
5528  return -1;
5529  }
5530  auto comm = map->getComm();
5531  if (comm.is_null()) {
5532  return -1;
5533  }
5534  return comm->getRank();
5535  } ();
5536 
5537  // NOTE (mfh 29 Jan 2020) This allocates the values array.
5538  if (! myGraph_->indicesAreAllocated()) {
5539  if (verbose) {
5540  std::ostringstream os;
5541  os << *prefix << "Call allocateIndices" << endl;
5542  std::cerr << os.str();
5543  }
5544  allocateValues(GlobalIndices, GraphNotYetAllocated, verbose);
5545  }
5546 
5547  // FIXME (mfh 10 Feb 2020) We shouldn't actually reallocate
5548  // row_ptrs_beg or allocate row_ptrs_end unless the allocation
5549  // size needs to increase. That should be the job of
5550  // padCrsArrays.
5551 
5552  // Making copies here because rowPtrsUnpacked_ has a const type. Otherwise, we
5553  // would use it directly.
5554 
5555  if (verbose) {
5556  std::ostringstream os;
5557  os << *prefix << "Allocate row_ptrs_beg: "
5558  << myGraph_->getRowPtrsUnpackedHost().extent(0) << endl;
5559  std::cerr << os.str();
5560  }
5561  using Kokkos::view_alloc;
5562  using Kokkos::WithoutInitializing;
5563  row_ptrs_type row_ptr_beg(view_alloc("row_ptr_beg", WithoutInitializing),
5564  myGraph_->rowPtrsUnpacked_dev_.extent(0));
5565  // DEEP_COPY REVIEW - DEVICE-TO-DEVICE
5566  Kokkos::deep_copy(execution_space(),row_ptr_beg, myGraph_->rowPtrsUnpacked_dev_);
5567 
5568  const size_t N = row_ptr_beg.extent(0) == 0 ? size_t(0) :
5569  size_t(row_ptr_beg.extent(0) - 1);
5570  if (verbose) {
5571  std::ostringstream os;
5572  os << *prefix << "Allocate row_ptrs_end: " << N << endl;
5573  std::cerr << os.str();
5574  }
5575  row_ptrs_type row_ptr_end(
5576  view_alloc("row_ptr_end", WithoutInitializing), N);
5577 
5578  row_ptrs_type num_row_entries_d;
5579 
5580  const bool refill_num_row_entries =
5581  myGraph_->k_numRowEntries_.extent(0) != 0;
5582 
5583  if (refill_num_row_entries) { // unpacked storage
5584  // We can't assume correct *this capture until C++17, and it's
5585  // likely more efficient just to capture what we need anyway.
5586  num_row_entries_d = create_mirror_view_and_copy(memory_space(),
5587  myGraph_->k_numRowEntries_);
5588  Kokkos::parallel_for
5589  ("Fill end row pointers", range_policy(0, N),
5590  KOKKOS_LAMBDA (const size_t i) {
5591  row_ptr_end(i) = row_ptr_beg(i) + num_row_entries_d(i);
5592  });
5593  }
5594  else {
5595  // FIXME (mfh 04 Feb 2020) Fix padCrsArrays so that if packed
5596  // storage, we don't need row_ptr_end to be separate allocation;
5597  // could just have it alias row_ptr_beg+1.
5598  Kokkos::parallel_for
5599  ("Fill end row pointers", range_policy(0, N),
5600  KOKKOS_LAMBDA (const size_t i) {
5601  row_ptr_end(i) = row_ptr_beg(i+1);
5602  });
5603  }
5604 
5605  if (myGraph_->isGloballyIndexed()) {
5606  padCrsArrays(row_ptr_beg, row_ptr_end,
5607  myGraph_->gblInds_wdv,
5608  valuesUnpacked_wdv, padding, myRank, verbose);
5609  const auto newValuesLen = valuesUnpacked_wdv.extent(0);
5610  const auto newColIndsLen = myGraph_->gblInds_wdv.extent(0);
5611  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5612  (newValuesLen != newColIndsLen, std::logic_error,
5613  ": After padding, valuesUnpacked_wdv.extent(0)=" << newValuesLen
5614  << " != myGraph_->gblInds_wdv.extent(0)=" << newColIndsLen
5615  << suffix);
5616  }
5617  else {
5618  padCrsArrays(row_ptr_beg, row_ptr_end,
5619  myGraph_->lclIndsUnpacked_wdv,
5620  valuesUnpacked_wdv, padding, myRank, verbose);
5621  const auto newValuesLen = valuesUnpacked_wdv.extent(0);
5622  const auto newColIndsLen = myGraph_->lclIndsUnpacked_wdv.extent(0);
5623  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5624  (newValuesLen != newColIndsLen, std::logic_error,
5625  ": After padding, valuesUnpacked_wdv.extent(0)=" << newValuesLen
5626  << " != myGraph_->lclIndsUnpacked_wdv.extent(0)=" << newColIndsLen
5627  << suffix);
5628  }
5629 
5630  if (refill_num_row_entries) {
5631  Kokkos::parallel_for
5632  ("Fill num entries", range_policy(0, N),
5633  KOKKOS_LAMBDA (const size_t i) {
5634  num_row_entries_d(i) = row_ptr_end(i) - row_ptr_beg(i);
5635  });
5636  Kokkos::deep_copy(myGraph_->k_numRowEntries_, num_row_entries_d);
5637  }
5638 
5639  if (verbose) {
5640  std::ostringstream os;
5641  os << *prefix << "Assign myGraph_->rowPtrsUnpacked_; "
5642  << "old size: " << myGraph_->rowPtrsUnpacked_host_.extent(0)
5643  << ", new size: " << row_ptr_beg.extent(0) << endl;
5644  std::cerr << os.str();
5645  TEUCHOS_ASSERT( myGraph_->getRowPtrsUnpackedHost().extent(0) ==
5646  row_ptr_beg.extent(0) );
5647  }
5648  myGraph_->setRowPtrsUnpacked(row_ptr_beg);
5649  }
5650 
5651  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
5652  void
5653  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
5654  copyAndPermuteStaticGraph(
5655  const RowMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>& srcMat,
5656  const size_t numSameIDs,
5657  const LocalOrdinal permuteToLIDs[],
5658  const LocalOrdinal permuteFromLIDs[],
5659  const size_t numPermutes)
5660  {
5661  using Details::ProfilingRegion;
5662  using Teuchos::Array;
5663  using Teuchos::ArrayView;
5664  using std::endl;
5665  using LO = LocalOrdinal;
5666  using GO = GlobalOrdinal;
5667  const char tfecfFuncName[] = "copyAndPermuteStaticGraph";
5668  const char suffix[] =
5669  " Please report this bug to the Tpetra developers.";
5670  ProfilingRegion regionCAP
5671  ("Tpetra::CrsMatrix::copyAndPermuteStaticGraph");
5672 
5673  const bool debug = Details::Behavior::debug("CrsGraph");
5674  const bool verbose = Details::Behavior::verbose("CrsGraph");
5675  std::unique_ptr<std::string> prefix;
5676  if (verbose) {
5677  prefix = this->createPrefix("CrsGraph", tfecfFuncName);
5678  std::ostringstream os;
5679  os << *prefix << "Start" << endl;
5680  }
5681  const char* const prefix_raw =
5682  verbose ? prefix.get()->c_str() : nullptr;
5683 
5684  const bool sourceIsLocallyIndexed = srcMat.isLocallyIndexed ();
5685  //
5686  // Copy the first numSame row from source to target (this matrix).
5687  // This involves copying rows corresponding to LIDs [0, numSame-1].
5688  //
5689  const map_type& srcRowMap = * (srcMat.getRowMap ());
5690  nonconst_global_inds_host_view_type rowInds;
5691  nonconst_values_host_view_type rowVals;
5692  const LO numSameIDs_as_LID = static_cast<LO> (numSameIDs);
5693  for (LO sourceLID = 0; sourceLID < numSameIDs_as_LID; ++sourceLID) {
5694  // Global ID for the current row index in the source matrix.
5695  // The first numSameIDs GIDs in the two input lists are the
5696  // same, so sourceGID == targetGID in this case.
5697  const GO sourceGID = srcRowMap.getGlobalElement (sourceLID);
5698  const GO targetGID = sourceGID;
5699 
5700  ArrayView<const GO>rowIndsConstView;
5701  ArrayView<const Scalar> rowValsConstView;
5702 
5703  if (sourceIsLocallyIndexed) {
5704  const size_t rowLength = srcMat.getNumEntriesInGlobalRow (sourceGID);
5705  if (rowLength > static_cast<size_t> (rowInds.size())) {
5706  Kokkos::resize(rowInds,rowLength);
5707  Kokkos::resize(rowVals,rowLength);
5708  }
5709  // Resizing invalidates an Array's views, so we must make new
5710  // ones, even if rowLength hasn't changed.
5711  nonconst_global_inds_host_view_type rowIndsView = Kokkos::subview(rowInds,std::make_pair((size_t)0, rowLength));
5712  nonconst_values_host_view_type rowValsView = Kokkos::subview(rowVals,std::make_pair((size_t)0, rowLength));
5713 
5714  // The source matrix is locally indexed, so we have to get a
5715  // copy. Really it's the GIDs that have to be copied (because
5716  // they have to be converted from LIDs).
5717  size_t checkRowLength = 0;
5718  srcMat.getGlobalRowCopy (sourceGID, rowIndsView,
5719  rowValsView, checkRowLength);
5720  if (debug) {
5721  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5722  (rowLength != checkRowLength, std::logic_error, "For "
5723  "global row index " << sourceGID << ", the source "
5724  "matrix's getNumEntriesInGlobalRow returns a row length "
5725  "of " << rowLength << ", but getGlobalRowCopy reports "
5726  "a row length of " << checkRowLength << "." << suffix);
5727  }
5728 
5729  // KDDKDD UVM TEMPORARY: refactor combineGlobalValues to take
5730  // KDDKDD UVM TEMPORARY: Kokkos::View instead of ArrayView
5731  // KDDKDD UVM TEMPORARY: For now, wrap the view in ArrayViews
5732  // KDDKDD UVM TEMPORARY: Should be safe because we hold the KokkosViews
5733  rowIndsConstView = Teuchos::ArrayView<const GO> ( // BAD BAD BAD
5734  rowIndsView.data(), rowIndsView.extent(0),
5735  Teuchos::RCP_DISABLE_NODE_LOOKUP);
5736  rowValsConstView = Teuchos::ArrayView<const Scalar> ( // BAD BAD BAD
5737  reinterpret_cast<const Scalar*>(rowValsView.data()), rowValsView.extent(0),
5738  Teuchos::RCP_DISABLE_NODE_LOOKUP);
5739  // KDDKDD UVM TEMPORARY: Add replace, sum, transform methods with
5740  // KDDKDD UVM TEMPORARY: KokkosView interface
5741  }
5742  else { // source matrix is globally indexed.
5743  global_inds_host_view_type rowIndsView;
5744  values_host_view_type rowValsView;
5745  srcMat.getGlobalRowView(sourceGID, rowIndsView, rowValsView);
5746  // KDDKDD UVM TEMPORARY: refactor combineGlobalValues to take
5747  // KDDKDD UVM TEMPORARY: Kokkos::View instead of ArrayView
5748  // KDDKDD UVM TEMPORARY: For now, wrap the view in ArrayViews
5749  // KDDKDD UVM TEMPORARY: Should be safe because we hold the KokkosViews
5750  rowIndsConstView = Teuchos::ArrayView<const GO> ( // BAD BAD BAD
5751  rowIndsView.data(), rowIndsView.extent(0),
5752  Teuchos::RCP_DISABLE_NODE_LOOKUP);
5753  rowValsConstView = Teuchos::ArrayView<const Scalar> ( // BAD BAD BAD
5754  reinterpret_cast<const Scalar*>(rowValsView.data()), rowValsView.extent(0),
5755  Teuchos::RCP_DISABLE_NODE_LOOKUP);
5756  // KDDKDD UVM TEMPORARY: Add replace, sum, transform methods with
5757  // KDDKDD UVM TEMPORARY: KokkosView interface
5758 
5759  }
5760 
5761  // Applying a permutation to a matrix with a static graph
5762  // means REPLACE-ing entries.
5763  combineGlobalValues(targetGID, rowIndsConstView,
5764  rowValsConstView, REPLACE,
5765  prefix_raw, debug, verbose);
5766  }
5767 
5768  if (verbose) {
5769  std::ostringstream os;
5770  os << *prefix << "Do permutes" << endl;
5771  }
5772 
5773  const map_type& tgtRowMap = * (this->getRowMap ());
5774  for (size_t p = 0; p < numPermutes; ++p) {
5775  const GO sourceGID = srcRowMap.getGlobalElement (permuteFromLIDs[p]);
5776  const GO targetGID = tgtRowMap.getGlobalElement (permuteToLIDs[p]);
5777 
5778  ArrayView<const GO> rowIndsConstView;
5779  ArrayView<const Scalar> rowValsConstView;
5780 
5781  if (sourceIsLocallyIndexed) {
5782  const size_t rowLength = srcMat.getNumEntriesInGlobalRow (sourceGID);
5783  if (rowLength > static_cast<size_t> (rowInds.size ())) {
5784  Kokkos::resize(rowInds,rowLength);
5785  Kokkos::resize(rowVals,rowLength);
5786  }
5787  // Resizing invalidates an Array's views, so we must make new
5788  // ones, even if rowLength hasn't changed.
5789  nonconst_global_inds_host_view_type rowIndsView = Kokkos::subview(rowInds,std::make_pair((size_t)0, rowLength));
5790  nonconst_values_host_view_type rowValsView = Kokkos::subview(rowVals,std::make_pair((size_t)0, rowLength));
5791 
5792  // The source matrix is locally indexed, so we have to get a
5793  // copy. Really it's the GIDs that have to be copied (because
5794  // they have to be converted from LIDs).
5795  size_t checkRowLength = 0;
5796  srcMat.getGlobalRowCopy(sourceGID, rowIndsView,
5797  rowValsView, checkRowLength);
5798  if (debug) {
5799  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5800  (rowLength != checkRowLength, std::logic_error, "For "
5801  "source matrix global row index " << sourceGID << ", "
5802  "getNumEntriesInGlobalRow returns a row length of " <<
5803  rowLength << ", but getGlobalRowCopy a row length of "
5804  << checkRowLength << "." << suffix);
5805  }
5806 
5807  // KDDKDD UVM TEMPORARY: refactor combineGlobalValues to take
5808  // KDDKDD UVM TEMPORARY: Kokkos::View instead of ArrayView
5809  // KDDKDD UVM TEMPORARY: For now, wrap the view in ArrayViews
5810  // KDDKDD UVM TEMPORARY: Should be safe because we hold the KokkosViews
5811  rowIndsConstView = Teuchos::ArrayView<const GO> ( // BAD BAD BAD
5812  rowIndsView.data(), rowIndsView.extent(0),
5813  Teuchos::RCP_DISABLE_NODE_LOOKUP);
5814  rowValsConstView = Teuchos::ArrayView<const Scalar> ( // BAD BAD BAD
5815  reinterpret_cast<const Scalar*>(rowValsView.data()), rowValsView.extent(0),
5816  Teuchos::RCP_DISABLE_NODE_LOOKUP);
5817  // KDDKDD UVM TEMPORARY: Add replace, sum, transform methods with
5818  // KDDKDD UVM TEMPORARY: KokkosView interface
5819  }
5820  else {
5821  global_inds_host_view_type rowIndsView;
5822  values_host_view_type rowValsView;
5823  srcMat.getGlobalRowView(sourceGID, rowIndsView, rowValsView);
5824  // KDDKDD UVM TEMPORARY: refactor combineGlobalValues to take
5825  // KDDKDD UVM TEMPORARY: Kokkos::View instead of ArrayView
5826  // KDDKDD UVM TEMPORARY: For now, wrap the view in ArrayViews
5827  // KDDKDD UVM TEMPORARY: Should be safe because we hold the KokkosViews
5828  rowIndsConstView = Teuchos::ArrayView<const GO> ( // BAD BAD BAD
5829  rowIndsView.data(), rowIndsView.extent(0),
5830  Teuchos::RCP_DISABLE_NODE_LOOKUP);
5831  rowValsConstView = Teuchos::ArrayView<const Scalar> ( // BAD BAD BAD
5832  reinterpret_cast<const Scalar*>(rowValsView.data()), rowValsView.extent(0),
5833  Teuchos::RCP_DISABLE_NODE_LOOKUP);
5834  // KDDKDD UVM TEMPORARY: Add replace, sum, transform methods with
5835  // KDDKDD UVM TEMPORARY: KokkosView interface
5836  }
5837 
5838  combineGlobalValues(targetGID, rowIndsConstView,
5839  rowValsConstView, REPLACE,
5840  prefix_raw, debug, verbose);
5841  }
5842 
5843  if (verbose) {
5844  std::ostringstream os;
5845  os << *prefix << "Done" << endl;
5846  }
5847  }
5848 
5849  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
5850  void
5851  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
5852  copyAndPermuteNonStaticGraph(
5853  const RowMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>& srcMat,
5854  const size_t numSameIDs,
5855  const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& permuteToLIDs_dv,
5856  const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& permuteFromLIDs_dv,
5857  const size_t numPermutes)
5858  {
5859  using Details::ProfilingRegion;
5860  using Teuchos::Array;
5861  using Teuchos::ArrayView;
5862  using std::endl;
5863  using LO = LocalOrdinal;
5864  using GO = GlobalOrdinal;
5865  const char tfecfFuncName[] = "copyAndPermuteNonStaticGraph";
5866  const char suffix[] =
5867  " Please report this bug to the Tpetra developers.";
5868  ProfilingRegion regionCAP
5869  ("Tpetra::CrsMatrix::copyAndPermuteNonStaticGraph");
5870 
5871  const bool debug = Details::Behavior::debug("CrsGraph");
5872  const bool verbose = Details::Behavior::verbose("CrsGraph");
5873  std::unique_ptr<std::string> prefix;
5874  if (verbose) {
5875  prefix = this->createPrefix("CrsGraph", tfecfFuncName);
5876  std::ostringstream os;
5877  os << *prefix << "Start" << endl;
5878  }
5879  const char* const prefix_raw =
5880  verbose ? prefix.get()->c_str() : nullptr;
5881 
5882  {
5883  using row_graph_type = RowGraph<LO, GO, Node>;
5884  const row_graph_type& srcGraph = *(srcMat.getGraph());
5885  auto padding =
5886  myGraph_->computeCrsPadding(srcGraph, numSameIDs,
5887  permuteToLIDs_dv, permuteFromLIDs_dv, verbose);
5888  applyCrsPadding(*padding, verbose);
5889  }
5890  const bool sourceIsLocallyIndexed = srcMat.isLocallyIndexed ();
5891  //
5892  // Copy the first numSame row from source to target (this matrix).
5893  // This involves copying rows corresponding to LIDs [0, numSame-1].
5894  //
5895  const map_type& srcRowMap = * (srcMat.getRowMap ());
5896  const LO numSameIDs_as_LID = static_cast<LO> (numSameIDs);
5897  using gids_type = nonconst_global_inds_host_view_type;
5898  using vals_type = nonconst_values_host_view_type;
5899  gids_type rowInds;
5900  vals_type rowVals;
5901  for (LO sourceLID = 0; sourceLID < numSameIDs_as_LID; ++sourceLID) {
5902  // Global ID for the current row index in the source matrix.
5903  // The first numSameIDs GIDs in the two input lists are the
5904  // same, so sourceGID == targetGID in this case.
5905  const GO sourceGID = srcRowMap.getGlobalElement (sourceLID);
5906  const GO targetGID = sourceGID;
5907 
5908  ArrayView<const GO> rowIndsConstView;
5909  ArrayView<const Scalar> rowValsConstView;
5910 
5911  if (sourceIsLocallyIndexed) {
5912 
5913  const size_t rowLength = srcMat.getNumEntriesInGlobalRow (sourceGID);
5914  if (rowLength > static_cast<size_t> (rowInds.extent(0))) {
5915  Kokkos::resize(rowInds,rowLength);
5916  Kokkos::resize(rowVals,rowLength);
5917  }
5918  // Resizing invalidates an Array's views, so we must make new
5919  // ones, even if rowLength hasn't changed.
5920  gids_type rowIndsView = Kokkos::subview(rowInds,std::make_pair((size_t)0, rowLength));
5921  vals_type rowValsView = Kokkos::subview(rowVals,std::make_pair((size_t)0, rowLength));
5922 
5923  // The source matrix is locally indexed, so we have to get a
5924  // copy. Really it's the GIDs that have to be copied (because
5925  // they have to be converted from LIDs).
5926  size_t checkRowLength = 0;
5927  srcMat.getGlobalRowCopy (sourceGID, rowIndsView, rowValsView,
5928  checkRowLength);
5929  if (debug) {
5930  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5931  (rowLength != checkRowLength, std::logic_error, ": For "
5932  "global row index " << sourceGID << ", the source "
5933  "matrix's getNumEntriesInGlobalRow returns a row length "
5934  "of " << rowLength << ", but getGlobalRowCopy reports "
5935  "a row length of " << checkRowLength << "." << suffix);
5936  }
5937  rowIndsConstView = Teuchos::ArrayView<const GO>(rowIndsView.data(), rowLength);
5938  rowValsConstView = Teuchos::ArrayView<const Scalar>(reinterpret_cast<Scalar *>(rowValsView.data()), rowLength);
5939  }
5940  else { // source matrix is globally indexed.
5941  global_inds_host_view_type rowIndsView;
5942  values_host_view_type rowValsView;
5943  srcMat.getGlobalRowView(sourceGID, rowIndsView, rowValsView);
5944 
5945  // KDDKDD UVM TEMPORARY: refactor combineGlobalValues to take
5946  // KDDKDD UVM TEMPORARY: Kokkos::View instead of ArrayView
5947  // KDDKDD UVM TEMPORARY: For now, wrap the view in ArrayViews
5948  // KDDKDD UVM TEMPORARY: Should be safe because we hold the KokkosViews
5949  rowIndsConstView = Teuchos::ArrayView<const GO> ( // BAD BAD BAD
5950  rowIndsView.data(), rowIndsView.extent(0),
5951  Teuchos::RCP_DISABLE_NODE_LOOKUP);
5952  rowValsConstView = Teuchos::ArrayView<const Scalar> ( // BAD BAD BAD
5953  reinterpret_cast<const Scalar*>(rowValsView.data()), rowValsView.extent(0),
5954  Teuchos::RCP_DISABLE_NODE_LOOKUP);
5955  // KDDKDD UVM TEMPORARY: Add replace, sum, transform methods with
5956  // KDDKDD UVM TEMPORARY: KokkosView interface
5957  }
5958 
5959  // Combine the data into the target matrix.
5960  insertGlobalValuesFilteredChecked(targetGID, rowIndsConstView,
5961  rowValsConstView, prefix_raw, debug, verbose);
5962  }
5963 
5964  if (verbose) {
5965  std::ostringstream os;
5966  os << *prefix << "Do permutes" << endl;
5967  }
5968  const LO* const permuteFromLIDs = permuteFromLIDs_dv.view_host().data();
5969  const LO* const permuteToLIDs = permuteToLIDs_dv.view_host().data();
5970 
5971  const map_type& tgtRowMap = * (this->getRowMap ());
5972  for (size_t p = 0; p < numPermutes; ++p) {
5973  const GO sourceGID = srcRowMap.getGlobalElement (permuteFromLIDs[p]);
5974  const GO targetGID = tgtRowMap.getGlobalElement (permuteToLIDs[p]);
5975 
5976  ArrayView<const GO> rowIndsConstView;
5977  ArrayView<const Scalar> rowValsConstView;
5978 
5979  if (sourceIsLocallyIndexed) {
5980  const size_t rowLength = srcMat.getNumEntriesInGlobalRow (sourceGID);
5981  if (rowLength > static_cast<size_t> (rowInds.extent(0))) {
5982  Kokkos::resize(rowInds,rowLength);
5983  Kokkos::resize(rowVals,rowLength);
5984  }
5985  // Resizing invalidates an Array's views, so we must make new
5986  // ones, even if rowLength hasn't changed.
5987  gids_type rowIndsView = Kokkos::subview(rowInds,std::make_pair((size_t)0, rowLength));
5988  vals_type rowValsView = Kokkos::subview(rowVals,std::make_pair((size_t)0, rowLength));
5989 
5990  // The source matrix is locally indexed, so we have to get a
5991  // copy. Really it's the GIDs that have to be copied (because
5992  // they have to be converted from LIDs).
5993  size_t checkRowLength = 0;
5994  srcMat.getGlobalRowCopy(sourceGID, rowIndsView,
5995  rowValsView, checkRowLength);
5996  if (debug) {
5997  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
5998  (rowLength != checkRowLength, std::logic_error, "For "
5999  "source matrix global row index " << sourceGID << ", "
6000  "getNumEntriesInGlobalRow returns a row length of " <<
6001  rowLength << ", but getGlobalRowCopy a row length of "
6002  << checkRowLength << "." << suffix);
6003  }
6004  rowIndsConstView = Teuchos::ArrayView<const GO>(rowIndsView.data(), rowLength);
6005  rowValsConstView = Teuchos::ArrayView<const Scalar>(reinterpret_cast<Scalar *>(rowValsView.data()), rowLength);
6006  }
6007  else {
6008  global_inds_host_view_type rowIndsView;
6009  values_host_view_type rowValsView;
6010  srcMat.getGlobalRowView(sourceGID, rowIndsView, rowValsView);
6011 
6012  // KDDKDD UVM TEMPORARY: refactor combineGlobalValues to take
6013  // KDDKDD UVM TEMPORARY: Kokkos::View instead of ArrayView
6014  // KDDKDD UVM TEMPORARY: For now, wrap the view in ArrayViews
6015  // KDDKDD UVM TEMPORARY: Should be safe because we hold the KokkosViews
6016  rowIndsConstView = Teuchos::ArrayView<const GO> ( // BAD BAD BAD
6017  rowIndsView.data(), rowIndsView.extent(0),
6018  Teuchos::RCP_DISABLE_NODE_LOOKUP);
6019  rowValsConstView = Teuchos::ArrayView<const Scalar> ( // BAD BAD BAD
6020  reinterpret_cast<const Scalar*>(rowValsView.data()), rowValsView.extent(0),
6021  Teuchos::RCP_DISABLE_NODE_LOOKUP);
6022  // KDDKDD UVM TEMPORARY: Add replace, sum, transform methods with
6023  // KDDKDD UVM TEMPORARY: KokkosView interface
6024  }
6025 
6026  // Combine the data into the target matrix.
6027  insertGlobalValuesFilteredChecked(targetGID, rowIndsConstView,
6028  rowValsConstView, prefix_raw, debug, verbose);
6029  }
6030 
6031  if (verbose) {
6032  std::ostringstream os;
6033  os << *prefix << "Done" << endl;
6034  }
6035  }
6036 
6037  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
6038  void
6039  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
6040  copyAndPermute(
6041  const SrcDistObject& srcObj,
6042  const size_t numSameIDs,
6043  const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& permuteToLIDs,
6044  const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& permuteFromLIDs,
6045  const CombineMode /*CM*/)
6046  {
6047  using Details::Behavior;
6048  using Details::dualViewStatusToString;
6049  using Details::ProfilingRegion;
6050  using std::endl;
6051 
6052  // Method name string for TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC.
6053  const char tfecfFuncName[] = "copyAndPermute: ";
6054  ProfilingRegion regionCAP("Tpetra::CrsMatrix::copyAndPermute");
6055 
6056  const bool verbose = Behavior::verbose("CrsMatrix");
6057  std::unique_ptr<std::string> prefix;
6058  if (verbose) {
6059  prefix = this->createPrefix("CrsMatrix", "copyAndPermute");
6060  std::ostringstream os;
6061  os << *prefix << endl
6062  << *prefix << " numSameIDs: " << numSameIDs << endl
6063  << *prefix << " numPermute: " << permuteToLIDs.extent(0)
6064  << endl
6065  << *prefix << " "
6066  << dualViewStatusToString (permuteToLIDs, "permuteToLIDs")
6067  << endl
6068  << *prefix << " "
6069  << dualViewStatusToString (permuteFromLIDs, "permuteFromLIDs")
6070  << endl
6071  << *prefix << " "
6072  << "isStaticGraph: " << (isStaticGraph() ? "true" : "false")
6073  << endl;
6074  std::cerr << os.str ();
6075  }
6076 
6077  const auto numPermute = permuteToLIDs.extent (0);
6078  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6079  (numPermute != permuteFromLIDs.extent (0),
6080  std::invalid_argument, "permuteToLIDs.extent(0) = "
6081  << numPermute << "!= permuteFromLIDs.extent(0) = "
6082  << permuteFromLIDs.extent (0) << ".");
6083 
6084  // This dynamic cast should succeed, because we've already tested
6085  // it in checkSizes().
6086  using RMT = RowMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>;
6087  const RMT& srcMat = dynamic_cast<const RMT&> (srcObj);
6088  if (isStaticGraph ()) {
6089  TEUCHOS_ASSERT( ! permuteToLIDs.need_sync_host () );
6090  auto permuteToLIDs_h = permuteToLIDs.view_host ();
6091  TEUCHOS_ASSERT( ! permuteFromLIDs.need_sync_host () );
6092  auto permuteFromLIDs_h = permuteFromLIDs.view_host ();
6093 
6094  copyAndPermuteStaticGraph(srcMat, numSameIDs,
6095  permuteToLIDs_h.data(),
6096  permuteFromLIDs_h.data(),
6097  numPermute);
6098  }
6099  else {
6100  copyAndPermuteNonStaticGraph(srcMat, numSameIDs, permuteToLIDs,
6101  permuteFromLIDs, numPermute);
6102  }
6103 
6104  if (verbose) {
6105  std::ostringstream os;
6106  os << *prefix << "Done" << endl;
6107  std::cerr << os.str();
6108  }
6109  }
6110 
6111  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
6112  void
6113  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
6114  packAndPrepare
6115  (const SrcDistObject& source,
6116  const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& exportLIDs,
6117  Kokkos::DualView<char*, buffer_device_type>& exports,
6118  Kokkos::DualView<size_t*, buffer_device_type> numPacketsPerLID,
6119  size_t& constantNumPackets)
6120  {
6121  using Details::Behavior;
6122  using Details::dualViewStatusToString;
6123  using Details::ProfilingRegion;
6124  using Teuchos::outArg;
6125  using Teuchos::REDUCE_MAX;
6126  using Teuchos::reduceAll;
6127  using std::endl;
6128  typedef LocalOrdinal LO;
6129  typedef GlobalOrdinal GO;
6130  const char tfecfFuncName[] = "packAndPrepare: ";
6131  ProfilingRegion regionPAP ("Tpetra::CrsMatrix::packAndPrepare");
6132 
6133  const bool debug = Behavior::debug("CrsMatrix");
6134  const bool verbose = Behavior::verbose("CrsMatrix");
6135 
6136  // Processes on which the communicator is null should not participate.
6137  Teuchos::RCP<const Teuchos::Comm<int> > pComm = this->getComm ();
6138  if (pComm.is_null ()) {
6139  return;
6140  }
6141  const Teuchos::Comm<int>& comm = *pComm;
6142  const int myRank = comm.getSize ();
6143 
6144  std::unique_ptr<std::string> prefix;
6145  if (verbose) {
6146  prefix = this->createPrefix("CrsMatrix", "packAndPrepare");
6147  std::ostringstream os;
6148  os << *prefix << "Start" << endl
6149  << *prefix << " "
6150  << dualViewStatusToString (exportLIDs, "exportLIDs")
6151  << endl
6152  << *prefix << " "
6153  << dualViewStatusToString (exports, "exports")
6154  << endl
6155  << *prefix << " "
6156  << dualViewStatusToString (numPacketsPerLID, "numPacketsPerLID")
6157  << endl;
6158  std::cerr << os.str ();
6159  }
6160 
6161  // Attempt to cast the source object to CrsMatrix. If successful,
6162  // use the source object's packNew() method to pack its data for
6163  // communication. Otherwise, attempt to cast to RowMatrix; if
6164  // successful, use the source object's pack() method. Otherwise,
6165  // the source object doesn't have the right type.
6166  //
6167  // FIXME (mfh 30 Jun 2013, 11 Sep 2017) We don't even need the
6168  // RowMatrix to have the same Node type. Unfortunately, we don't
6169  // have a way to ask if the RowMatrix is "a RowMatrix with any
6170  // Node type," since RowMatrix doesn't have a base class. A
6171  // hypothetical RowMatrixBase<Scalar, LO, GO> class, which does
6172  // not currently exist, would satisfy this requirement.
6173  //
6174  // Why RowMatrixBase<Scalar, LO, GO>? The source object's Scalar
6175  // type doesn't technically need to match the target object's
6176  // Scalar type, so we could just have RowMatrixBase<LO, GO>. LO
6177  // and GO need not be the same, as long as there is no overflow of
6178  // the indices. However, checking for index overflow is global
6179  // and therefore undesirable.
6180 
6181  std::ostringstream msg; // for collecting error messages
6182  int lclBad = 0; // to be set below
6183 
6184  using crs_matrix_type = CrsMatrix<Scalar, LO, GO, Node>;
6185  const crs_matrix_type* srcCrsMat =
6186  dynamic_cast<const crs_matrix_type*> (&source);
6187  if (srcCrsMat != nullptr) {
6188  if (verbose) {
6189  std::ostringstream os;
6190  os << *prefix << "Source matrix same (CrsMatrix) type as target; "
6191  "calling packNew" << endl;
6192  std::cerr << os.str ();
6193  }
6194  try {
6195  srcCrsMat->packNew (exportLIDs, exports, numPacketsPerLID,
6196  constantNumPackets);
6197  }
6198  catch (std::exception& e) {
6199  lclBad = 1;
6200  msg << "Proc " << myRank << ": " << e.what () << std::endl;
6201  }
6202  }
6203  else {
6204  using Kokkos::HostSpace;
6205  using Kokkos::subview;
6206  using exports_type = Kokkos::DualView<char*, buffer_device_type>;
6207  using range_type = Kokkos::pair<size_t, size_t>;
6208 
6209  if (verbose) {
6210  std::ostringstream os;
6211  os << *prefix << "Source matrix NOT same (CrsMatrix) type as target"
6212  << endl;
6213  std::cerr << os.str ();
6214  }
6215 
6216  const row_matrix_type* srcRowMat =
6217  dynamic_cast<const row_matrix_type*> (&source);
6218  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6219  (srcRowMat == nullptr, std::invalid_argument,
6220  "The source object of the Import or Export operation is neither a "
6221  "CrsMatrix (with the same template parameters as the target object), "
6222  "nor a RowMatrix (with the same first four template parameters as the "
6223  "target object).");
6224 
6225  // For the RowMatrix case, we need to convert from
6226  // Kokkos::DualView to Teuchos::Array*. This doesn't need to be
6227  // so terribly efficient, since packing a non-CrsMatrix
6228  // RowMatrix for Import/Export into a CrsMatrix is not a
6229  // critical case. Thus, we may allocate Teuchos::Array objects
6230  // here and copy to and from Kokkos::*View.
6231 
6232  // View exportLIDs's host data as a Teuchos::ArrayView.
6233  TEUCHOS_ASSERT( ! exportLIDs.need_sync_host () );
6234  auto exportLIDs_h = exportLIDs.view_host ();
6235  Teuchos::ArrayView<const LO> exportLIDs_av (exportLIDs_h.data (),
6236  exportLIDs_h.size ());
6237 
6238  // pack() will allocate exports_a as needed. We'll copy back
6239  // into exports (after (re)allocating exports if needed) below.
6240  Teuchos::Array<char> exports_a;
6241 
6242  // View exportLIDs' host data as a Teuchos::ArrayView. We don't
6243  // need to sync, since we're doing write-only access, but we do
6244  // need to mark the DualView as modified on host.
6245 
6246  numPacketsPerLID.clear_sync_state (); // write-only access
6247  numPacketsPerLID.modify_host ();
6248  auto numPacketsPerLID_h = numPacketsPerLID.view_host ();
6249  Teuchos::ArrayView<size_t> numPacketsPerLID_av (numPacketsPerLID_h.data (),
6250  numPacketsPerLID_h.size ());
6251 
6252  // Invoke RowMatrix's legacy pack() interface, using above
6253  // Teuchos::Array* objects.
6254  try {
6255  srcRowMat->pack (exportLIDs_av, exports_a, numPacketsPerLID_av,
6256  constantNumPackets);
6257  }
6258  catch (std::exception& e) {
6259  lclBad = 1;
6260  msg << "Proc " << myRank << ": " << e.what () << std::endl;
6261  }
6262 
6263  // Allocate 'exports', and copy exports_a back into it.
6264  const size_t newAllocSize = static_cast<size_t> (exports_a.size ());
6265  if (static_cast<size_t> (exports.extent (0)) < newAllocSize) {
6266  const std::string oldLabel = exports.view_device().label ();
6267  const std::string newLabel = (oldLabel == "") ? "exports" : oldLabel;
6268  exports = exports_type (newLabel, newAllocSize);
6269  }
6270  // It's safe to assume that we're working on host anyway, so
6271  // just keep exports sync'd to host.
6272  // ignore current device contents
6273  exports.modify_host();
6274 
6275  auto exports_h = exports.view_host ();
6276  auto exports_h_sub = subview (exports_h, range_type (0, newAllocSize));
6277 
6278  // Kokkos::deep_copy needs a Kokkos::View input, so turn
6279  // exports_a into a nonowning Kokkos::View first before copying.
6280  typedef typename exports_type::t_host::execution_space HES;
6281  typedef Kokkos::Device<HES, HostSpace> host_device_type;
6282  Kokkos::View<const char*, host_device_type>
6283  exports_a_kv (exports_a.getRawPtr (), newAllocSize);
6284  // DEEP_COPY REVIEW - NOT TESTED
6285  Kokkos::deep_copy (exports_h_sub, exports_a_kv);
6286  }
6287 
6288  if (debug) {
6289  int gblBad = 0; // output argument; to be set below
6290  reduceAll<int, int> (comm, REDUCE_MAX, lclBad, outArg (gblBad));
6291  if (gblBad != 0) {
6292  Tpetra::Details::gathervPrint (std::cerr, msg.str (), comm);
6293  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6294  (true, std::logic_error, "packNew() or pack() threw an exception on "
6295  "one or more participating processes.");
6296  }
6297  }
6298  else {
6299  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6300  (lclBad != 0, std::logic_error, "packNew threw an exception on one "
6301  "or more participating processes. Here is this process' error "
6302  "message: " << msg.str ());
6303  }
6304 
6305  if (verbose) {
6306  std::ostringstream os;
6307  os << *prefix << "packAndPrepare: Done!" << endl
6308  << *prefix << " "
6309  << dualViewStatusToString (exportLIDs, "exportLIDs")
6310  << endl
6311  << *prefix << " "
6312  << dualViewStatusToString (exports, "exports")
6313  << endl
6314  << *prefix << " "
6315  << dualViewStatusToString (numPacketsPerLID, "numPacketsPerLID")
6316  << endl;
6317  std::cerr << os.str ();
6318  }
6319  }
6320 
6321  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
6322  size_t
6323  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
6324  packRow (char exports[],
6325  const size_t offset,
6326  const size_t numEnt,
6327  const GlobalOrdinal gidsIn[],
6328  const impl_scalar_type valsIn[],
6329  const size_t numBytesPerValue) const
6330  {
6331  using Kokkos::View;
6332  using Kokkos::subview;
6333  using Tpetra::Details::PackTraits;
6334  typedef LocalOrdinal LO;
6335  typedef GlobalOrdinal GO;
6336  typedef impl_scalar_type ST;
6337 
6338  if (numEnt == 0) {
6339  // Empty rows always take zero bytes, to ensure sparsity.
6340  return 0;
6341  }
6342 
6343  const GO gid = 0; // packValueCount wants this
6344  const LO numEntLO = static_cast<size_t> (numEnt);
6345 
6346  const size_t numEntBeg = offset;
6347  const size_t numEntLen = PackTraits<LO>::packValueCount (numEntLO);
6348  const size_t gidsBeg = numEntBeg + numEntLen;
6349  const size_t gidsLen = numEnt * PackTraits<GO>::packValueCount (gid);
6350  const size_t valsBeg = gidsBeg + gidsLen;
6351  const size_t valsLen = numEnt * numBytesPerValue;
6352 
6353  char* const numEntOut = exports + numEntBeg;
6354  char* const gidsOut = exports + gidsBeg;
6355  char* const valsOut = exports + valsBeg;
6356 
6357  size_t numBytesOut = 0;
6358  int errorCode = 0;
6359  numBytesOut += PackTraits<LO>::packValue (numEntOut, numEntLO);
6360 
6361  {
6362  Kokkos::pair<int, size_t> p;
6363  p = PackTraits<GO>::packArray (gidsOut, gidsIn, numEnt);
6364  errorCode += p.first;
6365  numBytesOut += p.second;
6366 
6367  p = PackTraits<ST>::packArray (valsOut, valsIn, numEnt);
6368  errorCode += p.first;
6369  numBytesOut += p.second;
6370  }
6371 
6372  const size_t expectedNumBytes = numEntLen + gidsLen + valsLen;
6373  TEUCHOS_TEST_FOR_EXCEPTION
6374  (numBytesOut != expectedNumBytes, std::logic_error, "packRow: "
6375  "numBytesOut = " << numBytesOut << " != expectedNumBytes = "
6376  << expectedNumBytes << ".");
6377  TEUCHOS_TEST_FOR_EXCEPTION
6378  (errorCode != 0, std::runtime_error, "packRow: "
6379  "PackTraits::packArray returned a nonzero error code");
6380 
6381  return numBytesOut;
6382  }
6383 
6384  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
6385  size_t
6386  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
6387  unpackRow (GlobalOrdinal gidsOut[],
6388  impl_scalar_type valsOut[],
6389  const char imports[],
6390  const size_t offset,
6391  const size_t numBytes,
6392  const size_t numEnt,
6393  const size_t numBytesPerValue)
6394  {
6395  using Kokkos::View;
6396  using Kokkos::subview;
6397  using Tpetra::Details::PackTraits;
6398  typedef LocalOrdinal LO;
6399  typedef GlobalOrdinal GO;
6400  typedef impl_scalar_type ST;
6401 
6402  Details::ProfilingRegion region_upack_row(
6403  "Tpetra::CrsMatrix::unpackRow",
6404  "Import/Export"
6405  );
6406 
6407  if (numBytes == 0) {
6408  // Rows with zero bytes should always have zero entries.
6409  if (numEnt != 0) {
6410  const int myRank = this->getMap ()->getComm ()->getRank ();
6411  TEUCHOS_TEST_FOR_EXCEPTION
6412  (true, std::logic_error, "(Proc " << myRank << ") CrsMatrix::"
6413  "unpackRow: The number of bytes to unpack numBytes=0, but the "
6414  "number of entries to unpack (as reported by numPacketsPerLID) "
6415  "for this row numEnt=" << numEnt << " != 0.");
6416  }
6417  return 0;
6418  }
6419 
6420  if (numEnt == 0 && numBytes != 0) {
6421  const int myRank = this->getMap ()->getComm ()->getRank ();
6422  TEUCHOS_TEST_FOR_EXCEPTION
6423  (true, std::logic_error, "(Proc " << myRank << ") CrsMatrix::"
6424  "unpackRow: The number of entries to unpack (as reported by "
6425  "numPacketsPerLID) numEnt=0, but the number of bytes to unpack "
6426  "numBytes=" << numBytes << " != 0.");
6427  }
6428 
6429  const GO gid = 0; // packValueCount wants this
6430  const LO lid = 0; // packValueCount wants this
6431 
6432  const size_t numEntBeg = offset;
6433  const size_t numEntLen = PackTraits<LO>::packValueCount (lid);
6434  const size_t gidsBeg = numEntBeg + numEntLen;
6435  const size_t gidsLen = numEnt * PackTraits<GO>::packValueCount (gid);
6436  const size_t valsBeg = gidsBeg + gidsLen;
6437  const size_t valsLen = numEnt * numBytesPerValue;
6438 
6439  const char* const numEntIn = imports + numEntBeg;
6440  const char* const gidsIn = imports + gidsBeg;
6441  const char* const valsIn = imports + valsBeg;
6442 
6443  size_t numBytesOut = 0;
6444  int errorCode = 0;
6445  LO numEntOut;
6446  numBytesOut += PackTraits<LO>::unpackValue (numEntOut, numEntIn);
6447  if (static_cast<size_t> (numEntOut) != numEnt ||
6448  numEntOut == static_cast<LO> (0)) {
6449  const int myRank = this->getMap ()->getComm ()->getRank ();
6450  std::ostringstream os;
6451  os << "(Proc " << myRank << ") CrsMatrix::unpackRow: ";
6452  bool firstErrorCondition = false;
6453  if (static_cast<size_t> (numEntOut) != numEnt) {
6454  os << "Number of entries from numPacketsPerLID numEnt=" << numEnt
6455  << " does not equal number of entries unpacked from imports "
6456  "buffer numEntOut=" << numEntOut << ".";
6457  firstErrorCondition = true;
6458  }
6459  if (numEntOut == static_cast<LO> (0)) {
6460  if (firstErrorCondition) {
6461  os << " Also, ";
6462  }
6463  os << "Number of entries unpacked from imports buffer numEntOut=0, "
6464  "but number of bytes to unpack for this row numBytes=" << numBytes
6465  << " != 0. This should never happen, since packRow should only "
6466  "ever pack rows with a nonzero number of entries. In this case, "
6467  "the number of entries from numPacketsPerLID is numEnt=" << numEnt
6468  << ".";
6469  }
6470  TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error, os.str ());
6471  }
6472 
6473  {
6474  Kokkos::pair<int, size_t> p;
6475  p = PackTraits<GO>::unpackArray (gidsOut, gidsIn, numEnt);
6476  errorCode += p.first;
6477  numBytesOut += p.second;
6478 
6479  p = PackTraits<ST>::unpackArray (valsOut, valsIn, numEnt);
6480  errorCode += p.first;
6481  numBytesOut += p.second;
6482  }
6483 
6484  TEUCHOS_TEST_FOR_EXCEPTION
6485  (numBytesOut != numBytes, std::logic_error, "unpackRow: numBytesOut = "
6486  << numBytesOut << " != numBytes = " << numBytes << ".");
6487 
6488  const size_t expectedNumBytes = numEntLen + gidsLen + valsLen;
6489  TEUCHOS_TEST_FOR_EXCEPTION
6490  (numBytesOut != expectedNumBytes, std::logic_error, "unpackRow: "
6491  "numBytesOut = " << numBytesOut << " != expectedNumBytes = "
6492  << expectedNumBytes << ".");
6493 
6494  TEUCHOS_TEST_FOR_EXCEPTION
6495  (errorCode != 0, std::runtime_error, "unpackRow: "
6496  "PackTraits::unpackArray returned a nonzero error code");
6497 
6498  return numBytesOut;
6499  }
6500 
6501  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
6502  void
6503  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
6504  allocatePackSpaceNew (Kokkos::DualView<char*, buffer_device_type>& exports,
6505  size_t& totalNumEntries,
6506  const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& exportLIDs) const
6507  {
6508  using Details::Behavior;
6509  using Details::dualViewStatusToString;
6510  using std::endl;
6511  typedef impl_scalar_type IST;
6512  typedef LocalOrdinal LO;
6513  typedef GlobalOrdinal GO;
6514  //const char tfecfFuncName[] = "allocatePackSpaceNew: ";
6515 
6516  // mfh 18 Oct 2017: Set TPETRA_VERBOSE to true for copious debug
6517  // output to std::cerr on every MPI process. This is unwise for
6518  // runs with large numbers of MPI processes.
6519  const bool verbose = Behavior::verbose("CrsMatrix");
6520  std::unique_ptr<std::string> prefix;
6521  if (verbose) {
6522  prefix = this->createPrefix("CrsMatrix", "allocatePackSpaceNew");
6523  std::ostringstream os;
6524  os << *prefix << "Before:"
6525  << endl
6526  << *prefix << " "
6527  << dualViewStatusToString (exports, "exports")
6528  << endl
6529  << *prefix << " "
6530  << dualViewStatusToString (exportLIDs, "exportLIDs")
6531  << endl;
6532  std::cerr << os.str ();
6533  }
6534 
6535  // The number of export LIDs must fit in LocalOrdinal, assuming
6536  // that the LIDs are distinct and valid on the calling process.
6537  const LO numExportLIDs = static_cast<LO> (exportLIDs.extent (0));
6538 
6539  TEUCHOS_ASSERT( ! exportLIDs.need_sync_host () );
6540  auto exportLIDs_h = exportLIDs.view_host ();
6541 
6542  // Count the total number of matrix entries to send.
6543  totalNumEntries = 0;
6544  for (LO i = 0; i < numExportLIDs; ++i) {
6545  const LO lclRow = exportLIDs_h[i];
6546  size_t curNumEntries = this->getNumEntriesInLocalRow (lclRow);
6547  // FIXME (mfh 25 Jan 2015) We should actually report invalid row
6548  // indices as an error. Just consider them nonowned for now.
6549  if (curNumEntries == Teuchos::OrdinalTraits<size_t>::invalid ()) {
6550  curNumEntries = 0;
6551  }
6552  totalNumEntries += curNumEntries;
6553  }
6554 
6555  // FIXME (mfh 24 Feb 2013, 24 Mar 2017) This code is only correct
6556  // if sizeof(IST) is a meaningful representation of the amount of
6557  // data in a Scalar instance. (LO and GO are always built-in
6558  // integer types.)
6559  //
6560  // Allocate the exports array. It does NOT need padding for
6561  // alignment, since we use memcpy to write to / read from send /
6562  // receive buffers.
6563  const size_t allocSize =
6564  static_cast<size_t> (numExportLIDs) * sizeof (LO) +
6565  totalNumEntries * (sizeof (IST) + sizeof (GO));
6566  if (static_cast<size_t> (exports.extent (0)) < allocSize) {
6567  using exports_type = Kokkos::DualView<char*, buffer_device_type>;
6568 
6569  const std::string oldLabel = exports.view_device().label ();
6570  const std::string newLabel = (oldLabel == "") ? "exports" : oldLabel;
6571  exports = exports_type (newLabel, allocSize);
6572  }
6573 
6574  if (verbose) {
6575  std::ostringstream os;
6576  os << *prefix << "After:"
6577  << endl
6578  << *prefix << " "
6579  << dualViewStatusToString (exports, "exports")
6580  << endl
6581  << *prefix << " "
6582  << dualViewStatusToString (exportLIDs, "exportLIDs")
6583  << endl;
6584  std::cerr << os.str ();
6585  }
6586  }
6587 
6588  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
6589  void
6590  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
6591  packNew (const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& exportLIDs,
6592  Kokkos::DualView<char*, buffer_device_type>& exports,
6593  const Kokkos::DualView<size_t*, buffer_device_type>& numPacketsPerLID,
6594  size_t& constantNumPackets) const
6595  {
6596  // The call to packNew in packAndPrepare catches and handles any exceptions.
6597  Details::ProfilingRegion region_pack_new("Tpetra::CrsMatrix::packNew", "Import/Export");
6598  if (this->isStaticGraph ()) {
6599  using ::Tpetra::Details::packCrsMatrixNew;
6600  packCrsMatrixNew (*this, exports, numPacketsPerLID, exportLIDs,
6601  constantNumPackets);
6602  }
6603  else {
6604  this->packNonStaticNew (exportLIDs, exports, numPacketsPerLID,
6605  constantNumPackets);
6606  }
6607  }
6608 
6609  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
6610  void
6611  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
6612  packNonStaticNew (const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& exportLIDs,
6613  Kokkos::DualView<char*, buffer_device_type>& exports,
6614  const Kokkos::DualView<size_t*, buffer_device_type>& numPacketsPerLID,
6615  size_t& constantNumPackets) const
6616  {
6617  using Details::Behavior;
6618  using Details::dualViewStatusToString;
6619  using Details::PackTraits;
6620  using Details::create_mirror_view_from_raw_host_array;
6621  using Kokkos::View;
6622  using std::endl;
6623  using LO = LocalOrdinal;
6624  using GO = GlobalOrdinal;
6625  using ST = impl_scalar_type;
6626  const char tfecfFuncName[] = "packNonStaticNew: ";
6627 
6628  const bool verbose = Behavior::verbose("CrsMatrix");
6629  std::unique_ptr<std::string> prefix;
6630  if (verbose) {
6631  prefix = this->createPrefix("CrsMatrix", "packNonStaticNew");
6632  std::ostringstream os;
6633  os << *prefix << "Start" << endl;
6634  std::cerr << os.str ();
6635  }
6636 
6637  const size_t numExportLIDs = static_cast<size_t> (exportLIDs.extent (0));
6638  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6639  (numExportLIDs != static_cast<size_t> (numPacketsPerLID.extent (0)),
6640  std::invalid_argument, "exportLIDs.size() = " << numExportLIDs
6641  << " != numPacketsPerLID.size() = " << numPacketsPerLID.extent (0)
6642  << ".");
6643 
6644  // Setting this to zero tells the caller to expect a possibly
6645  // different ("nonconstant") number of packets per local index
6646  // (i.e., a possibly different number of entries per row).
6647  constantNumPackets = 0;
6648 
6649  // The pack buffer 'exports' enters this method possibly
6650  // unallocated. Do the first two parts of "Count, allocate, fill,
6651  // compute."
6652  size_t totalNumEntries = 0;
6653  this->allocatePackSpaceNew (exports, totalNumEntries, exportLIDs);
6654  const size_t bufSize = static_cast<size_t> (exports.extent (0));
6655 
6656  // Write-only host access
6657  exports.clear_sync_state();
6658  exports.modify_host();
6659  auto exports_h = exports.view_host ();
6660  if (verbose) {
6661  std::ostringstream os;
6662  os << *prefix << "After marking exports as modified on host, "
6663  << dualViewStatusToString (exports, "exports") << endl;
6664  std::cerr << os.str ();
6665  }
6666 
6667  // Read-only host access
6668  auto exportLIDs_h = exportLIDs.view_host ();
6669 
6670  // Write-only host access
6671  const_cast<Kokkos::DualView<size_t*, buffer_device_type>*>(&numPacketsPerLID)->clear_sync_state();
6672  const_cast<Kokkos::DualView<size_t*, buffer_device_type>*>(&numPacketsPerLID)->modify_host();
6673  auto numPacketsPerLID_h = numPacketsPerLID.view_host ();
6674 
6675  // Compute the number of "packets" (in this case, bytes) per
6676  // export LID (in this case, local index of the row to send), and
6677  // actually pack the data.
6678  auto maxRowNumEnt = this->getLocalMaxNumRowEntries();
6679 
6680 
6681  // Temporary buffer for global column indices.
6682  typename global_inds_host_view_type::non_const_type gidsIn_k;
6683  if (this->isLocallyIndexed()) { // Need storage for Global IDs
6684  gidsIn_k =
6685  typename global_inds_host_view_type::non_const_type("packGids",
6686  maxRowNumEnt);
6687  }
6688 
6689  size_t offset = 0; // current index into 'exports' array.
6690  for (size_t i = 0; i < numExportLIDs; ++i) {
6691  const LO lclRow = exportLIDs_h[i];
6692 
6693  size_t numBytes = 0;
6694  size_t numEnt = this->getNumEntriesInLocalRow (lclRow);
6695 
6696  // Only pack this row's data if it has a nonzero number of
6697  // entries. We can do this because receiving processes get the
6698  // number of packets, and will know that zero packets means zero
6699  // entries.
6700  if (numEnt == 0) {
6701  numPacketsPerLID_h[i] = 0;
6702  continue;
6703  }
6704 
6705  if (this->isLocallyIndexed ()) {
6706  typename global_inds_host_view_type::non_const_type gidsIn;
6707  values_host_view_type valsIn;
6708  // If the matrix is locally indexed on the calling process, we
6709  // have to use its column Map (which it _must_ have in this
6710  // case) to convert to global indices.
6711  local_inds_host_view_type lidsIn;
6712  this->getLocalRowView (lclRow, lidsIn, valsIn);
6713  const map_type& colMap = * (this->getColMap ());
6714  for (size_t k = 0; k < numEnt; ++k) {
6715  gidsIn_k[k] = colMap.getGlobalElement (lidsIn[k]);
6716  }
6717  gidsIn = Kokkos::subview(gidsIn_k, Kokkos::make_pair(GO(0),GO(numEnt)));
6718 
6719  const size_t numBytesPerValue =
6720  PackTraits<ST>::packValueCount (valsIn[0]);
6721  numBytes = this->packRow (exports_h.data (), offset, numEnt,
6722  gidsIn.data (), valsIn.data (),
6723  numBytesPerValue);
6724  }
6725  else if (this->isGloballyIndexed ()) {
6726  global_inds_host_view_type gidsIn;
6727  values_host_view_type valsIn;
6728  // If the matrix is globally indexed on the calling process,
6729  // then we can use the column indices directly. However, we
6730  // have to get the global row index. The calling process must
6731  // have a row Map, since otherwise it shouldn't be participating
6732  // in packing operations.
6733  const map_type& rowMap = * (this->getRowMap ());
6734  const GO gblRow = rowMap.getGlobalElement (lclRow);
6735  this->getGlobalRowView (gblRow, gidsIn, valsIn);
6736 
6737  const size_t numBytesPerValue =
6738  PackTraits<ST>::packValueCount (valsIn[0]);
6739  numBytes = this->packRow (exports_h.data (), offset, numEnt,
6740  gidsIn.data (), valsIn.data (),
6741  numBytesPerValue);
6742  }
6743  // mfh 11 Sep 2017: Currently, if the matrix is neither globally
6744  // nor locally indexed, then it has no entries. Therefore,
6745  // there is nothing to pack. No worries!
6746 
6747  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6748  (offset > bufSize || offset + numBytes > bufSize, std::logic_error,
6749  "First invalid offset into 'exports' pack buffer at index i = " << i
6750  << ". exportLIDs_h[i]: " << exportLIDs_h[i] << ", bufSize: " <<
6751  bufSize << ", offset: " << offset << ", numBytes: " << numBytes <<
6752  ".");
6753  // numPacketsPerLID_h[i] is the number of "packets" in the
6754  // current local row i. Packet=char (really "byte") so use the
6755  // number of bytes of the packed data for that row.
6756  numPacketsPerLID_h[i] = numBytes;
6757  offset += numBytes;
6758  }
6759 
6760  if (verbose) {
6761  std::ostringstream os;
6762  os << *prefix << "Tpetra::CrsMatrix::packNonStaticNew: After:" << endl
6763  << *prefix << " "
6764  << dualViewStatusToString (exports, "exports")
6765  << endl
6766  << *prefix << " "
6767  << dualViewStatusToString (exportLIDs, "exportLIDs")
6768  << endl;
6769  std::cerr << os.str ();
6770  }
6771  }
6772 
6773  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
6774  LocalOrdinal
6775  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
6776  combineGlobalValuesRaw(const LocalOrdinal lclRow,
6777  const LocalOrdinal numEnt,
6778  const impl_scalar_type vals[],
6779  const GlobalOrdinal cols[],
6780  const Tpetra::CombineMode combMode,
6781  const char* const prefix,
6782  const bool debug,
6783  const bool verbose)
6784  {
6785  using GO = GlobalOrdinal;
6786 
6787  // mfh 23 Mar 2017: This branch is not thread safe in a debug
6788  // build, due to use of Teuchos::ArrayView; see #229.
6789  const GO gblRow = myGraph_->rowMap_->getGlobalElement(lclRow);
6790  Teuchos::ArrayView<const GO> cols_av
6791  (numEnt == 0 ? nullptr : cols, numEnt);
6792  Teuchos::ArrayView<const Scalar> vals_av
6793  (numEnt == 0 ? nullptr : reinterpret_cast<const Scalar*> (vals), numEnt);
6794 
6795  // FIXME (mfh 23 Mar 2017) This is a work-around for less common
6796  // combine modes. combineGlobalValues throws on error; it does
6797  // not return an error code. Thus, if it returns, it succeeded.
6798  combineGlobalValues(gblRow, cols_av, vals_av, combMode,
6799  prefix, debug, verbose);
6800  return numEnt;
6801  }
6802 
6803  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
6804  void
6805  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
6806  combineGlobalValues(
6807  const GlobalOrdinal globalRowIndex,
6808  const Teuchos::ArrayView<const GlobalOrdinal>& columnIndices,
6809  const Teuchos::ArrayView<const Scalar>& values,
6810  const Tpetra::CombineMode combineMode,
6811  const char* const prefix,
6812  const bool debug,
6813  const bool verbose)
6814  {
6815  const char tfecfFuncName[] = "combineGlobalValues: ";
6816 
6817  if (isStaticGraph ()) {
6818  // INSERT doesn't make sense for a static graph, since you
6819  // aren't allowed to change the structure of the graph.
6820  // However, all the other combine modes work.
6821  if (combineMode == ADD) {
6822  sumIntoGlobalValues (globalRowIndex, columnIndices, values);
6823  }
6824  else if (combineMode == REPLACE) {
6825  replaceGlobalValues (globalRowIndex, columnIndices, values);
6826  }
6827  else if (combineMode == ABSMAX) {
6828  using ::Tpetra::Details::AbsMax;
6829  AbsMax<Scalar> f;
6830  this->template transformGlobalValues<AbsMax<Scalar> > (globalRowIndex,
6831  columnIndices,
6832  values, f);
6833  }
6834  else if (combineMode == INSERT) {
6835  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6836  (isStaticGraph() && combineMode == INSERT,
6837  std::invalid_argument, "INSERT combine mode is forbidden "
6838  "if the matrix has a static (const) graph (i.e., was "
6839  "constructed with the CrsMatrix constructor that takes a "
6840  "const CrsGraph pointer).");
6841  }
6842  else {
6843  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6844  (true, std::logic_error, "Invalid combine mode; should "
6845  "never get here! "
6846  "Please report this bug to the Tpetra developers.");
6847  }
6848  }
6849  else { // The matrix has a dynamic graph.
6850  if (combineMode == ADD || combineMode == INSERT) {
6851  // For a dynamic graph, all incoming column indices are
6852  // inserted into the target graph. Duplicate indices will
6853  // have their values summed. In this context, ADD and INSERT
6854  // are equivalent. We need to call insertGlobalValues()
6855  // anyway if the column indices don't yet exist in this row,
6856  // so we just call insertGlobalValues() for both cases.
6857  insertGlobalValuesFilteredChecked(globalRowIndex,
6858  columnIndices, values, prefix, debug, verbose);
6859  }
6860  // FIXME (mfh 14 Mar 2012):
6861  //
6862  // Implementing ABSMAX or REPLACE for a dynamic graph would
6863  // require modifying assembly to attach a possibly different
6864  // combine mode to each inserted (i, j, A_ij) entry. For
6865  // example, consider two different Export operations to the same
6866  // target CrsMatrix, the first with ABSMAX combine mode and the
6867  // second with REPLACE. This isn't a common use case, so we
6868  // won't mess with it for now.
6869  else if (combineMode == ABSMAX) {
6870  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
6871  ! isStaticGraph () && combineMode == ABSMAX, std::logic_error,
6872  "ABSMAX combine mode when the matrix has a dynamic graph is not yet "
6873  "implemented.");
6874  }
6875  else if (combineMode == REPLACE) {
6876  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
6877  ! isStaticGraph () && combineMode == REPLACE, std::logic_error,
6878  "REPLACE combine mode when the matrix has a dynamic graph is not yet "
6879  "implemented.");
6880  }
6881  else {
6882  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
6883  true, std::logic_error, "Should never get here! Please report this "
6884  "bug to the Tpetra developers.");
6885  }
6886  }
6887  }
6888 
6889  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
6890  void
6891  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
6892  unpackAndCombine
6893  (const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& importLIDs,
6894  Kokkos::DualView<char*, buffer_device_type> imports,
6895  Kokkos::DualView<size_t*, buffer_device_type> numPacketsPerLID,
6896  const size_t constantNumPackets,
6897  const CombineMode combineMode)
6898  {
6899  using Details::Behavior;
6900  using Details::dualViewStatusToString;
6901  using Details::ProfilingRegion;
6902  using std::endl;
6903  const char tfecfFuncName[] = "unpackAndCombine: ";
6904  ProfilingRegion regionUAC ("Tpetra::CrsMatrix::unpackAndCombine");
6905 
6906  const bool debug = Behavior::debug("CrsMatrix");
6907  const bool verbose = Behavior::verbose("CrsMatrix");
6908  constexpr int numValidModes = 5;
6909  const CombineMode validModes[numValidModes] =
6910  {ADD, REPLACE, ABSMAX, INSERT, ZERO};
6911  const char* validModeNames[numValidModes] =
6912  {"ADD", "REPLACE", "ABSMAX", "INSERT", "ZERO"};
6913 
6914  std::unique_ptr<std::string> prefix;
6915  if (verbose) {
6916  prefix = this->createPrefix("CrsMatrix", "unpackAndCombine");
6917  std::ostringstream os;
6918  os << *prefix << "Start:" << endl
6919  << *prefix << " "
6920  << dualViewStatusToString (importLIDs, "importLIDs")
6921  << endl
6922  << *prefix << " "
6923  << dualViewStatusToString (imports, "imports")
6924  << endl
6925  << *prefix << " "
6926  << dualViewStatusToString (numPacketsPerLID, "numPacketsPerLID")
6927  << endl
6928  << *prefix << " constantNumPackets: " << constantNumPackets
6929  << endl
6930  << *prefix << " combineMode: " << combineModeToString (combineMode)
6931  << endl;
6932  std::cerr << os.str ();
6933  }
6934 
6935  if (debug) {
6936  if (std::find (validModes, validModes+numValidModes, combineMode) ==
6937  validModes+numValidModes) {
6938  std::ostringstream os;
6939  os << "Invalid combine mode. Valid modes are {";
6940  for (int k = 0; k < numValidModes; ++k) {
6941  os << validModeNames[k];
6942  if (k < numValidModes - 1) {
6943  os << ", ";
6944  }
6945  }
6946  os << "}.";
6947  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6948  (true, std::invalid_argument, os.str ());
6949  }
6950  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6951  (importLIDs.extent(0) != numPacketsPerLID.extent(0),
6952  std::invalid_argument, "importLIDs.extent(0)="
6953  << importLIDs.extent(0)
6954  << " != numPacketsPerLID.extent(0)="
6955  << numPacketsPerLID.extent(0) << ".");
6956  }
6957 
6958  if (combineMode == ZERO) {
6959  return; // nothing to do
6960  }
6961 
6962  if (debug) {
6963  using Teuchos::reduceAll;
6964  std::unique_ptr<std::ostringstream> msg (new std::ostringstream ());
6965  int lclBad = 0;
6966  try {
6967  unpackAndCombineImpl(importLIDs, imports, numPacketsPerLID,
6968  constantNumPackets, combineMode,
6969  verbose);
6970  } catch (std::exception& e) {
6971  lclBad = 1;
6972  *msg << e.what ();
6973  }
6974  int gblBad = 0;
6975  const Teuchos::Comm<int>& comm = * (this->getComm ());
6976  reduceAll<int, int> (comm, Teuchos::REDUCE_MAX,
6977  lclBad, Teuchos::outArg (gblBad));
6978  if (gblBad != 0) {
6979  // mfh 22 Oct 2017: 'prefix' might be null, since it is only
6980  // initialized in a debug build. Thus, we get the process
6981  // rank again here. This is an error message, so the small
6982  // run-time cost doesn't matter. See #1887.
6983  std::ostringstream os;
6984  os << "Proc " << comm.getRank () << ": " << msg->str () << endl;
6985  msg = std::unique_ptr<std::ostringstream> (new std::ostringstream ());
6986  ::Tpetra::Details::gathervPrint (*msg, os.str (), comm);
6987  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
6988  (true, std::logic_error, std::endl << "unpackAndCombineImpl "
6989  "threw an exception on one or more participating processes: "
6990  << endl << msg->str ());
6991  }
6992  }
6993  else {
6994  unpackAndCombineImpl(importLIDs, imports, numPacketsPerLID,
6995  constantNumPackets, combineMode,
6996  verbose);
6997  }
6998 
6999  if (verbose) {
7000  std::ostringstream os;
7001  os << *prefix << "Done!" << endl
7002  << *prefix << " "
7003  << dualViewStatusToString (importLIDs, "importLIDs")
7004  << endl
7005  << *prefix << " "
7006  << dualViewStatusToString (imports, "imports")
7007  << endl
7008  << *prefix << " "
7009  << dualViewStatusToString (numPacketsPerLID, "numPacketsPerLID")
7010  << endl;
7011  std::cerr << os.str ();
7012  }
7013  }
7014 
7015  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
7016  void
7017  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
7018  unpackAndCombineImpl(
7019  const Kokkos::DualView<const local_ordinal_type*,
7020  buffer_device_type>& importLIDs,
7021  Kokkos::DualView<char*, buffer_device_type> imports,
7022  Kokkos::DualView<size_t*, buffer_device_type> numPacketsPerLID,
7023  const size_t constantNumPackets,
7024  const CombineMode combineMode,
7025  const bool verbose)
7026  {
7027  Details::ProfilingRegion region_unpack_and_combine_impl(
7028  "Tpetra::CrsMatrix::unpackAndCombineImpl",
7029  "Import/Export"
7030  );
7031  using std::endl;
7032  const char tfecfFuncName[] = "unpackAndCombineImpl";
7033  std::unique_ptr<std::string> prefix;
7034  if (verbose) {
7035  prefix = this->createPrefix("CrsMatrix", tfecfFuncName);
7036  std::ostringstream os;
7037  os << *prefix << "isStaticGraph(): "
7038  << (isStaticGraph() ? "true" : "false")
7039  << ", importLIDs.extent(0): "
7040  << importLIDs.extent(0)
7041  << ", imports.extent(0): "
7042  << imports.extent(0)
7043  << ", numPacketsPerLID.extent(0): "
7044  << numPacketsPerLID.extent(0)
7045  << endl;
7046  std::cerr << os.str();
7047  }
7048 
7049  if (isStaticGraph ()) {
7050  using Details::unpackCrsMatrixAndCombineNew;
7051  unpackCrsMatrixAndCombineNew(*this, imports, numPacketsPerLID,
7052  importLIDs, constantNumPackets,
7053  combineMode);
7054  }
7055  else {
7056  {
7057  using padding_type = typename crs_graph_type::padding_type;
7058  std::unique_ptr<padding_type> padding;
7059  try {
7060  padding = myGraph_->computePaddingForCrsMatrixUnpack(
7061  importLIDs, imports, numPacketsPerLID, verbose);
7062  }
7063  catch (std::exception& e) {
7064  const auto rowMap = getRowMap();
7065  const auto comm = rowMap.is_null() ? Teuchos::null :
7066  rowMap->getComm();
7067  const int myRank = comm.is_null() ? -1 : comm->getRank();
7068  TEUCHOS_TEST_FOR_EXCEPTION
7069  (true, std::runtime_error, "Proc " << myRank << ": "
7070  "Tpetra::CrsGraph::computePaddingForCrsMatrixUnpack "
7071  "threw an exception: " << e.what());
7072  }
7073  if (verbose) {
7074  std::ostringstream os;
7075  os << *prefix << "Call applyCrsPadding" << endl;
7076  std::cerr << os.str();
7077  }
7078  applyCrsPadding(*padding, verbose);
7079  }
7080  if (verbose) {
7081  std::ostringstream os;
7082  os << *prefix << "Call unpackAndCombineImplNonStatic" << endl;
7083  std::cerr << os.str();
7084  }
7085  unpackAndCombineImplNonStatic(importLIDs, imports,
7086  numPacketsPerLID,
7087  constantNumPackets,
7088  combineMode);
7089  }
7090 
7091  if (verbose) {
7092  std::ostringstream os;
7093  os << *prefix << "Done" << endl;
7094  std::cerr << os.str();
7095  }
7096  }
7097 
7098  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
7099  void
7100  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
7101  unpackAndCombineImplNonStatic(
7102  const Kokkos::DualView<const local_ordinal_type*,
7103  buffer_device_type>& importLIDs,
7104  Kokkos::DualView<char*, buffer_device_type> imports,
7105  Kokkos::DualView<size_t*, buffer_device_type> numPacketsPerLID,
7106  const size_t constantNumPackets,
7107  const CombineMode combineMode)
7108  {
7109  using Kokkos::View;
7110  using Kokkos::subview;
7111  using Kokkos::MemoryUnmanaged;
7112  using Details::Behavior;
7113  using Details::castAwayConstDualView;
7114  using Details::create_mirror_view_from_raw_host_array;
7115  using Details::PackTraits;
7116  using Details::ScalarViewTraits;
7117  using std::endl;
7118  using LO = LocalOrdinal;
7119  using GO = GlobalOrdinal;
7120  using ST = impl_scalar_type;
7121  using size_type = typename Teuchos::ArrayView<LO>::size_type;
7122  using HES =
7123  typename View<int*, device_type>::HostMirror::execution_space;
7124  using pair_type = std::pair<typename View<int*, HES>::size_type,
7125  typename View<int*, HES>::size_type>;
7126  using gids_out_type = View<GO*, HES, MemoryUnmanaged>;
7127  using vals_out_type = View<ST*, HES, MemoryUnmanaged>;
7128  const char tfecfFuncName[] = "unpackAndCombineImplNonStatic";
7129 
7130  const bool debug = Behavior::debug("CrsMatrix");
7131  const bool verbose = Behavior::verbose("CrsMatrix");
7132  std::unique_ptr<std::string> prefix;
7133  if (verbose) {
7134  prefix = this->createPrefix("CrsMatrix", tfecfFuncName);
7135  std::ostringstream os;
7136  os << *prefix << endl; // we've already printed DualViews' statuses
7137  std::cerr << os.str ();
7138  }
7139  const char* const prefix_raw =
7140  verbose ? prefix.get()->c_str() : nullptr;
7141 
7142  const size_type numImportLIDs = importLIDs.extent (0);
7143  if (combineMode == ZERO || numImportLIDs == 0) {
7144  return; // nothing to do; no need to combine entries
7145  }
7146 
7147  Details::ProfilingRegion region_unpack_and_combine_impl_non_static(
7148  "Tpetra::CrsMatrix::unpackAndCombineImplNonStatic",
7149  "Import/Export"
7150  );
7151 
7152  // We're unpacking on host. This is read-only host access.
7153  if (imports.need_sync_host()) {
7154  imports.sync_host ();
7155  }
7156  auto imports_h = imports.view_host();
7157 
7158  // Read-only host access.
7159  if (numPacketsPerLID.need_sync_host()) {
7160  numPacketsPerLID.sync_host ();
7161  }
7162  auto numPacketsPerLID_h = numPacketsPerLID.view_host();
7163 
7164  TEUCHOS_ASSERT( ! importLIDs.need_sync_host() );
7165  auto importLIDs_h = importLIDs.view_host();
7166 
7167  size_t numBytesPerValue;
7168  {
7169  // FIXME (mfh 17 Feb 2015, tjf 2 Aug 2017) What do I do about Scalar types
7170  // with run-time size? We already assume that all entries in both the
7171  // source and target matrices have the same size. If the calling process
7172  // owns at least one entry in either matrix, we can use that entry to set
7173  // the size. However, it is possible that the calling process owns no
7174  // entries. In that case, we're in trouble. One way to fix this would be
7175  // for each row's data to contain the run-time size. This is only
7176  // necessary if the size is not a compile-time constant.
7177  Scalar val;
7178  numBytesPerValue = PackTraits<ST>::packValueCount (val);
7179  }
7180 
7181  // Determine the maximum number of entries in any one row
7182  size_t offset = 0;
7183  size_t maxRowNumEnt = 0;
7184  for (size_type i = 0; i < numImportLIDs; ++i) {
7185  const size_t numBytes = numPacketsPerLID_h[i];
7186  if (numBytes == 0) {
7187  continue; // empty buffer for that row means that the row is empty
7188  }
7189  // We need to unpack a nonzero number of entries for this row.
7190  if (debug) {
7191  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
7192  (offset + numBytes > size_t(imports_h.extent (0)),
7193  std::logic_error, ": At local row index importLIDs_h[i="
7194  << i << "]=" << importLIDs_h[i] << ", offset (=" << offset
7195  << ") + numBytes (=" << numBytes << ") > "
7196  "imports_h.extent(0)=" << imports_h.extent (0) << ".");
7197  }
7198  LO numEntLO = 0;
7199 
7200  if (debug) {
7201  const size_t theNumBytes =
7202  PackTraits<LO>::packValueCount (numEntLO);
7203  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
7204  (theNumBytes > numBytes, std::logic_error, ": theNumBytes="
7205  << theNumBytes << " > numBytes = " << numBytes << ".");
7206  }
7207  const char* const inBuf = imports_h.data () + offset;
7208  const size_t actualNumBytes =
7209  PackTraits<LO>::unpackValue (numEntLO, inBuf);
7210 
7211  if (debug) {
7212  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
7213  (actualNumBytes > numBytes, std::logic_error, ": At i=" << i
7214  << ", actualNumBytes=" << actualNumBytes
7215  << " > numBytes=" << numBytes << ".");
7216  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
7217  (numEntLO == 0, std::logic_error, ": At local row index "
7218  "importLIDs_h[i=" << i << "]=" << importLIDs_h[i] << ", "
7219  "the number of entries read from the packed data is "
7220  "numEntLO=" << numEntLO << ", but numBytes=" << numBytes
7221  << " != 0.");
7222  }
7223 
7224  maxRowNumEnt = std::max(size_t(numEntLO), maxRowNumEnt);
7225  offset += numBytes;
7226  }
7227 
7228  // Temporary space to cache incoming global column indices and
7229  // values. Column indices come in as global indices, in case the
7230  // source object's column Map differs from the target object's
7231  // (this's) column Map.
7232  View<GO*, HES> gblColInds;
7233  View<LO*, HES> lclColInds;
7234  View<ST*, HES> vals;
7235  {
7236  GO gid = 0;
7237  LO lid = 0;
7238  // FIXME (mfh 17 Feb 2015, tjf 2 Aug 2017) What do I do about Scalar types
7239  // with run-time size? We already assume that all entries in both the
7240  // source and target matrices have the same size. If the calling process
7241  // owns at least one entry in either matrix, we can use that entry to set
7242  // the size. However, it is possible that the calling process owns no
7243  // entries. In that case, we're in trouble. One way to fix this would be
7244  // for each row's data to contain the run-time size. This is only
7245  // necessary if the size is not a compile-time constant.
7246  Scalar val;
7247  gblColInds = ScalarViewTraits<GO, HES>::allocateArray(
7248  gid, maxRowNumEnt, "gids");
7249  lclColInds = ScalarViewTraits<LO, HES>::allocateArray(
7250  lid, maxRowNumEnt, "lids");
7251  vals = ScalarViewTraits<ST, HES>::allocateArray(
7252  val, maxRowNumEnt, "vals");
7253  }
7254 
7255  offset = 0;
7256  for (size_type i = 0; i < numImportLIDs; ++i) {
7257  const size_t numBytes = numPacketsPerLID_h[i];
7258  if (numBytes == 0) {
7259  continue; // empty buffer for that row means that the row is empty
7260  }
7261  LO numEntLO = 0;
7262  const char* const inBuf = imports_h.data () + offset;
7263  (void) PackTraits<LO>::unpackValue (numEntLO, inBuf);
7264 
7265  const size_t numEnt = static_cast<size_t>(numEntLO);;
7266  const LO lclRow = importLIDs_h[i];
7267 
7268  gids_out_type gidsOut = subview (gblColInds, pair_type (0, numEnt));
7269  vals_out_type valsOut = subview (vals, pair_type (0, numEnt));
7270 
7271  const size_t numBytesOut =
7272  unpackRow (gidsOut.data (), valsOut.data (), imports_h.data (),
7273  offset, numBytes, numEnt, numBytesPerValue);
7274  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
7275  (numBytes != numBytesOut, std::logic_error, ": At i=" << i
7276  << ", numBytes=" << numBytes << " != numBytesOut="
7277  << numBytesOut << ".");
7278 
7279  const ST* const valsRaw = const_cast<const ST*> (valsOut.data ());
7280  const GO* const gidsRaw = const_cast<const GO*> (gidsOut.data ());
7281  combineGlobalValuesRaw(lclRow, numEnt, valsRaw, gidsRaw,
7282  combineMode, prefix_raw, debug, verbose);
7283  // Don't update offset until current LID has succeeded.
7284  offset += numBytes;
7285  } // for each import LID i
7286 
7287  if (verbose) {
7288  std::ostringstream os;
7289  os << *prefix << "Done" << endl;
7290  std::cerr << os.str();
7291  }
7292  }
7293 
7294  template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
7295  Teuchos::RCP<MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> >
7296  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
7297  getColumnMapMultiVector (const MV& X_domainMap,
7298  const bool force) const
7299  {
7300  using Teuchos::null;
7301  using Teuchos::RCP;
7302  using Teuchos::rcp;
7303 
7304  TEUCHOS_TEST_FOR_EXCEPTION(
7305  ! this->hasColMap (), std::runtime_error, "Tpetra::CrsMatrix::getColumn"
7306  "MapMultiVector: You may only call this method if the matrix has a "
7307  "column Map. If the matrix does not yet have a column Map, you should "
7308  "first call fillComplete (with domain and range Map if necessary).");
7309 
7310  // If the graph is not fill complete, then the Import object (if
7311  // one should exist) hasn't been constructed yet.
7312  TEUCHOS_TEST_FOR_EXCEPTION(
7313  ! this->getGraph ()->isFillComplete (), std::runtime_error, "Tpetra::"
7314  "CrsMatrix::getColumnMapMultiVector: You may only call this method if "
7315  "this matrix's graph is fill complete.");
7316 
7317  const size_t numVecs = X_domainMap.getNumVectors ();
7318  RCP<const import_type> importer = this->getGraph ()->getImporter ();
7319  RCP<const map_type> colMap = this->getColMap ();
7320 
7321  RCP<MV> X_colMap; // null by default
7322 
7323  // If the Import object is trivial (null), then we don't need a
7324  // separate column Map multivector. Just return null in that
7325  // case. The caller is responsible for knowing not to use the
7326  // returned null pointer.
7327  //
7328  // If the Import is nontrivial, then we do need a separate
7329  // column Map multivector for the Import operation. Check in
7330  // that case if we have to (re)create the column Map
7331  // multivector.
7332  if (! importer.is_null () || force) {
7333  if (importMV_.is_null () || importMV_->getNumVectors () != numVecs) {
7334  X_colMap = rcp (new MV (colMap, numVecs));
7335 
7336  // Cache the newly created multivector for later reuse.
7337  importMV_ = X_colMap;
7338  }
7339  else { // Yay, we can reuse the cached multivector!
7340  X_colMap = importMV_;
7341  // mfh 09 Jan 2013: We don't have to fill with zeros first,
7342  // because the Import uses INSERT combine mode, which overwrites
7343  // existing entries.
7344  //
7345  //X_colMap->putScalar (ZERO);
7346  }
7347  }
7348  return X_colMap;
7349  }
7350 
7351  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
7352  Teuchos::RCP<MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node> >
7353  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
7354  getRowMapMultiVector (const MultiVector<Scalar, LocalOrdinal, GlobalOrdinal, Node>& Y_rangeMap,
7355  const bool force) const
7356  {
7357  using Teuchos::null;
7358  using Teuchos::RCP;
7359  using Teuchos::rcp;
7360 
7361  // If the graph is not fill complete, then the Export object (if
7362  // one should exist) hasn't been constructed yet.
7363  TEUCHOS_TEST_FOR_EXCEPTION(
7364  ! this->getGraph ()->isFillComplete (), std::runtime_error, "Tpetra::"
7365  "CrsMatrix::getRowMapMultiVector: You may only call this method if this "
7366  "matrix's graph is fill complete.");
7367 
7368  const size_t numVecs = Y_rangeMap.getNumVectors ();
7369  RCP<const export_type> exporter = this->getGraph ()->getExporter ();
7370  // Every version of the constructor takes either a row Map, or a
7371  // graph (all of whose constructors take a row Map). Thus, the
7372  // matrix always has a row Map.
7373  RCP<const map_type> rowMap = this->getRowMap ();
7374 
7375  RCP<MV> Y_rowMap; // null by default
7376 
7377  // If the Export object is trivial (null), then we don't need a
7378  // separate row Map multivector. Just return null in that case.
7379  // The caller is responsible for knowing not to use the returned
7380  // null pointer.
7381  //
7382  // If the Export is nontrivial, then we do need a separate row
7383  // Map multivector for the Export operation. Check in that case
7384  // if we have to (re)create the row Map multivector.
7385  if (! exporter.is_null () || force) {
7386  if (exportMV_.is_null () || exportMV_->getNumVectors () != numVecs) {
7387  Y_rowMap = rcp (new MV (rowMap, numVecs));
7388  exportMV_ = Y_rowMap; // Cache the newly created MV for later reuse.
7389  }
7390  else { // Yay, we can reuse the cached multivector!
7391  Y_rowMap = exportMV_;
7392  }
7393  }
7394  return Y_rowMap;
7395  }
7396 
7397  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
7398  void
7399  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
7400  removeEmptyProcessesInPlace (const Teuchos::RCP<const map_type>& newMap)
7401  {
7402  TEUCHOS_TEST_FOR_EXCEPTION(
7403  myGraph_.is_null (), std::logic_error, "Tpetra::CrsMatrix::"
7404  "removeEmptyProcessesInPlace: This method does not work when the matrix "
7405  "was created with a constant graph (that is, when it was created using "
7406  "the version of its constructor that takes an RCP<const CrsGraph>). "
7407  "This is because the matrix is not allowed to modify the graph in that "
7408  "case, but removing empty processes requires modifying the graph.");
7409  myGraph_->removeEmptyProcessesInPlace (newMap);
7410  // Even though CrsMatrix's row Map (as returned by getRowMap())
7411  // comes from its CrsGraph, CrsMatrix still implements DistObject,
7412  // so we also have to change the DistObject's Map.
7413  this->map_ = this->getRowMap ();
7414  // In the nonconst graph case, staticGraph_ is just a const
7415  // pointer to myGraph_. This assignment is probably redundant,
7416  // but it doesn't hurt.
7417  staticGraph_ = Teuchos::rcp_const_cast<const Graph> (myGraph_);
7418  }
7419 
7420  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
7421  Teuchos::RCP<RowMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> >
7422  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
7423  add (const Scalar& alpha,
7424  const RowMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>& A,
7425  const Scalar& beta,
7426  const Teuchos::RCP<const map_type>& domainMap,
7427  const Teuchos::RCP<const map_type>& rangeMap,
7428  const Teuchos::RCP<Teuchos::ParameterList>& params) const
7429  {
7430  using Teuchos::Array;
7431  using Teuchos::ArrayView;
7432  using Teuchos::ParameterList;
7433  using Teuchos::RCP;
7434  using Teuchos::rcp;
7435  using Teuchos::rcp_implicit_cast;
7436  using Teuchos::sublist;
7437  using std::endl;
7438  using LO = local_ordinal_type;
7439  using GO = global_ordinal_type;
7440  using crs_matrix_type =
7441  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>;
7442  const char errPfx[] = "Tpetra::CrsMatrix::add: ";
7443 
7444  const bool debug = Details::Behavior::debug("CrsMatrix");
7445  const bool verbose = Details::Behavior::verbose("CrsMatrix");
7446  std::unique_ptr<std::string> prefix;
7447  if (verbose) {
7448  prefix = this->createPrefix("CrsMatrix", "add");
7449  std::ostringstream os;
7450  os << *prefix << "Start" << endl;
7451  std::cerr << os.str ();
7452  }
7453 
7454  const crs_matrix_type& B = *this; // a convenient abbreviation
7455  const Scalar ZERO = Teuchos::ScalarTraits<Scalar>::zero();
7456  const Scalar ONE = Teuchos::ScalarTraits<Scalar>::one();
7457 
7458  // If the user didn't supply a domain or range Map, then try to
7459  // get one from B first (if it has them), then from A (if it has
7460  // them). If we don't have any domain or range Maps, scold the
7461  // user.
7462  RCP<const map_type> A_domainMap = A.getDomainMap ();
7463  RCP<const map_type> A_rangeMap = A.getRangeMap ();
7464  RCP<const map_type> B_domainMap = B.getDomainMap ();
7465  RCP<const map_type> B_rangeMap = B.getRangeMap ();
7466 
7467  RCP<const map_type> theDomainMap = domainMap;
7468  RCP<const map_type> theRangeMap = rangeMap;
7469 
7470  if (domainMap.is_null ()) {
7471  if (B_domainMap.is_null ()) {
7472  TEUCHOS_TEST_FOR_EXCEPTION(
7473  A_domainMap.is_null (), std::invalid_argument,
7474  "Tpetra::CrsMatrix::add: If neither A nor B have a domain Map, "
7475  "then you must supply a nonnull domain Map to this method.");
7476  theDomainMap = A_domainMap;
7477  } else {
7478  theDomainMap = B_domainMap;
7479  }
7480  }
7481  if (rangeMap.is_null ()) {
7482  if (B_rangeMap.is_null ()) {
7483  TEUCHOS_TEST_FOR_EXCEPTION(
7484  A_rangeMap.is_null (), std::invalid_argument,
7485  "Tpetra::CrsMatrix::add: If neither A nor B have a range Map, "
7486  "then you must supply a nonnull range Map to this method.");
7487  theRangeMap = A_rangeMap;
7488  } else {
7489  theRangeMap = B_rangeMap;
7490  }
7491  }
7492 
7493  if (debug) {
7494  // In debug mode, check that A and B have matching domain and
7495  // range Maps, if they have domain and range Maps at all. (If
7496  // they aren't fill complete, then they may not yet have them.)
7497  if (! A_domainMap.is_null() && ! A_rangeMap.is_null()) {
7498  if (! B_domainMap.is_null() && ! B_rangeMap.is_null()) {
7499  TEUCHOS_TEST_FOR_EXCEPTION
7500  (! B_domainMap->isSameAs(*A_domainMap),
7501  std::invalid_argument,
7502  errPfx << "The input RowMatrix A must have a domain Map "
7503  "which is the same as (isSameAs) this RowMatrix's "
7504  "domain Map.");
7505  TEUCHOS_TEST_FOR_EXCEPTION
7506  (! B_rangeMap->isSameAs(*A_rangeMap), std::invalid_argument,
7507  errPfx << "The input RowMatrix A must have a range Map "
7508  "which is the same as (isSameAs) this RowMatrix's range "
7509  "Map.");
7510  TEUCHOS_TEST_FOR_EXCEPTION
7511  (! domainMap.is_null() &&
7512  ! domainMap->isSameAs(*B_domainMap),
7513  std::invalid_argument,
7514  errPfx << "The input domain Map must be the same as "
7515  "(isSameAs) this RowMatrix's domain Map.");
7516  TEUCHOS_TEST_FOR_EXCEPTION
7517  (! rangeMap.is_null() &&
7518  ! rangeMap->isSameAs(*B_rangeMap),
7519  std::invalid_argument,
7520  errPfx << "The input range Map must be the same as "
7521  "(isSameAs) this RowMatrix's range Map.");
7522  }
7523  }
7524  else if (! B_domainMap.is_null() && ! B_rangeMap.is_null()) {
7525  TEUCHOS_TEST_FOR_EXCEPTION
7526  (! domainMap.is_null() &&
7527  ! domainMap->isSameAs(*B_domainMap),
7528  std::invalid_argument,
7529  errPfx << "The input domain Map must be the same as "
7530  "(isSameAs) this RowMatrix's domain Map.");
7531  TEUCHOS_TEST_FOR_EXCEPTION
7532  (! rangeMap.is_null() && ! rangeMap->isSameAs(*B_rangeMap),
7533  std::invalid_argument,
7534  errPfx << "The input range Map must be the same as "
7535  "(isSameAs) this RowMatrix's range Map.");
7536  }
7537  else {
7538  TEUCHOS_TEST_FOR_EXCEPTION
7539  (domainMap.is_null() || rangeMap.is_null(),
7540  std::invalid_argument, errPfx << "If neither A nor B "
7541  "have a domain and range Map, then you must supply a "
7542  "nonnull domain and range Map to this method.");
7543  }
7544  }
7545 
7546  // What parameters do we pass to C's constructor? Do we call
7547  // fillComplete on C after filling it? And if so, what parameters
7548  // do we pass to C's fillComplete call?
7549  bool callFillComplete = true;
7550  RCP<ParameterList> constructorSublist;
7551  RCP<ParameterList> fillCompleteSublist;
7552  if (! params.is_null()) {
7553  callFillComplete =
7554  params->get("Call fillComplete", callFillComplete);
7555  constructorSublist = sublist(params, "Constructor parameters");
7556  fillCompleteSublist = sublist(params, "fillComplete parameters");
7557  }
7558 
7559  RCP<const map_type> A_rowMap = A.getRowMap ();
7560  RCP<const map_type> B_rowMap = B.getRowMap ();
7561  RCP<const map_type> C_rowMap = B_rowMap; // see discussion in documentation
7562  RCP<crs_matrix_type> C; // The result matrix.
7563 
7564  // If A and B's row Maps are the same, we can compute an upper
7565  // bound on the number of entries in each row of C, before
7566  // actually computing the sum. A reasonable upper bound is the
7567  // sum of the two entry counts in each row.
7568  if (A_rowMap->isSameAs (*B_rowMap)) {
7569  const LO localNumRows = static_cast<LO> (A_rowMap->getLocalNumElements ());
7570  Array<size_t> C_maxNumEntriesPerRow (localNumRows, 0);
7571 
7572  // Get the number of entries in each row of A.
7573  if (alpha != ZERO) {
7574  for (LO localRow = 0; localRow < localNumRows; ++localRow) {
7575  const size_t A_numEntries = A.getNumEntriesInLocalRow (localRow);
7576  C_maxNumEntriesPerRow[localRow] += A_numEntries;
7577  }
7578  }
7579  // Get the number of entries in each row of B.
7580  if (beta != ZERO) {
7581  for (LO localRow = 0; localRow < localNumRows; ++localRow) {
7582  const size_t B_numEntries = B.getNumEntriesInLocalRow (localRow);
7583  C_maxNumEntriesPerRow[localRow] += B_numEntries;
7584  }
7585  }
7586  // Construct the result matrix C.
7587  if (constructorSublist.is_null ()) {
7588  C = rcp (new crs_matrix_type (C_rowMap, C_maxNumEntriesPerRow ()));
7589  } else {
7590  C = rcp (new crs_matrix_type (C_rowMap, C_maxNumEntriesPerRow (),
7591  constructorSublist));
7592  }
7593  // Since A and B have the same row Maps, we could add them
7594  // together all at once and merge values before we call
7595  // insertGlobalValues. However, we don't really need to, since
7596  // we've already allocated enough space in each row of C for C
7597  // to do the merge itself.
7598  }
7599  else { // the row Maps of A and B are not the same
7600  // Construct the result matrix C.
7601  // true: !A_rowMap->isSameAs (*B_rowMap)
7602  TEUCHOS_TEST_FOR_EXCEPTION
7603  (true, std::invalid_argument, errPfx << "The row maps must "
7604  "be the same for statically allocated matrices, to ensure "
7605  "that there is sufficient space to do the addition.");
7606  }
7607 
7608  TEUCHOS_TEST_FOR_EXCEPTION
7609  (C.is_null (), std::logic_error,
7610  errPfx << "C should not be null at this point. "
7611  "Please report this bug to the Tpetra developers.");
7612 
7613  if (verbose) {
7614  std::ostringstream os;
7615  os << *prefix << "Compute C = alpha*A + beta*B" << endl;
7616  std::cerr << os.str ();
7617  }
7618  using gids_type = nonconst_global_inds_host_view_type;
7619  using vals_type = nonconst_values_host_view_type;
7620  gids_type ind;
7621  vals_type val;
7622 
7623  if (alpha != ZERO) {
7624  const LO A_localNumRows = static_cast<LO> (A_rowMap->getLocalNumElements ());
7625  for (LO localRow = 0; localRow < A_localNumRows; ++localRow) {
7626  size_t A_numEntries = A.getNumEntriesInLocalRow (localRow);
7627  const GO globalRow = A_rowMap->getGlobalElement (localRow);
7628  if (A_numEntries > static_cast<size_t> (ind.size ())) {
7629  Kokkos::resize(ind,A_numEntries);
7630  Kokkos::resize(val,A_numEntries);
7631  }
7632  gids_type indView = Kokkos::subview(ind,std::make_pair((size_t)0, A_numEntries));
7633  vals_type valView = Kokkos::subview(val,std::make_pair((size_t)0, A_numEntries));
7634  A.getGlobalRowCopy (globalRow, indView, valView, A_numEntries);
7635 
7636  if (alpha != ONE) {
7637  for (size_t k = 0; k < A_numEntries; ++k) {
7638  valView[k] *= alpha;
7639  }
7640  }
7641  C->insertGlobalValues (globalRow, A_numEntries,
7642  reinterpret_cast<Scalar *>(valView.data()),
7643  indView.data());
7644  }
7645  }
7646 
7647  if (beta != ZERO) {
7648  const LO B_localNumRows = static_cast<LO> (B_rowMap->getLocalNumElements ());
7649  for (LO localRow = 0; localRow < B_localNumRows; ++localRow) {
7650  size_t B_numEntries = B.getNumEntriesInLocalRow (localRow);
7651  const GO globalRow = B_rowMap->getGlobalElement (localRow);
7652  if (B_numEntries > static_cast<size_t> (ind.size ())) {
7653  Kokkos::resize(ind,B_numEntries);
7654  Kokkos::resize(val,B_numEntries);
7655  }
7656  gids_type indView = Kokkos::subview(ind,std::make_pair((size_t)0, B_numEntries));
7657  vals_type valView = Kokkos::subview(val,std::make_pair((size_t)0, B_numEntries));
7658  B.getGlobalRowCopy (globalRow, indView, valView, B_numEntries);
7659 
7660  if (beta != ONE) {
7661  for (size_t k = 0; k < B_numEntries; ++k) {
7662  valView[k] *= beta;
7663  }
7664  }
7665  C->insertGlobalValues (globalRow, B_numEntries,
7666  reinterpret_cast<Scalar *>(valView.data()),
7667  indView.data());
7668  }
7669  }
7670 
7671  if (callFillComplete) {
7672  if (verbose) {
7673  std::ostringstream os;
7674  os << *prefix << "Call fillComplete on C" << endl;
7675  std::cerr << os.str ();
7676  }
7677  if (fillCompleteSublist.is_null ()) {
7678  C->fillComplete (theDomainMap, theRangeMap);
7679  } else {
7680  C->fillComplete (theDomainMap, theRangeMap, fillCompleteSublist);
7681  }
7682  }
7683  else if (verbose) {
7684  std::ostringstream os;
7685  os << *prefix << "Do NOT call fillComplete on C" << endl;
7686  std::cerr << os.str ();
7687  }
7688 
7689  if (verbose) {
7690  std::ostringstream os;
7691  os << *prefix << "Done" << endl;
7692  std::cerr << os.str ();
7693  }
7694  return rcp_implicit_cast<row_matrix_type> (C);
7695  }
7696 
7697 
7698 
7699  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
7700  void
7701  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
7702  transferAndFillComplete (Teuchos::RCP<CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> > & destMat,
7703  const ::Tpetra::Details::Transfer<LocalOrdinal, GlobalOrdinal, Node>& rowTransfer,
7704  const Teuchos::RCP<const ::Tpetra::Details::Transfer<LocalOrdinal, GlobalOrdinal, Node> > & domainTransfer,
7705  const Teuchos::RCP<const map_type>& domainMap,
7706  const Teuchos::RCP<const map_type>& rangeMap,
7707  const Teuchos::RCP<Teuchos::ParameterList>& params) const
7708  {
7709  using Details::Behavior;
7710  using Details::getArrayViewFromDualView;
7711  using Details::packCrsMatrixWithOwningPIDs;
7712  using Details::unpackAndCombineWithOwningPIDsCount;
7713  using Details::unpackAndCombineIntoCrsArrays;
7714  using Teuchos::ArrayRCP;
7715  using Teuchos::ArrayView;
7716  using Teuchos::Comm;
7717  using Teuchos::ParameterList;
7718  using Teuchos::RCP;
7719  using std::endl;
7720  typedef LocalOrdinal LO;
7721  typedef GlobalOrdinal GO;
7722  typedef node_type NT;
7723  typedef CrsMatrix<Scalar, LO, GO, NT> this_CRS_type;
7724  typedef Vector<int, LO, GO, NT> IntVectorType;
7725  using Teuchos::as;
7726 
7727  const bool debug = Behavior::debug("CrsMatrix");
7728  const bool verbose = Behavior::verbose("CrsMatrix");
7729  int MyPID = getComm ()->getRank ();
7730 
7731  std::unique_ptr<std::string> verbosePrefix;
7732  if (verbose) {
7733  verbosePrefix =
7734  this->createPrefix("CrsMatrix", "transferAndFillComplete");
7735  std::ostringstream os;
7736  os << "Start" << endl;
7737  std::cerr << os.str();
7738  }
7739 
7740  //
7741  // Get the caller's parameters
7742  //
7743  bool isMM = false; // optimize for matrix-matrix ops.
7744  bool reverseMode = false; // Are we in reverse mode?
7745  bool restrictComm = false; // Do we need to restrict the communicator?
7746 
7747  int mm_optimization_core_count =
7748  Behavior::TAFC_OptimizationCoreCount();
7749  RCP<ParameterList> matrixparams; // parameters for the destination matrix
7750  bool overrideAllreduce = false;
7751  bool useKokkosPath = false;
7752  if (! params.is_null ()) {
7753  matrixparams = sublist (params, "CrsMatrix");
7754  reverseMode = params->get ("Reverse Mode", reverseMode);
7755  useKokkosPath = params->get ("TAFC: use kokkos path", useKokkosPath);
7756  restrictComm = params->get ("Restrict Communicator", restrictComm);
7757  auto & slist = params->sublist("matrixmatrix: kernel params",false);
7758  isMM = slist.get("isMatrixMatrix_TransferAndFillComplete",false);
7759  mm_optimization_core_count = slist.get("MM_TAFC_OptimizationCoreCount",mm_optimization_core_count);
7760 
7761  overrideAllreduce = slist.get("MM_TAFC_OverrideAllreduceCheck",false);
7762  if(getComm()->getSize() < mm_optimization_core_count && isMM) isMM = false;
7763  if(reverseMode) isMM = false;
7764  }
7765 
7766  // Only used in the sparse matrix-matrix multiply (isMM) case.
7767  std::shared_ptr< ::Tpetra::Details::CommRequest> iallreduceRequest;
7768  int mismatch = 0;
7769  int reduced_mismatch = 0;
7770  if (isMM && !overrideAllreduce) {
7771 
7772  // Test for pathological matrix transfer
7773  const bool source_vals = ! getGraph ()->getImporter ().is_null();
7774  const bool target_vals = ! (rowTransfer.getExportLIDs ().size() == 0 ||
7775  rowTransfer.getRemoteLIDs ().size() == 0);
7776  mismatch = (source_vals != target_vals) ? 1 : 0;
7777  iallreduceRequest =
7778  ::Tpetra::Details::iallreduce (mismatch, reduced_mismatch,
7779  Teuchos::REDUCE_MAX, * (getComm ()));
7780  }
7781 
7782 #ifdef HAVE_TPETRA_MMM_TIMINGS
7783  using Teuchos::TimeMonitor;
7784  std::string label;
7785  if(!params.is_null())
7786  label = params->get("Timer Label",label);
7787  std::string prefix = std::string("Tpetra ")+ label + std::string(": ");
7788  std::string tlstr;
7789  {
7790  std::ostringstream os;
7791  if(isMM) os<<":MMOpt";
7792  else os<<":MMLegacy";
7793  tlstr = os.str();
7794  }
7795 
7796  Teuchos::TimeMonitor MMall(*TimeMonitor::getNewTimer(prefix + std::string("TAFC All") +tlstr ));
7797 #endif
7798 
7799  // Make sure that the input argument rowTransfer is either an
7800  // Import or an Export. Import and Export are the only two
7801  // subclasses of Transfer that we defined, but users might
7802  // (unwisely, for now at least) decide to implement their own
7803  // subclasses. Exclude this possibility.
7804  const import_type* xferAsImport = dynamic_cast<const import_type*> (&rowTransfer);
7805  const export_type* xferAsExport = dynamic_cast<const export_type*> (&rowTransfer);
7806  TEUCHOS_TEST_FOR_EXCEPTION(
7807  xferAsImport == nullptr && xferAsExport == nullptr, std::invalid_argument,
7808  "Tpetra::CrsMatrix::transferAndFillComplete: The 'rowTransfer' input "
7809  "argument must be either an Import or an Export, and its template "
7810  "parameters must match the corresponding template parameters of the "
7811  "CrsMatrix.");
7812 
7813  // Make sure that the input argument domainTransfer is either an
7814  // Import or an Export. Import and Export are the only two
7815  // subclasses of Transfer that we defined, but users might
7816  // (unwisely, for now at least) decide to implement their own
7817  // subclasses. Exclude this possibility.
7818  Teuchos::RCP<const import_type> xferDomainAsImport = Teuchos::rcp_dynamic_cast<const import_type> (domainTransfer);
7819  Teuchos::RCP<const export_type> xferDomainAsExport = Teuchos::rcp_dynamic_cast<const export_type> (domainTransfer);
7820 
7821  if(! domainTransfer.is_null()) {
7822  TEUCHOS_TEST_FOR_EXCEPTION(
7823  (xferDomainAsImport.is_null() && xferDomainAsExport.is_null()), std::invalid_argument,
7824  "Tpetra::CrsMatrix::transferAndFillComplete: The 'domainTransfer' input "
7825  "argument must be either an Import or an Export, and its template "
7826  "parameters must match the corresponding template parameters of the "
7827  "CrsMatrix.");
7828 
7829  TEUCHOS_TEST_FOR_EXCEPTION(
7830  ( xferAsImport != nullptr || ! xferDomainAsImport.is_null() ) &&
7831  (( xferAsImport != nullptr && xferDomainAsImport.is_null() ) ||
7832  ( xferAsImport == nullptr && ! xferDomainAsImport.is_null() )), std::invalid_argument,
7833  "Tpetra::CrsMatrix::transferAndFillComplete: The 'rowTransfer' and 'domainTransfer' input "
7834  "arguments must be of the same type (either Import or Export).");
7835 
7836  TEUCHOS_TEST_FOR_EXCEPTION(
7837  ( xferAsExport != nullptr || ! xferDomainAsExport.is_null() ) &&
7838  (( xferAsExport != nullptr && xferDomainAsExport.is_null() ) ||
7839  ( xferAsExport == nullptr && ! xferDomainAsExport.is_null() )), std::invalid_argument,
7840  "Tpetra::CrsMatrix::transferAndFillComplete: The 'rowTransfer' and 'domainTransfer' input "
7841  "arguments must be of the same type (either Import or Export).");
7842  } // domainTransfer != null
7843 
7844 
7845  // FIXME (mfh 15 May 2014) Wouldn't communication still be needed,
7846  // if the source Map is not distributed but the target Map is?
7847  const bool communication_needed = rowTransfer.getSourceMap ()->isDistributed ();
7848 
7849  // Get the new domain and range Maps. We need some of them for
7850  // error checking, now that we have the reverseMode parameter.
7851  RCP<const map_type> MyRowMap = reverseMode ?
7852  rowTransfer.getSourceMap () : rowTransfer.getTargetMap ();
7853  RCP<const map_type> MyColMap; // create this below
7854  RCP<const map_type> MyDomainMap = ! domainMap.is_null () ?
7855  domainMap : getDomainMap ();
7856  RCP<const map_type> MyRangeMap = ! rangeMap.is_null () ?
7857  rangeMap : getRangeMap ();
7858  RCP<const map_type> BaseRowMap = MyRowMap;
7859  RCP<const map_type> BaseDomainMap = MyDomainMap;
7860 
7861  // If the user gave us a nonnull destMat, then check whether it's
7862  // "pristine." That means that it has no entries.
7863  //
7864  // FIXME (mfh 15 May 2014) If this is not true on all processes,
7865  // then this exception test may hang. It would be better to
7866  // forward an error flag to the next communication phase.
7867  if (! destMat.is_null ()) {
7868  // FIXME (mfh 15 May 2014): The Epetra idiom for checking
7869  // whether a graph or matrix has no entries on the calling
7870  // process, is that it is neither locally nor globally indexed.
7871  // This may change eventually with the Kokkos refactor version
7872  // of Tpetra, so it would be better just to check the quantity
7873  // of interest directly. Note that with the Kokkos refactor
7874  // version of Tpetra, asking for the total number of entries in
7875  // a graph or matrix that is not fill complete might require
7876  // computation (kernel launch), since it is not thread scalable
7877  // to update a count every time an entry is inserted.
7878  const bool NewFlag = ! destMat->getGraph ()->isLocallyIndexed () &&
7879  ! destMat->getGraph ()->isGloballyIndexed ();
7880  TEUCHOS_TEST_FOR_EXCEPTION(
7881  ! NewFlag, std::invalid_argument, "Tpetra::CrsMatrix::"
7882  "transferAndFillComplete: The input argument 'destMat' is only allowed "
7883  "to be nonnull, if its graph is empty (neither locally nor globally "
7884  "indexed).");
7885  // FIXME (mfh 15 May 2014) At some point, we want to change
7886  // graphs and matrices so that their DistObject Map
7887  // (this->getMap()) may differ from their row Map. This will
7888  // make redistribution for 2-D distributions more efficient. I
7889  // hesitate to change this check, because I'm not sure how much
7890  // the code here depends on getMap() and getRowMap() being the
7891  // same.
7892  TEUCHOS_TEST_FOR_EXCEPTION(
7893  ! destMat->getRowMap ()->isSameAs (*MyRowMap), std::invalid_argument,
7894  "Tpetra::CrsMatrix::transferAndFillComplete: The (row) Map of the "
7895  "input argument 'destMat' is not the same as the (row) Map specified "
7896  "by the input argument 'rowTransfer'.");
7897  TEUCHOS_TEST_FOR_EXCEPTION(
7898  ! destMat->checkSizes (*this), std::invalid_argument,
7899  "Tpetra::CrsMatrix::transferAndFillComplete: You provided a nonnull "
7900  "destination matrix, but checkSizes() indicates that it is not a legal "
7901  "legal target for redistribution from the source matrix (*this). This "
7902  "may mean that they do not have the same dimensions.");
7903  }
7904 
7905  // If forward mode (the default), then *this's (row) Map must be
7906  // the same as the source Map of the Transfer. If reverse mode,
7907  // then *this's (row) Map must be the same as the target Map of
7908  // the Transfer.
7909  //
7910  // FIXME (mfh 15 May 2014) At some point, we want to change graphs
7911  // and matrices so that their DistObject Map (this->getMap()) may
7912  // differ from their row Map. This will make redistribution for
7913  // 2-D distributions more efficient. I hesitate to change this
7914  // check, because I'm not sure how much the code here depends on
7915  // getMap() and getRowMap() being the same.
7916  TEUCHOS_TEST_FOR_EXCEPTION(
7917  ! (reverseMode || getRowMap ()->isSameAs (*rowTransfer.getSourceMap ())),
7918  std::invalid_argument, "Tpetra::CrsMatrix::transferAndFillComplete: "
7919  "rowTransfer->getSourceMap() must match this->getRowMap() in forward mode.");
7920  TEUCHOS_TEST_FOR_EXCEPTION(
7921  ! (! reverseMode || getRowMap ()->isSameAs (*rowTransfer.getTargetMap ())),
7922  std::invalid_argument, "Tpetra::CrsMatrix::transferAndFillComplete: "
7923  "rowTransfer->getTargetMap() must match this->getRowMap() in reverse mode.");
7924 
7925  // checks for domainTransfer
7926  TEUCHOS_TEST_FOR_EXCEPTION(
7927  ! xferDomainAsImport.is_null() && ! xferDomainAsImport->getTargetMap()->isSameAs(*domainMap),
7928  std::invalid_argument,
7929  "Tpetra::CrsMatrix::transferAndFillComplete: The target map of the 'domainTransfer' input "
7930  "argument must be the same as the rebalanced domain map 'domainMap'");
7931 
7932  TEUCHOS_TEST_FOR_EXCEPTION(
7933  ! xferDomainAsExport.is_null() && ! xferDomainAsExport->getSourceMap()->isSameAs(*domainMap),
7934  std::invalid_argument,
7935  "Tpetra::CrsMatrix::transferAndFillComplete: The source map of the 'domainTransfer' input "
7936  "argument must be the same as the rebalanced domain map 'domainMap'");
7937 
7938  // The basic algorithm here is:
7939  //
7940  // 1. Call the moral equivalent of "Distor.do" to handle the import.
7941  // 2. Copy all the Imported and Copy/Permuted data into the raw
7942  // CrsMatrix / CrsGraphData pointers, still using GIDs.
7943  // 3. Call an optimized version of MakeColMap that avoids the
7944  // Directory lookups (since the importer knows who owns all the
7945  // GIDs) AND reindexes to LIDs.
7946  // 4. Call expertStaticFillComplete()
7947 
7948  // Get information from the Importer
7949  const size_t NumSameIDs = rowTransfer.getNumSameIDs();
7950  ArrayView<const LO> ExportLIDs = reverseMode ?
7951  rowTransfer.getRemoteLIDs () : rowTransfer.getExportLIDs ();
7952  auto RemoteLIDs = reverseMode ?
7953  rowTransfer.getExportLIDs_dv() : rowTransfer.getRemoteLIDs_dv();
7954  auto PermuteToLIDs = reverseMode ?
7955  rowTransfer.getPermuteFromLIDs_dv() : rowTransfer.getPermuteToLIDs_dv();
7956  auto PermuteFromLIDs = reverseMode ?
7957  rowTransfer.getPermuteToLIDs_dv() : rowTransfer.getPermuteFromLIDs_dv();
7958  Distributor& Distor = rowTransfer.getDistributor ();
7959 
7960  // Owning PIDs
7961  Teuchos::Array<int> SourcePids;
7962 
7963  // Temp variables for sub-communicators
7964  RCP<const map_type> ReducedRowMap, ReducedColMap,
7965  ReducedDomainMap, ReducedRangeMap;
7966  RCP<const Comm<int> > ReducedComm;
7967 
7968  // If the user gave us a null destMat, then construct the new
7969  // destination matrix. We will replace its column Map later.
7970  if (destMat.is_null ()) {
7971  destMat = rcp (new this_CRS_type (MyRowMap, 0, matrixparams));
7972  }
7973 
7974  /***************************************************/
7975  /***** 1) First communicator restriction phase ****/
7976  /***************************************************/
7977  if (restrictComm) {
7978 #ifdef HAVE_TPETRA_MMM_TIMINGS
7979  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC restrictComm")));
7980 #endif
7981  ReducedRowMap = MyRowMap->removeEmptyProcesses ();
7982  ReducedComm = ReducedRowMap.is_null () ?
7983  Teuchos::null :
7984  ReducedRowMap->getComm ();
7985  destMat->removeEmptyProcessesInPlace (ReducedRowMap);
7986 
7987  ReducedDomainMap = MyRowMap.getRawPtr () == MyDomainMap.getRawPtr () ?
7988  ReducedRowMap :
7989  MyDomainMap->replaceCommWithSubset (ReducedComm);
7990  ReducedRangeMap = MyRowMap.getRawPtr () == MyRangeMap.getRawPtr () ?
7991  ReducedRowMap :
7992  MyRangeMap->replaceCommWithSubset (ReducedComm);
7993 
7994  // Reset the "my" maps
7995  MyRowMap = ReducedRowMap;
7996  MyDomainMap = ReducedDomainMap;
7997  MyRangeMap = ReducedRangeMap;
7998 
7999  // Update my PID, if we've restricted the communicator
8000  if (! ReducedComm.is_null ()) {
8001  MyPID = ReducedComm->getRank ();
8002  }
8003  else {
8004  MyPID = -2; // For debugging
8005  }
8006  }
8007  else {
8008  ReducedComm = MyRowMap->getComm ();
8009  }
8010 
8011 
8012 
8013  /***************************************************/
8014  /***** 2) From Tpetra::DistObject::doTransfer() ****/
8015  /***************************************************/
8016  // Get the owning PIDs
8017  RCP<const import_type> MyImporter = getGraph ()->getImporter ();
8018 
8019  // check whether domain maps of source matrix and base domain map is the same
8020  bool bSameDomainMap = BaseDomainMap->isSameAs (*getDomainMap ());
8021 
8022  if (! restrictComm && ! MyImporter.is_null () && bSameDomainMap ) {
8023 #ifdef HAVE_TPETRA_MMM_TIMINGS
8024  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC getOwningPIDs same map")));
8025 #endif
8026  // Same domain map as source matrix
8027  //
8028  // NOTE: This won't work for restrictComm (because the Import
8029  // doesn't know the restricted PIDs), though writing an
8030  // optimized version for that case would be easy (Import an
8031  // IntVector of the new PIDs). Might want to add this later.
8032  Import_Util::getPids (*MyImporter, SourcePids, false);
8033  }
8034  else if (restrictComm && ! MyImporter.is_null () && bSameDomainMap) {
8035  // Same domain map as source matrix (restricted communicator)
8036  // We need one import from the domain to the column map
8037 #ifdef HAVE_TPETRA_MMM_TIMINGS
8038  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC getOwningPIDs restricted comm")));
8039 #endif
8040  IntVectorType SourceDomain_pids(getDomainMap (),true);
8041  IntVectorType SourceCol_pids(getColMap());
8042  // SourceDomain_pids contains the restricted pids
8043  SourceDomain_pids.putScalar(MyPID);
8044 
8045  SourceCol_pids.doImport (SourceDomain_pids, *MyImporter, INSERT);
8046  SourcePids.resize (getColMap ()->getLocalNumElements ());
8047  SourceCol_pids.get1dCopy (SourcePids ());
8048  }
8049  else if (MyImporter.is_null ()) {
8050  // Matrix has no off-process entries
8051 #ifdef HAVE_TPETRA_MMM_TIMINGS
8052  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC getOwningPIDs all local entries")));
8053 #endif
8054  SourcePids.resize (getColMap ()->getLocalNumElements ());
8055  SourcePids.assign (getColMap ()->getLocalNumElements (), MyPID);
8056  }
8057  else if ( ! MyImporter.is_null () &&
8058  ! domainTransfer.is_null () ) {
8059  // general implementation for rectangular matrices with
8060  // domain map different than SourceMatrix domain map.
8061  // User has to provide a DomainTransfer object. We need
8062  // to communications (import/export)
8063 #ifdef HAVE_TPETRA_MMM_TIMINGS
8064  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC getOwningPIDs rectangular case")));
8065 #endif
8066 
8067  // TargetDomain_pids lives on the rebalanced new domain map
8068  IntVectorType TargetDomain_pids (domainMap);
8069  TargetDomain_pids.putScalar (MyPID);
8070 
8071  // SourceDomain_pids lives on the non-rebalanced old domain map
8072  IntVectorType SourceDomain_pids (getDomainMap ());
8073 
8074  // SourceCol_pids lives on the non-rebalanced old column map
8075  IntVectorType SourceCol_pids (getColMap ());
8076 
8077  if (! reverseMode && ! xferDomainAsImport.is_null() ) {
8078  SourceDomain_pids.doExport (TargetDomain_pids, *xferDomainAsImport, INSERT);
8079  }
8080  else if (reverseMode && ! xferDomainAsExport.is_null() ) {
8081  SourceDomain_pids.doExport (TargetDomain_pids, *xferDomainAsExport, INSERT);
8082  }
8083  else if (! reverseMode && ! xferDomainAsExport.is_null() ) {
8084  SourceDomain_pids.doImport (TargetDomain_pids, *xferDomainAsExport, INSERT);
8085  }
8086  else if (reverseMode && ! xferDomainAsImport.is_null() ) {
8087  SourceDomain_pids.doImport (TargetDomain_pids, *xferDomainAsImport, INSERT);
8088  }
8089  else {
8090  TEUCHOS_TEST_FOR_EXCEPTION(
8091  true, std::logic_error, "Tpetra::CrsMatrix::"
8092  "transferAndFillComplete: Should never get here! "
8093  "Please report this bug to a Tpetra developer.");
8094  }
8095  SourceCol_pids.doImport (SourceDomain_pids, *MyImporter, INSERT);
8096  SourcePids.resize (getColMap ()->getLocalNumElements ());
8097  SourceCol_pids.get1dCopy (SourcePids ());
8098  }
8099  else if ( ! MyImporter.is_null () &&
8100  BaseDomainMap->isSameAs (*BaseRowMap) &&
8101  getDomainMap ()->isSameAs (*getRowMap ())) {
8102  // We can use the rowTransfer + SourceMatrix's Import to find out who owns what.
8103 #ifdef HAVE_TPETRA_MMM_TIMINGS
8104  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC getOwningPIDs query import")));
8105 #endif
8106 
8107  IntVectorType TargetRow_pids (domainMap);
8108  IntVectorType SourceRow_pids (getRowMap ());
8109  IntVectorType SourceCol_pids (getColMap ());
8110 
8111  TargetRow_pids.putScalar (MyPID);
8112  if (! reverseMode && xferAsImport != nullptr) {
8113  SourceRow_pids.doExport (TargetRow_pids, *xferAsImport, INSERT);
8114  }
8115  else if (reverseMode && xferAsExport != nullptr) {
8116  SourceRow_pids.doExport (TargetRow_pids, *xferAsExport, INSERT);
8117  }
8118  else if (! reverseMode && xferAsExport != nullptr) {
8119  SourceRow_pids.doImport (TargetRow_pids, *xferAsExport, INSERT);
8120  }
8121  else if (reverseMode && xferAsImport != nullptr) {
8122  SourceRow_pids.doImport (TargetRow_pids, *xferAsImport, INSERT);
8123  }
8124  else {
8125  TEUCHOS_TEST_FOR_EXCEPTION(
8126  true, std::logic_error, "Tpetra::CrsMatrix::"
8127  "transferAndFillComplete: Should never get here! "
8128  "Please report this bug to a Tpetra developer.");
8129  }
8130 
8131  SourceCol_pids.doImport (SourceRow_pids, *MyImporter, INSERT);
8132  SourcePids.resize (getColMap ()->getLocalNumElements ());
8133  SourceCol_pids.get1dCopy (SourcePids ());
8134  }
8135  else {
8136  TEUCHOS_TEST_FOR_EXCEPTION(
8137  true, std::invalid_argument, "Tpetra::CrsMatrix::"
8138  "transferAndFillComplete: This method only allows either domainMap == "
8139  "getDomainMap (), or (domainMap == rowTransfer.getTargetMap () and "
8140  "getDomainMap () == getRowMap ()).");
8141  }
8142 
8143  // Tpetra-specific stuff
8144  size_t constantNumPackets = destMat->constantNumberOfPackets ();
8145  {
8146 #ifdef HAVE_TPETRA_MMM_TIMINGS
8147  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC reallocate buffers")));
8148 #endif
8149  if (constantNumPackets == 0) {
8150  destMat->reallocArraysForNumPacketsPerLid (ExportLIDs.size (),
8151  RemoteLIDs.view_host().size ());
8152  }
8153  else {
8154  // There are a constant number of packets per element. We
8155  // already know (from the number of "remote" (incoming)
8156  // elements) how many incoming elements we expect, so we can
8157  // resize the buffer accordingly.
8158  const size_t rbufLen = RemoteLIDs.view_host().size() * constantNumPackets;
8159  destMat->reallocImportsIfNeeded (rbufLen, false, nullptr);
8160  }
8161  }
8162 
8163  // Pack & Prepare w/ owning PIDs
8164  {
8165 #ifdef HAVE_TPETRA_MMM_TIMINGS
8166  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC pack and prepare")));
8167 #endif
8168  if (debug) {
8169  using Teuchos::outArg;
8170  using Teuchos::REDUCE_MAX;
8171  using Teuchos::reduceAll;
8172  using std::cerr;
8173  using std::endl;
8174  RCP<const Teuchos::Comm<int> > comm = this->getComm ();
8175  const int myRank = comm->getRank ();
8176 
8177  std::ostringstream errStrm;
8178  int lclErr = 0;
8179  int gblErr = 0;
8180 
8181  Teuchos::ArrayView<size_t> numExportPacketsPerLID;
8182  try {
8183  // packAndPrepare* methods modify numExportPacketsPerLID_.
8184  destMat->numExportPacketsPerLID_.modify_host ();
8185  numExportPacketsPerLID =
8186  getArrayViewFromDualView (destMat->numExportPacketsPerLID_);
8187  }
8188  catch (std::exception& e) {
8189  errStrm << "Proc " << myRank << ": getArrayViewFromDualView threw: "
8190  << e.what () << std::endl;
8191  lclErr = 1;
8192  }
8193  catch (...) {
8194  errStrm << "Proc " << myRank << ": getArrayViewFromDualView threw "
8195  "an exception not a subclass of std::exception" << std::endl;
8196  lclErr = 1;
8197  }
8198 
8199  if (! comm.is_null ()) {
8200  reduceAll<int, int> (*comm, REDUCE_MAX, lclErr, outArg (gblErr));
8201  }
8202  if (gblErr != 0) {
8203  ::Tpetra::Details::gathervPrint (cerr, errStrm.str (), *comm);
8204  TEUCHOS_TEST_FOR_EXCEPTION(
8205  true, std::runtime_error, "getArrayViewFromDualView threw an "
8206  "exception on at least one process.");
8207  }
8208 
8209  if (verbose) {
8210  std::ostringstream os;
8211  os << *verbosePrefix << "Calling packCrsMatrixWithOwningPIDs"
8212  << std::endl;
8213  std::cerr << os.str ();
8214  }
8215  try {
8216  packCrsMatrixWithOwningPIDs (*this,
8217  destMat->exports_,
8218  numExportPacketsPerLID,
8219  ExportLIDs,
8220  SourcePids,
8221  constantNumPackets);
8222  }
8223  catch (std::exception& e) {
8224  errStrm << "Proc " << myRank << ": packCrsMatrixWithOwningPIDs threw: "
8225  << e.what () << std::endl;
8226  lclErr = 1;
8227  }
8228  catch (...) {
8229  errStrm << "Proc " << myRank << ": packCrsMatrixWithOwningPIDs threw "
8230  "an exception not a subclass of std::exception" << std::endl;
8231  lclErr = 1;
8232  }
8233 
8234  if (verbose) {
8235  std::ostringstream os;
8236  os << *verbosePrefix << "Done with packCrsMatrixWithOwningPIDs"
8237  << std::endl;
8238  std::cerr << os.str ();
8239  }
8240 
8241  if (! comm.is_null ()) {
8242  reduceAll<int, int> (*comm, REDUCE_MAX, lclErr, outArg (gblErr));
8243  }
8244  if (gblErr != 0) {
8245  ::Tpetra::Details::gathervPrint (cerr, errStrm.str (), *comm);
8246  TEUCHOS_TEST_FOR_EXCEPTION(
8247  true, std::runtime_error, "packCrsMatrixWithOwningPIDs threw an "
8248  "exception on at least one process.");
8249  }
8250  }
8251  else {
8252  // packAndPrepare* methods modify numExportPacketsPerLID_.
8253  destMat->numExportPacketsPerLID_.modify_host ();
8254  Teuchos::ArrayView<size_t> numExportPacketsPerLID =
8255  getArrayViewFromDualView (destMat->numExportPacketsPerLID_);
8256  if (verbose) {
8257  std::ostringstream os;
8258  os << *verbosePrefix << "Calling packCrsMatrixWithOwningPIDs"
8259  << std::endl;
8260  std::cerr << os.str ();
8261  }
8262  packCrsMatrixWithOwningPIDs (*this,
8263  destMat->exports_,
8264  numExportPacketsPerLID,
8265  ExportLIDs,
8266  SourcePids,
8267  constantNumPackets);
8268  if (verbose) {
8269  std::ostringstream os;
8270  os << *verbosePrefix << "Done with packCrsMatrixWithOwningPIDs"
8271  << std::endl;
8272  std::cerr << os.str ();
8273  }
8274  }
8275  }
8276 
8277  // Do the exchange of remote data.
8278  {
8279 #ifdef HAVE_TPETRA_MMM_TIMINGS
8280  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC getOwningPIDs exchange remote data")));
8281 #endif
8282  if (! communication_needed) {
8283  if (verbose) {
8284  std::ostringstream os;
8285  os << *verbosePrefix << "Communication not needed" << std::endl;
8286  std::cerr << os.str ();
8287  }
8288  }
8289  else {
8290  if (reverseMode) {
8291  if (constantNumPackets == 0) { // variable number of packets per LID
8292  if (verbose) {
8293  std::ostringstream os;
8294  os << *verbosePrefix << "Reverse mode, variable # packets / LID"
8295  << std::endl;
8296  std::cerr << os.str ();
8297  }
8298  // Make sure that host has the latest version, since we're
8299  // using the version on host. If host has the latest
8300  // version, syncing to host does nothing.
8301  destMat->numExportPacketsPerLID_.sync_host ();
8302  Teuchos::ArrayView<const size_t> numExportPacketsPerLID =
8303  getArrayViewFromDualView (destMat->numExportPacketsPerLID_);
8304  destMat->numImportPacketsPerLID_.sync_host ();
8305  Teuchos::ArrayView<size_t> numImportPacketsPerLID =
8306  getArrayViewFromDualView (destMat->numImportPacketsPerLID_);
8307 
8308  if (verbose) {
8309  std::ostringstream os;
8310  os << *verbosePrefix << "Calling 3-arg doReversePostsAndWaits"
8311  << std::endl;
8312  std::cerr << os.str ();
8313  }
8314  Distor.doReversePostsAndWaits(destMat->numExportPacketsPerLID_.view_host(), 1,
8315  destMat->numImportPacketsPerLID_.view_host());
8316  if (verbose) {
8317  std::ostringstream os;
8318  os << *verbosePrefix << "Finished 3-arg doReversePostsAndWaits"
8319  << std::endl;
8320  std::cerr << os.str ();
8321  }
8322 
8323  size_t totalImportPackets = 0;
8324  for (Array_size_type i = 0; i < numImportPacketsPerLID.size (); ++i) {
8325  totalImportPackets += numImportPacketsPerLID[i];
8326  }
8327 
8328  // Reallocation MUST go before setting the modified flag,
8329  // because it may clear out the flags.
8330  destMat->reallocImportsIfNeeded (totalImportPackets, verbose,
8331  verbosePrefix.get ());
8332  destMat->imports_.modify_host ();
8333  auto hostImports = destMat->imports_.view_host();
8334  // This is a legacy host pack/unpack path, so use the host
8335  // version of exports_.
8336  destMat->exports_.sync_host ();
8337  auto hostExports = destMat->exports_.view_host();
8338  if (verbose) {
8339  std::ostringstream os;
8340  os << *verbosePrefix << "Calling 4-arg doReversePostsAndWaits"
8341  << std::endl;
8342  std::cerr << os.str ();
8343  }
8344  Distor.doReversePostsAndWaits (hostExports,
8345  numExportPacketsPerLID,
8346  hostImports,
8347  numImportPacketsPerLID);
8348  if (verbose) {
8349  std::ostringstream os;
8350  os << *verbosePrefix << "Finished 4-arg doReversePostsAndWaits"
8351  << std::endl;
8352  std::cerr << os.str ();
8353  }
8354  }
8355  else { // constant number of packets per LID
8356  if (verbose) {
8357  std::ostringstream os;
8358  os << *verbosePrefix << "Reverse mode, constant # packets / LID"
8359  << std::endl;
8360  std::cerr << os.str ();
8361  }
8362  destMat->imports_.modify_host ();
8363  auto hostImports = destMat->imports_.view_host();
8364  // This is a legacy host pack/unpack path, so use the host
8365  // version of exports_.
8366  destMat->exports_.sync_host ();
8367  auto hostExports = destMat->exports_.view_host();
8368  if (verbose) {
8369  std::ostringstream os;
8370  os << *verbosePrefix << "Calling 3-arg doReversePostsAndWaits"
8371  << std::endl;
8372  std::cerr << os.str ();
8373  }
8374  Distor.doReversePostsAndWaits (hostExports,
8375  constantNumPackets,
8376  hostImports);
8377  if (verbose) {
8378  std::ostringstream os;
8379  os << *verbosePrefix << "Finished 3-arg doReversePostsAndWaits"
8380  << std::endl;
8381  std::cerr << os.str ();
8382  }
8383  }
8384  }
8385  else { // forward mode (the default)
8386  if (constantNumPackets == 0) { // variable number of packets per LID
8387  if (verbose) {
8388  std::ostringstream os;
8389  os << *verbosePrefix << "Forward mode, variable # packets / LID"
8390  << std::endl;
8391  std::cerr << os.str ();
8392  }
8393  // Make sure that host has the latest version, since we're
8394  // using the version on host. If host has the latest
8395  // version, syncing to host does nothing.
8396  destMat->numExportPacketsPerLID_.sync_host ();
8397  Teuchos::ArrayView<const size_t> numExportPacketsPerLID =
8398  getArrayViewFromDualView (destMat->numExportPacketsPerLID_);
8399  destMat->numImportPacketsPerLID_.sync_host ();
8400  Teuchos::ArrayView<size_t> numImportPacketsPerLID =
8401  getArrayViewFromDualView (destMat->numImportPacketsPerLID_);
8402  if (verbose) {
8403  std::ostringstream os;
8404  os << *verbosePrefix << "Calling 3-arg doPostsAndWaits"
8405  << std::endl;
8406  std::cerr << os.str ();
8407  }
8408  Distor.doPostsAndWaits(destMat->numExportPacketsPerLID_.view_host(), 1,
8409  destMat->numImportPacketsPerLID_.view_host());
8410  if (verbose) {
8411  std::ostringstream os;
8412  os << *verbosePrefix << "Finished 3-arg doPostsAndWaits"
8413  << std::endl;
8414  std::cerr << os.str ();
8415  }
8416 
8417  size_t totalImportPackets = 0;
8418  for (Array_size_type i = 0; i < numImportPacketsPerLID.size (); ++i) {
8419  totalImportPackets += numImportPacketsPerLID[i];
8420  }
8421 
8422  // Reallocation MUST go before setting the modified flag,
8423  // because it may clear out the flags.
8424  destMat->reallocImportsIfNeeded (totalImportPackets, verbose,
8425  verbosePrefix.get ());
8426  destMat->imports_.modify_host ();
8427  auto hostImports = destMat->imports_.view_host();
8428  // This is a legacy host pack/unpack path, so use the host
8429  // version of exports_.
8430  destMat->exports_.sync_host ();
8431  auto hostExports = destMat->exports_.view_host();
8432  if (verbose) {
8433  std::ostringstream os;
8434  os << *verbosePrefix << "Calling 4-arg doPostsAndWaits"
8435  << std::endl;
8436  std::cerr << os.str ();
8437  }
8438  Distor.doPostsAndWaits (hostExports,
8439  numExportPacketsPerLID,
8440  hostImports,
8441  numImportPacketsPerLID);
8442  if (verbose) {
8443  std::ostringstream os;
8444  os << *verbosePrefix << "Finished 4-arg doPostsAndWaits"
8445  << std::endl;
8446  std::cerr << os.str ();
8447  }
8448  }
8449  else { // constant number of packets per LID
8450  if (verbose) {
8451  std::ostringstream os;
8452  os << *verbosePrefix << "Forward mode, constant # packets / LID"
8453  << std::endl;
8454  std::cerr << os.str ();
8455  }
8456  destMat->imports_.modify_host ();
8457  auto hostImports = destMat->imports_.view_host();
8458  // This is a legacy host pack/unpack path, so use the host
8459  // version of exports_.
8460  destMat->exports_.sync_host ();
8461  auto hostExports = destMat->exports_.view_host();
8462  if (verbose) {
8463  std::ostringstream os;
8464  os << *verbosePrefix << "Calling 3-arg doPostsAndWaits"
8465  << std::endl;
8466  std::cerr << os.str ();
8467  }
8468  Distor.doPostsAndWaits (hostExports,
8469  constantNumPackets,
8470  hostImports);
8471  if (verbose) {
8472  std::ostringstream os;
8473  os << *verbosePrefix << "Finished 3-arg doPostsAndWaits"
8474  << std::endl;
8475  std::cerr << os.str ();
8476  }
8477  }
8478  }
8479  }
8480  }
8481 
8482  /*********************************************************************/
8483  /**** 3) Copy all of the Same/Permute/Remote data into CSR_arrays ****/
8484  /*********************************************************************/
8485 
8486  bool runOnHost = std::is_same_v<typename device_type::memory_space, Kokkos::HostSpace> && !useKokkosPath;
8487 
8488  Teuchos::Array<int> RemotePids;
8489  if (runOnHost) {
8490  Teuchos::Array<int> TargetPids;
8491  // Backwards compatibility measure. We'll use this again below.
8492 
8493  // TODO JHU Need to track down why numImportPacketsPerLID_ has not been corrently marked as modified on host (which it has been)
8494  // TODO JHU somewhere above, e.g., call to Distor.doPostsAndWaits().
8495  // TODO JHU This only becomes apparent as we begin to convert TAFC to run on device.
8496  destMat->numImportPacketsPerLID_.modify_host(); //FIXME
8497 
8498 # ifdef HAVE_TPETRA_MMM_TIMINGS
8499  RCP<TimeMonitor> tmCopySPRdata = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string("TAFC unpack-count-resize + copy same-perm-remote data"))));
8500 # endif
8501  ArrayRCP<size_t> CSR_rowptr;
8502  ArrayRCP<GO> CSR_colind_GID;
8503  ArrayRCP<LO> CSR_colind_LID;
8504  ArrayRCP<Scalar> CSR_vals;
8505 
8506  destMat->imports_.sync_device ();
8507  destMat->numImportPacketsPerLID_.sync_device ();
8508 
8509  size_t N = BaseRowMap->getLocalNumElements ();
8510 
8511  auto RemoteLIDs_d = RemoteLIDs.view_device();
8512  auto PermuteToLIDs_d = PermuteToLIDs.view_device();
8513  auto PermuteFromLIDs_d = PermuteFromLIDs.view_device();
8514 
8515  Details::unpackAndCombineIntoCrsArrays(
8516  *this,
8517  RemoteLIDs_d,
8518  destMat->imports_.view_device(), //hostImports
8519  destMat->numImportPacketsPerLID_.view_device(), //numImportPacketsPerLID
8520  NumSameIDs,
8521  PermuteToLIDs_d,
8522  PermuteFromLIDs_d,
8523  N,
8524  MyPID,
8525  CSR_rowptr,
8526  CSR_colind_GID,
8527  CSR_vals,
8528  SourcePids(),
8529  TargetPids);
8530 
8531  // If LO and GO are the same, we can reuse memory when
8532  // converting the column indices from global to local indices.
8533  if (typeid (LO) == typeid (GO)) {
8534  CSR_colind_LID = Teuchos::arcp_reinterpret_cast<LO> (CSR_colind_GID);
8535  }
8536  else {
8537  CSR_colind_LID.resize (CSR_colind_GID.size());
8538  }
8539  CSR_colind_LID.resize (CSR_colind_GID.size());
8540 
8541  // On return from unpackAndCombineIntoCrsArrays TargetPids[i] == -1 for locally
8542  // owned entries. Convert them to the actual PID.
8543  // JHU FIXME This can be done within unpackAndCombineIntoCrsArrays with a parallel_for.
8544  for(size_t i=0; i<static_cast<size_t>(TargetPids.size()); i++)
8545  {
8546  if(TargetPids[i] == -1) TargetPids[i] = MyPID;
8547  }
8548 #ifdef HAVE_TPETRA_MMM_TIMINGS
8549  tmCopySPRdata = Teuchos::null;
8550 #endif
8551  /**************************************************************/
8552  /**** 4) Call Optimized MakeColMap w/ no Directory Lookups ****/
8553  /**************************************************************/
8554  // Call an optimized version of makeColMap that avoids the
8555  // Directory lookups (since the Import object knows who owns all
8556  // the GIDs).
8557  if (verbose) {
8558  std::ostringstream os;
8559  os << *verbosePrefix << "Calling lowCommunicationMakeColMapAndReindex"
8560  << std::endl;
8561  std::cerr << os.str ();
8562  }
8563  {
8564 #ifdef HAVE_TPETRA_MMM_TIMINGS
8565  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC makeColMap")));
8566 #endif
8567  Import_Util::lowCommunicationMakeColMapAndReindexSerial(CSR_rowptr (),
8568  CSR_colind_LID (),
8569  CSR_colind_GID (),
8570  BaseDomainMap,
8571  TargetPids,
8572  RemotePids,
8573  MyColMap);
8574  }
8575 
8576  if (verbose) {
8577  std::ostringstream os;
8578  os << *verbosePrefix << "restrictComm="
8579  << (restrictComm ? "true" : "false") << std::endl;
8580  std::cerr << os.str ();
8581  }
8582 
8583  /*******************************************************/
8584  /**** 4) Second communicator restriction phase ****/
8585  /*******************************************************/
8586  {
8587 #ifdef HAVE_TPETRA_MMM_TIMINGS
8588  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC restrict colmap")));
8589 #endif
8590  if (restrictComm) {
8591  ReducedColMap = (MyRowMap.getRawPtr () == MyColMap.getRawPtr ()) ?
8592  ReducedRowMap :
8593  MyColMap->replaceCommWithSubset (ReducedComm);
8594  MyColMap = ReducedColMap; // Reset the "my" maps
8595  }
8596 
8597  // Replace the col map
8598  if (verbose) {
8599  std::ostringstream os;
8600  os << *verbosePrefix << "Calling replaceColMap" << std::endl;
8601  std::cerr << os.str ();
8602  }
8603  destMat->replaceColMap (MyColMap);
8604 
8605  // Short circuit if the processor is no longer in the communicator
8606  //
8607  // NOTE: Epetra replaces modifies all "removed" processes so they
8608  // have a dummy (serial) Map that doesn't touch the original
8609  // communicator. Duplicating that here might be a good idea.
8610  if (ReducedComm.is_null ()) {
8611  if (verbose) {
8612  std::ostringstream os;
8613  os << *verbosePrefix << "I am no longer in the communicator; "
8614  "returning" << std::endl;
8615  std::cerr << os.str ();
8616  }
8617  return;
8618  }
8619  }
8620 
8621  /***************************************************/
8622  /**** 5) Sort ****/
8623  /***************************************************/
8624  if ((! reverseMode && xferAsImport != nullptr) ||
8625  (reverseMode && xferAsExport != nullptr)) {
8626  if (verbose) {
8627  std::ostringstream os;
8628  os << *verbosePrefix << "Calling sortCrsEntries" << endl;
8629  std::cerr << os.str ();
8630  }
8631 #ifdef HAVE_TPETRA_MMM_TIMINGS
8632  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC sortCrsEntries")));
8633 #endif
8634  Import_Util::sortCrsEntries (CSR_rowptr(),
8635  CSR_colind_LID(),
8636  CSR_vals());
8637  }
8638  else if ((! reverseMode && xferAsExport != nullptr) ||
8639  (reverseMode && xferAsImport != nullptr)) {
8640  if (verbose) {
8641  std::ostringstream os;
8642  os << *verbosePrefix << "Calling sortAndMergeCrsEntries"
8643  << endl;
8644  std::cerr << os.str();
8645  }
8646 #ifdef HAVE_TPETRA_MMM_TIMINGS
8647  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC sortAndMergeCrsEntries")));
8648 #endif
8649  Import_Util::sortAndMergeCrsEntries (CSR_rowptr(),
8650  CSR_colind_LID(),
8651  CSR_vals());
8652  if (CSR_rowptr[N] != static_cast<size_t>(CSR_vals.size())) {
8653  CSR_colind_LID.resize (CSR_rowptr[N]);
8654  CSR_vals.resize (CSR_rowptr[N]);
8655  }
8656  }
8657  else {
8658  TEUCHOS_TEST_FOR_EXCEPTION(
8659  true, std::logic_error, "Tpetra::CrsMatrix::"
8660  "transferAndFillComplete: Should never get here! "
8661  "Please report this bug to a Tpetra developer.");
8662  }
8663  /***************************************************/
8664  /**** 6) Reset the colmap and the arrays ****/
8665  /***************************************************/
8666 
8667  if (verbose) {
8668  std::ostringstream os;
8669  os << *verbosePrefix << "Calling destMat->setAllValues" << endl;
8670  std::cerr << os.str ();
8671  }
8672 
8673  // Call constructor for the new matrix (restricted as needed)
8674  //
8675  // NOTE (mfh 15 May 2014) This should work fine for the Kokkos
8676  // refactor version of CrsMatrix, though it reserves the right to
8677  // make a deep copy of the arrays.
8678  {
8679 #ifdef HAVE_TPETRA_MMM_TIMINGS
8680  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC setAllValues")));
8681 #endif
8682  destMat->setAllValues (CSR_rowptr, CSR_colind_LID, CSR_vals);
8683  }
8684 
8685  } else {
8686  // run on device
8687 
8688 
8689  // Backwards compatibility measure. We'll use this again below.
8690 
8691  // TODO JHU Need to track down why numImportPacketsPerLID_ has not been corrently marked as modified on host (which it has been)
8692  // TODO JHU somewhere above, e.g., call to Distor.doPostsAndWaits().
8693  // TODO JHU This only becomes apparent as we begin to convert TAFC to run on device.
8694  destMat->numImportPacketsPerLID_.modify_host(); //FIXME
8695 
8696 # ifdef HAVE_TPETRA_MMM_TIMINGS
8697  RCP<TimeMonitor> tmCopySPRdata = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string("TAFC unpack-count-resize + copy same-perm-remote data"))));
8698 # endif
8699  ArrayRCP<size_t> CSR_rowptr;
8700  ArrayRCP<GO> CSR_colind_GID;
8701  ArrayRCP<LO> CSR_colind_LID;
8702  ArrayRCP<Scalar> CSR_vals;
8703 
8704  destMat->imports_.sync_device ();
8705  destMat->numImportPacketsPerLID_.sync_device ();
8706 
8707  size_t N = BaseRowMap->getLocalNumElements ();
8708 
8709  auto RemoteLIDs_d = RemoteLIDs.view_device();
8710  auto PermuteToLIDs_d = PermuteToLIDs.view_device();
8711  auto PermuteFromLIDs_d = PermuteFromLIDs.view_device();
8712 
8713  Kokkos::View<size_t*,device_type> CSR_rowptr_d;
8714  Kokkos::View<GO*,device_type> CSR_colind_GID_d;
8715  Kokkos::View<LO*,device_type> CSR_colind_LID_d;
8716  Kokkos::View<impl_scalar_type*,device_type> CSR_vals_d;
8717  Kokkos::View<int*,device_type> TargetPids_d;
8718 
8719  Details::unpackAndCombineIntoCrsArrays(
8720  *this,
8721  RemoteLIDs_d,
8722  destMat->imports_.view_device(), //hostImports
8723  destMat->numImportPacketsPerLID_.view_device(), //numImportPacketsPerLID
8724  NumSameIDs,
8725  PermuteToLIDs_d,
8726  PermuteFromLIDs_d,
8727  N,
8728  MyPID,
8729  CSR_rowptr_d,
8730  CSR_colind_GID_d,
8731  CSR_vals_d,
8732  SourcePids(),
8733  TargetPids_d);
8734 
8735  Kokkos::resize (CSR_colind_LID_d, CSR_colind_GID_d.size());
8736 
8737 #ifdef HAVE_TPETRA_MMM_TIMINGS
8738  tmCopySPRdata = Teuchos::null;
8739 #endif
8740  /**************************************************************/
8741  /**** 4) Call Optimized MakeColMap w/ no Directory Lookups ****/
8742  /**************************************************************/
8743  // Call an optimized version of makeColMap that avoids the
8744  // Directory lookups (since the Import object knows who owns all
8745  // the GIDs).
8746  if (verbose) {
8747  std::ostringstream os;
8748  os << *verbosePrefix << "Calling lowCommunicationMakeColMapAndReindex"
8749  << std::endl;
8750  std::cerr << os.str ();
8751  }
8752  {
8753 #ifdef HAVE_TPETRA_MMM_TIMINGS
8754  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC makeColMap")));
8755 #endif
8756  Import_Util::lowCommunicationMakeColMapAndReindex(CSR_rowptr_d,
8757  CSR_colind_LID_d,
8758  CSR_colind_GID_d,
8759  BaseDomainMap,
8760  TargetPids_d,
8761  RemotePids,
8762  MyColMap);
8763  }
8764 
8765  if (verbose) {
8766  std::ostringstream os;
8767  os << *verbosePrefix << "restrictComm="
8768  << (restrictComm ? "true" : "false") << std::endl;
8769  std::cerr << os.str ();
8770  }
8771 
8772  /*******************************************************/
8773  /**** 4) Second communicator restriction phase ****/
8774  /*******************************************************/
8775  {
8776 #ifdef HAVE_TPETRA_MMM_TIMINGS
8777  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC restrict colmap")));
8778 #endif
8779  if (restrictComm) {
8780  ReducedColMap = (MyRowMap.getRawPtr () == MyColMap.getRawPtr ()) ?
8781  ReducedRowMap :
8782  MyColMap->replaceCommWithSubset (ReducedComm);
8783  MyColMap = ReducedColMap; // Reset the "my" maps
8784  }
8785 
8786  // Replace the col map
8787  if (verbose) {
8788  std::ostringstream os;
8789  os << *verbosePrefix << "Calling replaceColMap" << std::endl;
8790  std::cerr << os.str ();
8791  }
8792  destMat->replaceColMap (MyColMap);
8793 
8794  // Short circuit if the processor is no longer in the communicator
8795  //
8796  // NOTE: Epetra replaces modifies all "removed" processes so they
8797  // have a dummy (serial) Map that doesn't touch the original
8798  // communicator. Duplicating that here might be a good idea.
8799  if (ReducedComm.is_null ()) {
8800  if (verbose) {
8801  std::ostringstream os;
8802  os << *verbosePrefix << "I am no longer in the communicator; "
8803  "returning" << std::endl;
8804  std::cerr << os.str ();
8805  }
8806  return;
8807  }
8808  }
8809 
8810  /***************************************************/
8811  /**** 5) Sort ****/
8812  /***************************************************/
8813 
8814  if ((! reverseMode && xferAsImport != nullptr) ||
8815  (reverseMode && xferAsExport != nullptr)) {
8816  if (verbose) {
8817  std::ostringstream os;
8818  os << *verbosePrefix << "Calling sortCrsEntries" << endl;
8819  std::cerr << os.str ();
8820  }
8821 #ifdef HAVE_TPETRA_MMM_TIMINGS
8822  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC sortCrsEntries")));
8823 #endif
8824  Import_Util::sortCrsEntries (CSR_rowptr_d,
8825  CSR_colind_LID_d,
8826  CSR_vals_d);
8827  }
8828  else if ((! reverseMode && xferAsExport != nullptr) ||
8829  (reverseMode && xferAsImport != nullptr)) {
8830  if (verbose) {
8831  std::ostringstream os;
8832  os << *verbosePrefix << "Calling sortAndMergeCrsEntries"
8833  << endl;
8834  std::cerr << os.str();
8835  }
8836 #ifdef HAVE_TPETRA_MMM_TIMINGS
8837  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC sortAndMergeCrsEntries")));
8838 #endif
8839  Import_Util::sortAndMergeCrsEntries (CSR_rowptr_d,
8840  CSR_colind_LID_d,
8841  CSR_vals_d);
8842  }
8843  else {
8844  TEUCHOS_TEST_FOR_EXCEPTION(
8845  true, std::logic_error, "Tpetra::CrsMatrix::"
8846  "transferAndFillComplete: Should never get here! "
8847  "Please report this bug to a Tpetra developer.");
8848  }
8849 
8850  /***************************************************/
8851  /**** 6) Reset the colmap and the arrays ****/
8852  /***************************************************/
8853 
8854  if (verbose) {
8855  std::ostringstream os;
8856  os << *verbosePrefix << "Calling destMat->setAllValues" << endl;
8857  std::cerr << os.str ();
8858  }
8859 
8860  {
8861 #ifdef HAVE_TPETRA_MMM_TIMINGS
8862  Teuchos::TimeMonitor MMrc(*TimeMonitor::getNewTimer(prefix + std::string("TAFC setAllValues")));
8863 #endif
8864  destMat->setAllValues (CSR_rowptr_d, CSR_colind_LID_d, CSR_vals_d);
8865  }
8866 
8867  } //if (runOnHost) .. else ..
8868 
8869  /***************************************************/
8870  /**** 7) Build Importer & Call ESFC ****/
8871  /***************************************************/
8872 #ifdef HAVE_TPETRA_MMM_TIMINGS
8873  RCP<TimeMonitor> tmIESFC = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix + std::string("TAFC build importer and esfc"))));
8874 #endif
8875  // Pre-build the importer using the existing PIDs
8876  Teuchos::ParameterList esfc_params;
8877 
8878  RCP<import_type> MyImport;
8879 
8880  // Fulfull the non-blocking allreduce on reduced_mismatch.
8881  if (iallreduceRequest.get () != nullptr) {
8882  if (verbose) {
8883  std::ostringstream os;
8884  os << *verbosePrefix << "Calling iallreduceRequest->wait()"
8885  << endl;
8886  std::cerr << os.str ();
8887  }
8888  iallreduceRequest->wait ();
8889  if (reduced_mismatch != 0) {
8890  isMM = false;
8891  }
8892  }
8893 
8894  if( isMM ) {
8895 #ifdef HAVE_TPETRA_MMM_TIMINGS
8896  Teuchos::TimeMonitor MMisMM (*TimeMonitor::getNewTimer(prefix + std::string("isMM Block")));
8897 #endif
8898  // Combine all type1/2/3 lists, [filter them], then call the expert import constructor.
8899 
8900  if (verbose) {
8901  std::ostringstream os;
8902  os << *verbosePrefix << "Getting CRS pointers" << endl;
8903  std::cerr << os.str ();
8904  }
8905 
8906  Teuchos::ArrayRCP<LocalOrdinal> type3LIDs;
8907  Teuchos::ArrayRCP<int> type3PIDs;
8908  auto rowptr = getCrsGraph()->getLocalRowPtrsHost();
8909  auto colind = getCrsGraph()->getLocalIndicesHost();
8910 
8911  if (verbose) {
8912  std::ostringstream os;
8913  os << *verbosePrefix << "Calling reverseNeighborDiscovery" << std::endl;
8914  std::cerr << os.str ();
8915  }
8916 
8917  {
8918 #ifdef HAVE_TPETRA_MMM_TIMINGS
8919  TimeMonitor tm_rnd (*TimeMonitor::getNewTimer(prefix + std::string("isMMrevNeighDis")));
8920 #endif
8921  Import_Util::reverseNeighborDiscovery(*this,
8922  rowptr,
8923  colind,
8924  rowTransfer,
8925  MyImporter,
8926  MyDomainMap,
8927  type3PIDs,
8928  type3LIDs,
8929  ReducedComm);
8930  }
8931 
8932  if (verbose) {
8933  std::ostringstream os;
8934  os << *verbosePrefix << "Done with reverseNeighborDiscovery" << std::endl;
8935  std::cerr << os.str ();
8936  }
8937 
8938  Teuchos::ArrayView<const int> EPID1 = MyImporter.is_null() ? Teuchos::ArrayView<const int>() : MyImporter->getExportPIDs();
8939  Teuchos::ArrayView<const LO> ELID1 = MyImporter.is_null() ? Teuchos::ArrayView<const LO>() : MyImporter->getExportLIDs();
8940 
8941  Teuchos::ArrayView<const int> TEPID2 = rowTransfer.getExportPIDs(); // row matrix
8942  Teuchos::ArrayView<const LO> TELID2 = rowTransfer.getExportLIDs();
8943 
8944  const int numCols = getGraph()->getColMap()->getLocalNumElements(); // may be dup
8945  // from EpetraExt_MMHelpers.cpp: build_type2_exports
8946  std::vector<bool> IsOwned(numCols,true);
8947  std::vector<int> SentTo(numCols,-1);
8948  if (! MyImporter.is_null ()) {
8949  for (auto && rlid : MyImporter->getRemoteLIDs()) { // the remoteLIDs must be from sourcematrix
8950  IsOwned[rlid]=false;
8951  }
8952  }
8953 
8954  std::vector<std::pair<int,GO> > usrtg;
8955  usrtg.reserve(TEPID2.size());
8956 
8957  {
8958  const auto& colMap = * (this->getColMap ()); // *this is sourcematrix
8959  for (Array_size_type i = 0; i < TEPID2.size (); ++i) {
8960  const LO row = TELID2[i];
8961  const int pid = TEPID2[i];
8962  for (auto j = rowptr[row]; j < rowptr[row+1]; ++j) {
8963  const int col = colind[j];
8964  if (IsOwned[col] && SentTo[col] != pid) {
8965  SentTo[col] = pid;
8966  GO gid = colMap.getGlobalElement (col);
8967  usrtg.push_back (std::pair<int,GO> (pid, gid));
8968  }
8969  }
8970  }
8971  }
8972 
8973 // This sort can _not_ be omitted.[
8974  std::sort(usrtg.begin(),usrtg.end()); // default comparator does the right thing, now sorted in gid order
8975  auto eopg = std ::unique(usrtg.begin(),usrtg.end());
8976  // 25 Jul 2018: Could just ignore the entries at and after eopg.
8977  usrtg.erase(eopg,usrtg.end());
8978 
8979  const Array_size_type type2_us_size = usrtg.size();
8980  Teuchos::ArrayRCP<int> EPID2=Teuchos::arcp(new int[type2_us_size],0,type2_us_size,true);
8981  Teuchos::ArrayRCP< LO> ELID2=Teuchos::arcp(new LO[type2_us_size],0,type2_us_size,true);
8982 
8983  int pos=0;
8984  for(auto && p : usrtg) {
8985  EPID2[pos]= p.first;
8986  ELID2[pos]= this->getDomainMap()->getLocalElement(p.second);
8987  pos++;
8988  }
8989 
8990  Teuchos::ArrayView<int> EPID3 = type3PIDs();
8991  Teuchos::ArrayView< LO> ELID3 = type3LIDs();
8992  GO InfGID = std::numeric_limits<GO>::max();
8993  int InfPID = INT_MAX;
8994 #ifdef TPETRA_MIN3
8995 # undef TPETRA_MIN3
8996 #endif // TPETRA_MIN3
8997 #define TPETRA_MIN3(x,y,z) ((x)<(y)?(std::min(x,z)):(std::min(y,z)))
8998  int i1=0, i2=0, i3=0;
8999  int Len1 = EPID1.size();
9000  int Len2 = EPID2.size();
9001  int Len3 = EPID3.size();
9002 
9003  int MyLen=Len1+Len2+Len3;
9004  Teuchos::ArrayRCP<LO> userExportLIDs = Teuchos::arcp(new LO[MyLen],0,MyLen,true);
9005  Teuchos::ArrayRCP<int> userExportPIDs = Teuchos::arcp(new int[MyLen],0,MyLen,true);
9006  int iloc = 0; // will be the size of the userExportLID/PIDs
9007 
9008  while(i1 < Len1 || i2 < Len2 || i3 < Len3){
9009  int PID1 = (i1<Len1)?(EPID1[i1]):InfPID;
9010  int PID2 = (i2<Len2)?(EPID2[i2]):InfPID;
9011  int PID3 = (i3<Len3)?(EPID3[i3]):InfPID;
9012 
9013  GO GID1 = (i1<Len1)?getDomainMap()->getGlobalElement(ELID1[i1]):InfGID;
9014  GO GID2 = (i2<Len2)?getDomainMap()->getGlobalElement(ELID2[i2]):InfGID;
9015  GO GID3 = (i3<Len3)?getDomainMap()->getGlobalElement(ELID3[i3]):InfGID;
9016 
9017  int MIN_PID = TPETRA_MIN3(PID1,PID2,PID3);
9018  GO MIN_GID = TPETRA_MIN3( ((PID1==MIN_PID)?GID1:InfGID), ((PID2==MIN_PID)?GID2:InfGID), ((PID3==MIN_PID)?GID3:InfGID));
9019 #ifdef TPETRA_MIN3
9020 # undef TPETRA_MIN3
9021 #endif // TPETRA_MIN3
9022  bool added_entry=false;
9023 
9024  if(PID1 == MIN_PID && GID1 == MIN_GID){
9025  userExportLIDs[iloc]=ELID1[i1];
9026  userExportPIDs[iloc]=EPID1[i1];
9027  i1++;
9028  added_entry=true;
9029  iloc++;
9030  }
9031  if(PID2 == MIN_PID && GID2 == MIN_GID){
9032  if(!added_entry) {
9033  userExportLIDs[iloc]=ELID2[i2];
9034  userExportPIDs[iloc]=EPID2[i2];
9035  added_entry=true;
9036  iloc++;
9037  }
9038  i2++;
9039  }
9040  if(PID3 == MIN_PID && GID3 == MIN_GID){
9041  if(!added_entry) {
9042  userExportLIDs[iloc]=ELID3[i3];
9043  userExportPIDs[iloc]=EPID3[i3];
9044  iloc++;
9045  }
9046  i3++;
9047  }
9048  }
9049 
9050  if (verbose) {
9051  std::ostringstream os;
9052  os << *verbosePrefix << "Create Import" << std::endl;
9053  std::cerr << os.str ();
9054  }
9055 
9056 #ifdef HAVE_TPETRA_MMM_TIMINGS
9057  auto ismmIctor(*TimeMonitor::getNewTimer(prefix + std::string("isMMIportCtor")));
9058 #endif
9059  Teuchos::RCP<Teuchos::ParameterList> plist = rcp(new Teuchos::ParameterList());
9060  // 25 Jul 2018: Test for equality with the non-isMM path's Import object.
9061  if ((MyDomainMap != MyColMap) && (!MyDomainMap->isSameAs(*MyColMap)))
9062  MyImport = rcp ( new import_type (MyDomainMap,
9063  MyColMap,
9064  RemotePids,
9065  userExportLIDs.view(0,iloc).getConst(),
9066  userExportPIDs.view(0,iloc).getConst(),
9067  plist)
9068  );
9069 
9070  if (verbose) {
9071  std::ostringstream os;
9072  os << *verbosePrefix << "Call expertStaticFillComplete" << std::endl;
9073  std::cerr << os.str ();
9074  }
9075 
9076  {
9077 #ifdef HAVE_TPETRA_MMM_TIMINGS
9078  TimeMonitor esfc (*TimeMonitor::getNewTimer(prefix + std::string("isMM::destMat->eSFC")));
9079  esfc_params.set("Timer Label",label+std::string("isMM eSFC"));
9080 #endif
9081  if(!params.is_null())
9082  esfc_params.set("compute global constants",params->get("compute global constants",true));
9083  destMat->expertStaticFillComplete (MyDomainMap, MyRangeMap, MyImport,Teuchos::null,rcp(new Teuchos::ParameterList(esfc_params)));
9084 
9085  }
9086 
9087  } // if(isMM)
9088  else {
9089 #ifdef HAVE_TPETRA_MMM_TIMINGS
9090  TimeMonitor MMnotMMblock (*TimeMonitor::getNewTimer(prefix + std::string("TAFC notMMblock")));
9091 #endif
9092  if (verbose) {
9093  std::ostringstream os;
9094  os << *verbosePrefix << "Create Import" << std::endl;
9095  std::cerr << os.str ();
9096  }
9097 
9098 #ifdef HAVE_TPETRA_MMM_TIMINGS
9099  TimeMonitor notMMIcTor(*TimeMonitor::getNewTimer(prefix + std::string("TAFC notMMCreateImporter")));
9100 #endif
9101  Teuchos::RCP<Teuchos::ParameterList> mypars = rcp(new Teuchos::ParameterList);
9102  mypars->set("Timer Label","notMMFrom_tAFC");
9103  if ((MyDomainMap != MyColMap) && (!MyDomainMap->isSameAs(*MyColMap)))
9104  MyImport = rcp (new import_type (MyDomainMap, MyColMap, RemotePids, mypars));
9105 
9106  if (verbose) {
9107  std::ostringstream os;
9108  os << *verbosePrefix << "Call expertStaticFillComplete" << endl;
9109  std::cerr << os.str ();
9110  }
9111 
9112 #ifdef HAVE_TPETRA_MMM_TIMINGS
9113  TimeMonitor esfcnotmm(*TimeMonitor::getNewTimer(prefix + std::string("notMMdestMat->expertStaticFillComplete")));
9114  esfc_params.set("Timer Label",prefix+std::string("notMM eSFC"));
9115 #else
9116  esfc_params.set("Timer Label",std::string("notMM eSFC"));
9117 #endif
9118 
9119  if (!params.is_null ()) {
9120  esfc_params.set ("compute global constants",
9121  params->get ("compute global constants", true));
9122  }
9123  destMat->expertStaticFillComplete (MyDomainMap, MyRangeMap,
9124  MyImport, Teuchos::null,
9125  rcp (new Teuchos::ParameterList (esfc_params)));
9126  }
9127 
9128 #ifdef HAVE_TPETRA_MMM_TIMINGS
9129  tmIESFC = Teuchos::null;
9130 #endif
9131 
9132  if (verbose) {
9133  std::ostringstream os;
9134  os << *verbosePrefix << "Done" << endl;
9135  std::cerr << os.str ();
9136  }
9137  } //transferAndFillComplete
9138 
9139 
9140  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
9141  void
9142  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
9143  importAndFillComplete (Teuchos::RCP<CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> >& destMatrix,
9144  const import_type& importer,
9145  const Teuchos::RCP<const map_type>& domainMap,
9146  const Teuchos::RCP<const map_type>& rangeMap,
9147  const Teuchos::RCP<Teuchos::ParameterList>& params) const
9148  {
9149  transferAndFillComplete (destMatrix, importer, Teuchos::null, domainMap, rangeMap, params);
9150  }
9151 
9152  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
9153  void
9154  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
9155  importAndFillComplete (Teuchos::RCP<CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> >& destMatrix,
9156  const import_type& rowImporter,
9157  const import_type& domainImporter,
9158  const Teuchos::RCP<const map_type>& domainMap,
9159  const Teuchos::RCP<const map_type>& rangeMap,
9160  const Teuchos::RCP<Teuchos::ParameterList>& params) const
9161  {
9162  transferAndFillComplete (destMatrix, rowImporter, Teuchos::rcpFromRef(domainImporter), domainMap, rangeMap, params);
9163  }
9164 
9165  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
9166  void
9167  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
9168  exportAndFillComplete (Teuchos::RCP<CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> >& destMatrix,
9169  const export_type& exporter,
9170  const Teuchos::RCP<const map_type>& domainMap,
9171  const Teuchos::RCP<const map_type>& rangeMap,
9172  const Teuchos::RCP<Teuchos::ParameterList>& params) const
9173  {
9174  transferAndFillComplete (destMatrix, exporter, Teuchos::null, domainMap, rangeMap, params);
9175  }
9176 
9177  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
9178  void
9179  CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
9180  exportAndFillComplete (Teuchos::RCP<CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> >& destMatrix,
9181  const export_type& rowExporter,
9182  const export_type& domainExporter,
9183  const Teuchos::RCP<const map_type>& domainMap,
9184  const Teuchos::RCP<const map_type>& rangeMap,
9185  const Teuchos::RCP<Teuchos::ParameterList>& params) const
9186  {
9187  transferAndFillComplete (destMatrix, rowExporter, Teuchos::rcpFromRef(domainExporter), domainMap, rangeMap, params);
9188  }
9189 
9190 } // namespace Tpetra
9191 
9192 //
9193 // Explicit instantiation macro
9194 //
9195 // Must be expanded from within the Tpetra namespace!
9196 //
9197 
9198 #define TPETRA_CRSMATRIX_MATRIX_INSTANT(SCALAR,LO,GO,NODE) \
9199  \
9200  template class CrsMatrix< SCALAR , LO , GO , NODE >;
9201 
9202 #define TPETRA_CRSMATRIX_CONVERT_INSTANT(SO,SI,LO,GO,NODE) \
9203  \
9204  template Teuchos::RCP< CrsMatrix< SO , LO , GO , NODE > > \
9205  CrsMatrix< SI , LO , GO , NODE >::convert< SO > () const;
9206 
9207 #define TPETRA_CRSMATRIX_IMPORT_AND_FILL_COMPLETE_INSTANT(SCALAR, LO, GO, NODE) \
9208  template<> \
9209  Teuchos::RCP<CrsMatrix<SCALAR, LO, GO, NODE> > \
9210  importAndFillCompleteCrsMatrix (const Teuchos::RCP<const CrsMatrix<SCALAR, LO, GO, NODE> >& sourceMatrix, \
9211  const Import<CrsMatrix<SCALAR, LO, GO, NODE>::local_ordinal_type, \
9212  CrsMatrix<SCALAR, LO, GO, NODE>::global_ordinal_type, \
9213  CrsMatrix<SCALAR, LO, GO, NODE>::node_type>& importer, \
9214  const Teuchos::RCP<const Map<CrsMatrix<SCALAR, LO, GO, NODE>::local_ordinal_type, \
9215  CrsMatrix<SCALAR, LO, GO, NODE>::global_ordinal_type, \
9216  CrsMatrix<SCALAR, LO, GO, NODE>::node_type> >& domainMap, \
9217  const Teuchos::RCP<const Map<CrsMatrix<SCALAR, LO, GO, NODE>::local_ordinal_type, \
9218  CrsMatrix<SCALAR, LO, GO, NODE>::global_ordinal_type, \
9219  CrsMatrix<SCALAR, LO, GO, NODE>::node_type> >& rangeMap, \
9220  const Teuchos::RCP<Teuchos::ParameterList>& params);
9221 
9222 #define TPETRA_CRSMATRIX_IMPORT_AND_FILL_COMPLETE_INSTANT_TWO(SCALAR, LO, GO, NODE) \
9223  template<> \
9224  Teuchos::RCP<CrsMatrix<SCALAR, LO, GO, NODE> > \
9225  importAndFillCompleteCrsMatrix (const Teuchos::RCP<const CrsMatrix<SCALAR, LO, GO, NODE> >& sourceMatrix, \
9226  const Import<CrsMatrix<SCALAR, LO, GO, NODE>::local_ordinal_type, \
9227  CrsMatrix<SCALAR, LO, GO, NODE>::global_ordinal_type, \
9228  CrsMatrix<SCALAR, LO, GO, NODE>::node_type>& rowImporter, \
9229  const Import<CrsMatrix<SCALAR, LO, GO, NODE>::local_ordinal_type, \
9230  CrsMatrix<SCALAR, LO, GO, NODE>::global_ordinal_type, \
9231  CrsMatrix<SCALAR, LO, GO, NODE>::node_type>& domainImporter, \
9232  const Teuchos::RCP<const Map<CrsMatrix<SCALAR, LO, GO, NODE>::local_ordinal_type, \
9233  CrsMatrix<SCALAR, LO, GO, NODE>::global_ordinal_type, \
9234  CrsMatrix<SCALAR, LO, GO, NODE>::node_type> >& domainMap, \
9235  const Teuchos::RCP<const Map<CrsMatrix<SCALAR, LO, GO, NODE>::local_ordinal_type, \
9236  CrsMatrix<SCALAR, LO, GO, NODE>::global_ordinal_type, \
9237  CrsMatrix<SCALAR, LO, GO, NODE>::node_type> >& rangeMap, \
9238  const Teuchos::RCP<Teuchos::ParameterList>& params);
9239 
9240 
9241 #define TPETRA_CRSMATRIX_EXPORT_AND_FILL_COMPLETE_INSTANT(SCALAR, LO, GO, NODE) \
9242  template<> \
9243  Teuchos::RCP<CrsMatrix<SCALAR, LO, GO, NODE> > \
9244  exportAndFillCompleteCrsMatrix (const Teuchos::RCP<const CrsMatrix<SCALAR, LO, GO, NODE> >& sourceMatrix, \
9245  const Export<CrsMatrix<SCALAR, LO, GO, NODE>::local_ordinal_type, \
9246  CrsMatrix<SCALAR, LO, GO, NODE>::global_ordinal_type, \
9247  CrsMatrix<SCALAR, LO, GO, NODE>::node_type>& exporter, \
9248  const Teuchos::RCP<const Map<CrsMatrix<SCALAR, LO, GO, NODE>::local_ordinal_type, \
9249  CrsMatrix<SCALAR, LO, GO, NODE>::global_ordinal_type, \
9250  CrsMatrix<SCALAR, LO, GO, NODE>::node_type> >& domainMap, \
9251  const Teuchos::RCP<const Map<CrsMatrix<SCALAR, LO, GO, NODE>::local_ordinal_type, \
9252  CrsMatrix<SCALAR, LO, GO, NODE>::global_ordinal_type, \
9253  CrsMatrix<SCALAR, LO, GO, NODE>::node_type> >& rangeMap, \
9254  const Teuchos::RCP<Teuchos::ParameterList>& params);
9255 
9256 #define TPETRA_CRSMATRIX_EXPORT_AND_FILL_COMPLETE_INSTANT_TWO(SCALAR, LO, GO, NODE) \
9257  template<> \
9258  Teuchos::RCP<CrsMatrix<SCALAR, LO, GO, NODE> > \
9259  exportAndFillCompleteCrsMatrix (const Teuchos::RCP<const CrsMatrix<SCALAR, LO, GO, NODE> >& sourceMatrix, \
9260  const Export<CrsMatrix<SCALAR, LO, GO, NODE>::local_ordinal_type, \
9261  CrsMatrix<SCALAR, LO, GO, NODE>::global_ordinal_type, \
9262  CrsMatrix<SCALAR, LO, GO, NODE>::node_type>& rowExporter, \
9263  const Export<CrsMatrix<SCALAR, LO, GO, NODE>::local_ordinal_type, \
9264  CrsMatrix<SCALAR, LO, GO, NODE>::global_ordinal_type, \
9265  CrsMatrix<SCALAR, LO, GO, NODE>::node_type>& domainExporter, \
9266  const Teuchos::RCP<const Map<CrsMatrix<SCALAR, LO, GO, NODE>::local_ordinal_type, \
9267  CrsMatrix<SCALAR, LO, GO, NODE>::global_ordinal_type, \
9268  CrsMatrix<SCALAR, LO, GO, NODE>::node_type> >& domainMap, \
9269  const Teuchos::RCP<const Map<CrsMatrix<SCALAR, LO, GO, NODE>::local_ordinal_type, \
9270  CrsMatrix<SCALAR, LO, GO, NODE>::global_ordinal_type, \
9271  CrsMatrix<SCALAR, LO, GO, NODE>::node_type> >& rangeMap, \
9272  const Teuchos::RCP<Teuchos::ParameterList>& params);
9273 
9274 
9275 #define TPETRA_CRSMATRIX_INSTANT(SCALAR, LO, GO ,NODE) \
9276  TPETRA_CRSMATRIX_MATRIX_INSTANT(SCALAR, LO, GO, NODE) \
9277  TPETRA_CRSMATRIX_IMPORT_AND_FILL_COMPLETE_INSTANT(SCALAR, LO, GO, NODE) \
9278  TPETRA_CRSMATRIX_EXPORT_AND_FILL_COMPLETE_INSTANT(SCALAR, LO, GO, NODE) \
9279  TPETRA_CRSMATRIX_IMPORT_AND_FILL_COMPLETE_INSTANT_TWO(SCALAR, LO, GO, NODE) \
9280  TPETRA_CRSMATRIX_EXPORT_AND_FILL_COMPLETE_INSTANT_TWO(SCALAR, LO, GO, NODE)
9281 
9282 #endif // TPETRA_CRSMATRIX_DEF_HPP
Tpetra::Import
Communication plan for data redistribution from a uniquely-owned to a (possibly) multiply-owned distr...
Definition: Tpetra_Import_decl.hpp:77
Tpetra::CrsGraph::getRowMap
Teuchos::RCP< const map_type > getRowMap() const override
Returns the Map that describes the row distribution in this graph.
Definition: Tpetra_CrsGraph_def.hpp:878
Tpetra::CrsMatrix::hasColMap
bool hasColMap() const override
Whether the matrix has a well-defined column Map.
Definition: Tpetra_CrsMatrix_def.hpp:822
Tpetra_Details_ScalarViewTraits.hpp
Declaration and generic definition of traits class that tells Tpetra::CrsMatrix how to pack and unpac...
Tpetra::CrsGraph::indicesAreSorted_
bool indicesAreSorted_
Whether the graph&#39;s indices are sorted in each row, on this process.
Definition: Tpetra_CrsGraph_decl.hpp:2478
Tpetra::CrsMatrix::getGlobalNumCols
global_size_t getGlobalNumCols() const override
The number of global columns in the matrix.
Definition: Tpetra_CrsMatrix_def.hpp:850
Tpetra::Details::create_mirror_view_from_raw_host_array
Impl::CreateMirrorViewFromUnmanagedHostArray< ValueType, OutputDeviceType >::output_view_type create_mirror_view_from_raw_host_array(const OutputDeviceType &, ValueType *inPtr, const size_t inSize, const bool copy=true, const char label[]="")
Variant of Kokkos::create_mirror_view that takes a raw host 1-d array as input.
Definition: Tpetra_Details_createMirrorView.hpp:173
Tpetra::Details::AbsMax
Functor for the the ABSMAX CombineMode of Import and Export operations.
Definition: Tpetra_CrsMatrix_def.hpp:99
Tpetra::CrsMatrix::checkInternalState
void checkInternalState() const
Check that this object&#39;s state is sane; throw if it&#39;s not.
Definition: Tpetra_CrsMatrix_def.hpp:5183
Tpetra::CrsMatrix
Sparse matrix that presents a row-oriented interface that lets users read or modify entries...
Definition: Tpetra_CrsMatrix_decl.hpp:402
Tpetra::Details::copyOffsets
void copyOffsets(const OutputViewType &dst, const InputViewType &src)
Copy row offsets (in a sparse graph or matrix) from src to dst. The offsets may have different types...
Definition: Tpetra_Details_copyOffsets.hpp:503
Tpetra::CrsMatrix::crs_graph_type
CrsGraph< LocalOrdinal, GlobalOrdinal, Node > crs_graph_type
The CrsGraph specialization suitable for this CrsMatrix specialization.
Definition: Tpetra_CrsMatrix_decl.hpp:468
Tpetra::CrsMatrix::replaceColMap
void replaceColMap(const Teuchos::RCP< const map_type > &newColMap)
Replace the matrix&#39;s column Map with the given Map.
Definition: Tpetra_CrsMatrix_def.hpp:3878
Tpetra::CrsMatrix::replaceGlobalValuesImpl
virtual LocalOrdinal replaceGlobalValuesImpl(impl_scalar_type rowVals[], const crs_graph_type &graph, const RowInfo &rowInfo, const GlobalOrdinal inds[], const impl_scalar_type newVals[], const LocalOrdinal numElts)
Implementation detail of replaceGlobalValues.
Definition: Tpetra_CrsMatrix_def.hpp:2419
Tpetra::CrsMatrix::supportsRowViews
virtual bool supportsRowViews() const override
Return true if getLocalRowView() and getGlobalRowView() are valid for this object.
Definition: Tpetra_CrsMatrix_def.hpp:1056
Tpetra::CrsMatrix::replaceDomainMapAndImporter
void replaceDomainMapAndImporter(const Teuchos::RCP< const map_type > &newDomainMap, Teuchos::RCP< const import_type > &newImporter)
Replace the current domain Map and Import with the given objects.
Definition: Tpetra_CrsMatrix_def.hpp:3947
Tpetra::CrsGraph::getLocalIndsViewHost
local_inds_dualv_type::t_host::const_type getLocalIndsViewHost(const RowInfo &rowinfo) const
Get a const, locally indexed view of the locally owned row myRow, such that rowinfo = getRowInfo(myRo...
Definition: Tpetra_CrsGraph_def.hpp:1367
Tpetra::merge2
void merge2(IT1 &indResultOut, IT2 &valResultOut, IT1 indBeg, IT1 indEnd, IT2 valBeg, IT2)
Merge values in place, additively, with the same index.
Definition: Tpetra_Util.hpp:729
Tpetra::CrsMatrix::mergeRowIndicesAndValues
static size_t mergeRowIndicesAndValues(size_t rowLen, local_ordinal_type *cols, impl_scalar_type *vals)
Merge duplicate row indices in the given row, along with their corresponding values.
Definition: Tpetra_CrsMatrix_def.hpp:4579
Tpetra::CrsMatrix::getGlobalRowCopy
void getGlobalRowCopy(GlobalOrdinal GlobalRow, nonconst_global_inds_host_view_type &Indices, nonconst_values_host_view_type &Values, size_t &NumEntries) const override
Fill given arrays with a deep copy of the locally owned entries of the matrix in a given row...
Definition: Tpetra_CrsMatrix_def.hpp:3213
Tpetra::CrsMatrix::getRangeMap
Teuchos::RCP< const map_type > getRangeMap() const override
The range Map of this matrix.
Definition: Tpetra_CrsMatrix_def.hpp:929
Tpetra::CrsMatrix::getGlobalNumRows
global_size_t getGlobalNumRows() const override
Number of global elements in the row map of this matrix.
Definition: Tpetra_CrsMatrix_def.hpp:843
Tpetra::CrsGraph::insertGlobalIndicesImpl
size_t insertGlobalIndicesImpl(const local_ordinal_type lclRow, const global_ordinal_type inputGblColInds[], const size_t numInputInds)
Insert global indices, using an input local row index.
Definition: Tpetra_CrsGraph_def.hpp:1691
Tpetra::CrsMatrix::nonlocals_
std::map< GlobalOrdinal, std::pair< Teuchos::Array< GlobalOrdinal >, Teuchos::Array< Scalar > > > nonlocals_
Nonlocal data added using insertGlobalValues().
Definition: Tpetra_CrsMatrix_decl.hpp:4062
Tpetra::Details::PackTraits< LO >::unpackValue
static KOKKOS_INLINE_FUNCTION size_t unpackValue(LO &outVal, const char inBuf[])
Unpack the given value from the given output buffer.
Definition: Tpetra_Details_PackTraits.hpp:277
Tpetra::CrsMatrix::sortAndMergeIndicesAndValues
void sortAndMergeIndicesAndValues(const bool sorted, const bool merged)
Sort and merge duplicate local column indices in all rows on the calling process, along with their co...
Definition: Tpetra_CrsMatrix_def.hpp:4615
Tpetra::CrsMatrix::execution_space
typename device_type::execution_space execution_space
The Kokkos execution space.
Definition: Tpetra_CrsMatrix_decl.hpp:428
Tpetra::CrsMatrix::packNew
void packNew(const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &exportLIDs, Kokkos::DualView< char *, buffer_device_type > &exports, const Kokkos::DualView< size_t *, buffer_device_type > &numPacketsPerLID, size_t &constantNumPackets) const
Pack this object&#39;s data for an Import or Export.
Definition: Tpetra_CrsMatrix_def.hpp:6591
Tpetra::CrsMatrix::insertGlobalValuesImpl
virtual void insertGlobalValuesImpl(crs_graph_type &graph, RowInfo &rowInfo, const GlobalOrdinal gblColInds[], const impl_scalar_type vals[], const size_t numInputEnt)
Common implementation detail of insertGlobalValues and insertGlobalValuesFiltered.
Definition: Tpetra_CrsMatrix_def.hpp:1856
Tpetra_Details_Profiling.hpp
Declaration of Tpetra::Details::Profiling, a scope guard for Kokkos Profiling.
Tpetra::CrsGraph::getNumEntriesInLocalRow
size_t getNumEntriesInLocalRow(local_ordinal_type localRow) const override
Get the number of entries in the given row (local index).
Definition: Tpetra_CrsGraph_def.hpp:2185
Tpetra::CrsMatrix::mag_type
typename Kokkos::ArithTraits< impl_scalar_type >::mag_type mag_type
Type of a norm result.
Definition: Tpetra_CrsMatrix_decl.hpp:465
Tpetra::MultiVector::putScalar
void putScalar(const Scalar &value)
Set all values in the multivector with the given value.
Definition: Tpetra_MultiVector_def.hpp:2694
Tpetra::MultiVector::getNumVectors
size_t getNumVectors() const
Number of columns in the multivector.
Definition: Tpetra_MultiVector_def.hpp:2206
Tpetra::CrsMatrix::getGlobalRowView
void getGlobalRowView(GlobalOrdinal GlobalRow, global_inds_host_view_type &indices, values_host_view_type &values) const override
Get a constant, nonpersisting view of a row of this matrix, using global row and column indices...
Definition: Tpetra_CrsMatrix_def.hpp:3319
Tpetra::MultiVector::getLocalLength
size_t getLocalLength() const
Local number of rows on the calling process.
Definition: Tpetra_MultiVector_def.hpp:1017
Tpetra_Details_gathervPrint.hpp
Declaration of a function that prints strings from each process.
Tpetra::CrsMatrix::setAllToScalar
void setAllToScalar(const Scalar &alpha)
Set all matrix entries equal to alpha.
Definition: Tpetra_CrsMatrix_def.hpp:3397
Tpetra::MultiVector::isConstantStride
bool isConstantStride() const
Whether this multivector has constant stride between columns.
Definition: Tpetra_MultiVector_def.hpp:1010
Tpetra::Details::PackTraits
Traits class for packing / unpacking data of type T.
Definition: Tpetra_Details_PackTraits.hpp:29
Tpetra::CrsMatrix::replaceRangeMapAndExporter
void replaceRangeMapAndExporter(const Teuchos::RCP< const map_type > &newRangeMap, Teuchos::RCP< const export_type > &newExporter)
Replace the current Range Map and Export with the given objects.
Definition: Tpetra_CrsMatrix_def.hpp:3978
Tpetra::RowMatrix::getNumEntriesInLocalRow
virtual size_t getNumEntriesInLocalRow(LocalOrdinal localRow) const =0
The current number of entries on the calling process in the specified local row.
Tpetra::CrsMatrix::scale
void scale(const Scalar &alpha)
Scale the matrix&#39;s values: this := alpha*this.
Definition: Tpetra_CrsMatrix_def.hpp:3376
Tpetra::CrsGraph::getDomainMap
Teuchos::RCP< const map_type > getDomainMap() const override
Returns the Map associated with the domain of this graph.
Definition: Tpetra_CrsGraph_def.hpp:894
Tpetra::CrsMatrix::getLocalNumCols
size_t getLocalNumCols() const override
The number of columns connected to the locally owned rows of this matrix.
Definition: Tpetra_CrsMatrix_def.hpp:865
Tpetra_Details_copyOffsets.hpp
Declare and define Tpetra::Details::copyOffsets, an implementation detail of Tpetra (in particular...
Tpetra::CrsMatrix::fillLocalGraphAndMatrix
void fillLocalGraphAndMatrix(const Teuchos::RCP< Teuchos::ParameterList > &params)
Fill data into the local graph and matrix.
Definition: Tpetra_CrsMatrix_def.hpp:1173
Tpetra::CrsMatrix::resumeFill
void resumeFill(const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null)
Resume operations that may change the values or structure of the matrix.
Definition: Tpetra_CrsMatrix_def.hpp:4269
Tpetra::CrsMatrix::leftScale
void leftScale(const Vector< Scalar, LocalOrdinal, GlobalOrdinal, Node > &x) override
Scale the matrix on the left with the given Vector.
Definition: Tpetra_CrsMatrix_def.hpp:3717
Tpetra::CrsGraph::noRedundancies_
bool noRedundancies_
Whether the graph&#39;s indices are non-redundant (merged) in each row, on this process.
Definition: Tpetra_CrsGraph_decl.hpp:2481
Tpetra::CrsMatrix::global_ordinal_type
GlobalOrdinal global_ordinal_type
The type of each global index in the matrix.
Definition: Tpetra_CrsMatrix_decl.hpp:424
Tpetra::Details::padCrsArrays
void padCrsArrays(const RowPtr &rowPtrBeg, const RowPtr &rowPtrEnd, Indices &indices_wdv, const Padding &padding, const int my_rank, const bool verbose)
Determine if the row pointers and indices arrays need to be resized to accommodate new entries...
Definition: Tpetra_Details_crsUtils.hpp:540
Tpetra::DistObject::isDistributed
bool isDistributed() const
Whether this is a globally distributed object.
Definition: Tpetra_DistObject_def.hpp:630
Tpetra::CrsMatrix::reindexColumns
void reindexColumns(crs_graph_type *const graph, const Teuchos::RCP< const map_type > &newColMap, const Teuchos::RCP< const import_type > &newImport=Teuchos::null, const bool sortEachRow=true)
Reindex the column indices in place, and replace the column Map. Optionally, replace the Import objec...
Definition: Tpetra_CrsMatrix_def.hpp:3896
Tpetra::CrsMatrix::getCrsGraph
Teuchos::RCP< const crs_graph_type > getCrsGraph() const
This matrix&#39;s graph, as a CrsGraph.
Definition: Tpetra_CrsMatrix_def.hpp:946
Tpetra::CrsGraph::getGlobalIndsViewHost
global_inds_dualv_type::t_host::const_type getGlobalIndsViewHost(const RowInfo &rowinfo) const
Get a const, globally indexed view of the locally owned row myRow, such that rowinfo = getRowInfo(myR...
Definition: Tpetra_CrsGraph_def.hpp:1395
Tpetra::Details::Behavior::debug
static bool debug()
Whether Tpetra is in debug mode.
Definition: Tpetra_Details_Behavior.cpp:442
Tpetra::CrsMatrix::getGlobalMaxNumRowEntries
size_t getGlobalMaxNumRowEntries() const override
Maximum number of entries in any row of the matrix, over all processes in the matrix&#39;s communicator...
Definition: Tpetra_CrsMatrix_def.hpp:887
Tpetra::Operator::getRangeMap
virtual Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getRangeMap() const =0
The Map associated with the range of this operator, which must be compatible with Y...
Tpetra::CrsMatrix::applyNonTranspose
void applyNonTranspose(const MV &X_in, MV &Y_in, Scalar alpha, Scalar beta) const
Special case of apply() for mode == Teuchos::NO_TRANS.
Definition: Tpetra_CrsMatrix_def.hpp:4675
Tpetra::CrsMatrix::convert
Teuchos::RCP< CrsMatrix< T, LocalOrdinal, GlobalOrdinal, Node > > convert() const
Return another CrsMatrix with the same entries, but converted to a different Scalar type T...
Definition: Tpetra_CrsMatrix_def.hpp:5152
Tpetra::CrsMatrix::scalar_type
Scalar scalar_type
The type of each entry in the matrix.
Definition: Tpetra_CrsMatrix_decl.hpp:420
Tpetra::RowInfo
Allocation information for a locally owned row in a CrsGraph or CrsMatrix.
Definition: Tpetra_CrsGraph_decl.hpp:78
Tpetra::CrsMatrix::swap
void swap(CrsMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node > &matrix)
Swaps the data from *this with the data and maps from crsMatrix.
Definition: Tpetra_CrsMatrix_def.hpp:764
Tpetra::CrsMatrix::fillLocalMatrix
void fillLocalMatrix(const Teuchos::RCP< Teuchos::ParameterList > &params)
Fill data into the local matrix.
Definition: Tpetra_CrsMatrix_def.hpp:1552
Tpetra::CrsGraph::lclIndsUnpacked_wdv
local_inds_wdv_type lclIndsUnpacked_wdv
Local ordinals of column indices for all rows Valid when isLocallyIndexed is true If OptimizedStorage...
Definition: Tpetra_CrsGraph_decl.hpp:2294
Tpetra::Details::verbosePrintArray
void verbosePrintArray(std::ostream &out, const ArrayType &x, const char name[], const size_t maxNumToPrint)
Print min(x.size(), maxNumToPrint) entries of x.
Definition: Tpetra_Util.hpp:1062
Tpetra::CrsGraph::isGloballyIndexed
bool isGloballyIndexed() const override
Whether the graph&#39;s column indices are stored as global indices.
Definition: Tpetra_CrsGraph_def.hpp:1075
Tpetra::Details::leftScaleLocalCrsMatrix
void leftScaleLocalCrsMatrix(const LocalSparseMatrixType &A_lcl, const ScalingFactorsViewType &scalingFactors, const bool assumeSymmetric, const bool divide=true)
Left-scale a KokkosSparse::CrsMatrix.
Definition: Tpetra_Details_leftScaleLocalCrsMatrix.hpp:110
Tpetra::Array_size_type
Teuchos_Ordinal Array_size_type
Size type for Teuchos Array objects.
Definition: Tpetra_ConfigDefs.hpp:21
Tpetra::CrsMatrix::globalAssemble
void globalAssemble()
Communicate nonlocal contributions to other processes.
Definition: Tpetra_CrsMatrix_def.hpp:4020
Tpetra::CrsMatrix::getLocalDiagCopy
void getLocalDiagCopy(Vector< Scalar, LocalOrdinal, GlobalOrdinal, Node > &diag) const override
Get a constant, nonpersisting view of a row of this matrix, using local row and column indices...
Definition: Tpetra_CrsMatrix_def.hpp:3568
Tpetra::CrsGraph::findGlobalIndices
size_t findGlobalIndices(const RowInfo &rowInfo, const Teuchos::ArrayView< const global_ordinal_type > &indices, std::function< void(const size_t, const size_t, const size_t)> fun) const
Finds indices in the given row.
Definition: Tpetra_CrsGraph_def.hpp:1831
Tpetra::CrsMatrix::local_matrix_device_type
KokkosSparse::CrsMatrix< impl_scalar_type, local_ordinal_type, device_type, void, typename local_graph_device_type::size_type > local_matrix_device_type
The specialization of Kokkos::CrsMatrix that represents the part of the sparse matrix on each MPI pro...
Definition: Tpetra_CrsMatrix_decl.hpp:481
Tpetra::CrsMatrix::sumIntoLocalValuesImpl
virtual LocalOrdinal sumIntoLocalValuesImpl(impl_scalar_type rowVals[], const crs_graph_type &graph, const RowInfo &rowInfo, const LocalOrdinal inds[], const impl_scalar_type newVals[], const LocalOrdinal numElts, const bool atomic=useAtomicUpdatesByDefault)
Implementation detail of sumIntoLocalValues.
Definition: Tpetra_CrsMatrix_def.hpp:2950
Tpetra::sort
void sort(View &view, const size_t &size)
Convenience wrapper for std::sort for host-accessible views.
Definition: Tpetra_Util.hpp:614
Tpetra::Details::gathervPrint
void gathervPrint(std::ostream &out, const std::string &s, const Teuchos::Comm< int > &comm)
On Process 0 in the given communicator, print strings from each process in that communicator, in rank order.
Definition: Tpetra_Details_gathervPrint.cpp:18
Tpetra::CrsMatrix::isFillActive
bool isFillActive() const
Whether the matrix is not fill complete.
Definition: Tpetra_CrsMatrix_def.hpp:794
Tpetra::CrsMatrix::importMV_
Teuchos::RCP< MV > importMV_
Column Map MultiVector used in apply().
Definition: Tpetra_CrsMatrix_decl.hpp:3825
Tpetra_Details_copyConvert.hpp
Declare and define Tpetra::Details::copyConvert, an implementation detail of Tpetra (in particular...
Tpetra::CrsMatrix::isStaticGraph
bool isStaticGraph() const
Indicates that the graph is static, so that new entries cannot be added to this matrix.
Definition: Tpetra_CrsMatrix_def.hpp:1042
Tpetra::global_size_t
size_t global_size_t
Global size_t object.
Definition: Tpetra_ConfigDefs.hpp:71
Tpetra::CrsMatrix::hasTransposeApply
bool hasTransposeApply() const override
Whether apply() allows applying the transpose or conjugate transpose.
Definition: Tpetra_CrsMatrix_def.hpp:1049
Tpetra::CrsGraph::reindexColumns
void reindexColumns(const Teuchos::RCP< const map_type > &newColMap, const Teuchos::RCP< const import_type > &newImport=Teuchos::null, const bool sortIndicesInEachRow=true)
Reindex the column indices in place, and replace the column Map. Optionally, replace the Import objec...
Definition: Tpetra_CrsGraph_def.hpp:3786
Tpetra::deep_copy
void deep_copy(MultiVector< DS, DL, DG, DN > &dst, const MultiVector< SS, SL, SG, SN > &src)
Copy the contents of the MultiVector src into dst.
Definition: Tpetra_MultiVector_decl.hpp:2456
Tpetra::MultiVector< Scalar, LocalOrdinal, GlobalOrdinal, Node >::getLocalViewHost
dual_view_type::t_host::const_type getLocalViewHost(Access::ReadOnlyStruct) const
Return a read-only, up-to-date view of this MultiVector&#39;s local data on host. This requires that ther...
Tpetra::CrsMatrix::getLocalMaxNumRowEntries
size_t getLocalMaxNumRowEntries() const override
Maximum number of entries in any row of the matrix, on this process.
Definition: Tpetra_CrsMatrix_def.hpp:894
Tpetra::CrsMatrix::exportAndFillComplete
void exportAndFillComplete(Teuchos::RCP< CrsMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node > > &destMatrix, const export_type &exporter, const Teuchos::RCP< const map_type > &domainMap=Teuchos::null, const Teuchos::RCP< const map_type > &rangeMap=Teuchos::null, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null) const
Export from this to the given destination matrix, and make the result fill complete.
Definition: Tpetra_CrsMatrix_def.hpp:9168
Tpetra::Details::PackTraits< LO >::packValue
static KOKKOS_INLINE_FUNCTION size_t packValue(char outBuf[], const LO &inVal)
Pack the given value of type value_type into the given output buffer of bytes (char).
Definition: Tpetra_Details_PackTraits.hpp:220
Tpetra::INSERT
Insert new values that don&#39;t currently exist.
Definition: Tpetra_CombineMode.hpp:67
Tpetra::CrsMatrix::getValuesViewDevice
values_dualv_type::t_dev::const_type getValuesViewDevice(const RowInfo &rowinfo) const
Get a const Device view of the locally owned values row myRow, such that rowinfo = getRowInfo(myRow)...
Definition: Tpetra_CrsMatrix_def.hpp:3131
Tpetra::CrsMatrix::isFillComplete
bool isFillComplete() const override
Whether the matrix is fill complete.
Definition: Tpetra_CrsMatrix_def.hpp:787
Tpetra::CrsGraph::isSorted
bool isSorted() const
Whether graph indices in all rows are known to be sorted.
Definition: Tpetra_CrsGraph_def.hpp:1152
Tpetra::CrsMatrix::getComm
Teuchos::RCP< const Teuchos::Comm< int > > getComm() const override
The communicator over which the matrix is distributed.
Definition: Tpetra_CrsMatrix_def.hpp:780
Tpetra::createOneToOne
Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > createOneToOne(const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &M)
Nonmember constructor for a contiguous Map with user-defined weights and a user-specified, possibly nondefault Kokkos Node type.
Tpetra::CrsMatrix::importAndFillComplete
void importAndFillComplete(Teuchos::RCP< CrsMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node > > &destMatrix, const import_type &importer, const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null) const
Import from this to the given destination matrix, and make the result fill complete.
Definition: Tpetra_CrsMatrix_def.hpp:9143
Tpetra::MultiVector::scale
void scale(const Scalar &alpha)
Scale in place: this = alpha*this.
Definition: Tpetra_MultiVector_def.hpp:2821
Tpetra::RowMatrix::getGlobalRowCopy
virtual void getGlobalRowCopy(GlobalOrdinal GlobalRow, nonconst_global_inds_host_view_type &Indices, nonconst_values_host_view_type &Values, size_t &NumEntries) const =0
Get a copy of the given global row&#39;s entries.
Tpetra::CrsGraph::rowMap_
Teuchos::RCP< const map_type > rowMap_
The Map describing the distribution of rows of the graph.
Definition: Tpetra_CrsGraph_decl.hpp:2112
Tpetra::CrsMatrix::getLocalMatrixDevice
TPETRA_DETAILS_ALWAYS_INLINE local_matrix_device_type getLocalMatrixDevice() const
The local sparse matrix.
Definition: Tpetra_CrsMatrix_def.hpp:982
Tpetra::CrsGraph::k_numRowEntries_
num_row_entries_type k_numRowEntries_
The number of local entries in each locally owned row.
Definition: Tpetra_CrsGraph_decl.hpp:2449
Tpetra::Map::getGlobalElement
global_ordinal_type getGlobalElement(local_ordinal_type localIndex) const
The global index corresponding to the given local index.
Definition: Tpetra_Map_def.hpp:1259
Tpetra::Map::isNodeLocalElement
bool isNodeLocalElement(local_ordinal_type localIndex) const
Whether the given local index is valid for this Map on the calling process.
Definition: Tpetra_Map_def.hpp:1280
Tpetra_Details_crsUtils.hpp
Functions for manipulating CRS arrays.
Tpetra::CrsGraph::num_row_entries_type
Kokkos::View< size_t *, Kokkos::LayoutLeft, device_type >::HostMirror num_row_entries_type
Row offsets for &quot;1-D&quot; storage.
Definition: Tpetra_CrsGraph_decl.hpp:2431
Tpetra::CrsMatrix::getIndexBase
GlobalOrdinal getIndexBase() const override
The index base for global indices for this matrix.
Definition: Tpetra_CrsMatrix_def.hpp:901
Tpetra_Details_computeOffsets.hpp
Declare and define the functions Tpetra::Details::computeOffsetsFromCounts and Tpetra::computeOffsets...
Tpetra::Export
Communication plan for data redistribution from a (possibly) multiply-owned to a uniquely-owned distr...
Definition: Tpetra_Export_decl.hpp:85
Tpetra::Details::unpackAndCombineIntoCrsArrays
void unpackAndCombineIntoCrsArrays(const CrsGraph< LO, GO, NT > &sourceGraph, const Teuchos::ArrayView< const LO > &importLIDs, const Teuchos::ArrayView< const typename CrsGraph< LO, GO, NT >::packet_type > &imports, const Teuchos::ArrayView< const size_t > &numPacketsPerLID, const size_t constantNumPackets, const CombineMode combineMode, const size_t numSameIDs, const Teuchos::ArrayView< const LO > &permuteToLIDs, const Teuchos::ArrayView< const LO > &permuteFromLIDs, size_t TargetNumRows, size_t TargetNumNonzeros, const int MyTargetPID, const Teuchos::ArrayView< size_t > &CRS_rowptr, const Teuchos::ArrayView< GO > &CRS_colind, const Teuchos::ArrayView< const int > &SourcePids, Teuchos::Array< int > &TargetPids)
unpackAndCombineIntoCrsArrays
Tpetra::sort2
void sort2(const IT1 &first1, const IT1 &last1, const IT2 &first2, const bool stableSort=false)
Sort the first array, and apply the resulting permutation to the second array.
Definition: Tpetra_Util.hpp:557
Tpetra::Operator::getDomainMap
virtual Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getDomainMap() const =0
The Map associated with the domain of this operator, which must be compatible with X...
Tpetra::CrsMatrix::getColumnMapMultiVector
Teuchos::RCP< MV > getColumnMapMultiVector(const MV &X_domainMap, const bool force=false) const
Create a (or fetch a cached) column Map MultiVector.
Definition: Tpetra_CrsMatrix_def.hpp:7297
TPETRA_ABUSE_WARNING
#define TPETRA_ABUSE_WARNING(throw_exception_test, Exception, msg)
Handle an abuse warning, according to HAVE_TPETRA_THROW_ABUSE_WARNINGS and HAVE_TPETRA_PRINT_ABUSE_WA...
Definition: Tpetra_Util.hpp:119
Tpetra::Map::isNodeGlobalElement
bool isNodeGlobalElement(global_ordinal_type globalIndex) const
Whether the given global index is owned by this Map on the calling process.
Definition: Tpetra_Map_def.hpp:1292
Tpetra::Details::packCrsMatrixNew
void packCrsMatrixNew(const CrsMatrix< ST, LO, GO, NT > &sourceMatrix, Kokkos::DualView< char *, typename DistObject< char, LO, GO, NT >::buffer_device_type > &exports, const Kokkos::DualView< size_t *, typename DistObject< char, LO, GO, NT >::buffer_device_type > &numPacketsPerLID, const Kokkos::DualView< const LO *, typename DistObject< char, LO, GO, NT >::buffer_device_type > &exportLIDs, size_t &constantNumPackets)
Pack specified entries of the given local sparse matrix for communication, for &quot;new&quot; DistObject inter...
Definition: Tpetra_Details_packCrsMatrix_def.hpp:897
Tpetra::CrsMatrix::describe
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const override
Print this object with the given verbosity level to the given output stream.
Definition: Tpetra_CrsMatrix_def.hpp:5246
Tpetra::CrsMatrix::getGraph
Teuchos::RCP< const RowGraph< LocalOrdinal, GlobalOrdinal, Node > > getGraph() const override
This matrix&#39;s graph, as a RowGraph.
Definition: Tpetra_CrsMatrix_def.hpp:936
Tpetra::Details::Behavior::verbose
static bool verbose()
Whether Tpetra is in verbose mode.
Definition: Tpetra_Details_Behavior.cpp:451
Tpetra::CombineMode
CombineMode
Rule for combining data in an Import or Export.
Definition: Tpetra_CombineMode.hpp:65
Tpetra::ADD
Sum new values.
Definition: Tpetra_CombineMode.hpp:66
Tpetra::CrsMatrix::unpackAndCombine
void unpackAndCombine(const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &importLIDs, Kokkos::DualView< char *, buffer_device_type > imports, Kokkos::DualView< size_t *, buffer_device_type > numPacketsPerLID, const size_t constantNumPackets, const CombineMode CM) override
Unpack the imported column indices and values, and combine into matrix.
Definition: Tpetra_CrsMatrix_def.hpp:6893
Tpetra::CrsGraph::isFillComplete
bool isFillComplete() const override
Whether fillComplete() has been called and the graph is in compute mode.
Definition: Tpetra_CrsGraph_def.hpp:1050
Tpetra::CrsMatrix::haveGlobalConstants
bool haveGlobalConstants() const
Returns true if globalConstants have been computed; false otherwise.
Definition: Tpetra_CrsMatrix_def.hpp:4284
Tpetra::CrsMatrix::isGloballyIndexed
bool isGloballyIndexed() const override
Whether the matrix is globally indexed on the calling process.
Definition: Tpetra_CrsMatrix_def.hpp:815
Tpetra::CrsGraph::getRowInfoFromGlobalRowIndex
RowInfo getRowInfoFromGlobalRowIndex(const global_ordinal_type gblRow) const
Get information about the locally owned row with global index gblRow.
Definition: Tpetra_CrsGraph_def.hpp:1484
Tpetra::CrsMatrix::sumIntoGlobalValues
LocalOrdinal sumIntoGlobalValues(const GlobalOrdinal globalRow, const Teuchos::ArrayView< const GlobalOrdinal > &cols, const Teuchos::ArrayView< const Scalar > &vals, const bool atomic=useAtomicUpdatesByDefault)
Sum into one or more sparse matrix entries, using global indices.
Definition: Tpetra_CrsMatrix_def.hpp:2596
Tpetra_Import_Util2.hpp
Utility functions for packing and unpacking sparse matrix entries.
Tpetra::Details::copyConvert
void copyConvert(const OutputViewType &dst, const InputViewType &src)
Copy values from the 1-D Kokkos::View src, to the 1-D Kokkos::View dst, of the same length...
Definition: Tpetra_Details_copyConvert.hpp:328
Tpetra::CrsMatrix::add
virtual Teuchos::RCP< RowMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node > > add(const Scalar &alpha, const RowMatrix< Scalar, LocalOrdinal, GlobalOrdinal, Node > &A, const Scalar &beta, const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &domainMap, const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &rangeMap, const Teuchos::RCP< Teuchos::ParameterList > &params) const override
Implementation of RowMatrix::add: return alpha*A + beta*this.
Definition: Tpetra_CrsMatrix_def.hpp:7423
Tpetra::CrsMatrix::fillComplete_
bool fillComplete_
Whether the matrix is fill complete.
Definition: Tpetra_CrsMatrix_decl.hpp:4032
Tpetra::CrsMatrix::replaceGlobalValues
local_ordinal_type replaceGlobalValues(const global_ordinal_type globalRow, const Kokkos::View< const global_ordinal_type *, Kokkos::AnonymousSpace > &inputInds, const Kokkos::View< const impl_scalar_type *, Kokkos::AnonymousSpace > &inputVals)
Replace one or more entries&#39; values, using global indices.
Definition: Tpetra_CrsMatrix_def.hpp:2487
Tpetra::ABSMAX
Replace old value with maximum of magnitudes of old and new values.
Definition: Tpetra_CombineMode.hpp:69
Tpetra::CrsMatrix::sumIntoGlobalValuesImpl
virtual LocalOrdinal sumIntoGlobalValuesImpl(impl_scalar_type rowVals[], const crs_graph_type &graph, const RowInfo &rowInfo, const GlobalOrdinal inds[], const impl_scalar_type newVals[], const LocalOrdinal numElts, const bool atomic=useAtomicUpdatesByDefault)
Implementation detail of sumIntoGlobalValues.
Definition: Tpetra_CrsMatrix_def.hpp:2512
Tpetra::SrcDistObject
Abstract base class for objects that can be the source of an Import or Export operation.
Definition: Tpetra_SrcDistObject.hpp:57
Tpetra::CrsMatrix::sumIntoLocalValues
local_ordinal_type sumIntoLocalValues(const local_ordinal_type localRow, const Kokkos::View< const local_ordinal_type *, Kokkos::AnonymousSpace > &inputInds, const Kokkos::View< const impl_scalar_type *, Kokkos::AnonymousSpace > &inputVals, const bool atomic=useAtomicUpdatesByDefault)
Sum into one or more sparse matrix entries, using local row and column indices.
Definition: Tpetra_CrsMatrix_def.hpp:3052
Tpetra::CrsMatrix::device_type
typename Node::device_type device_type
The Kokkos device type.
Definition: Tpetra_CrsMatrix_decl.hpp:426
Tpetra::CrsMatrix::getNumEntriesInLocalRow
size_t getNumEntriesInLocalRow(local_ordinal_type localRow) const override
Number of entries in the sparse matrix in the given local row, on the calling (MPI) process...
Definition: Tpetra_CrsMatrix_def.hpp:880
Tpetra::Details::getDiagCopyWithoutOffsets
static LocalMapType::local_ordinal_type getDiagCopyWithoutOffsets(const DiagType &D, const LocalMapType &rowMap, const LocalMapType &colMap, const CrsMatrixType &A)
Given a locally indexed, local sparse matrix, and corresponding local row and column Maps...
Definition: Tpetra_Details_getDiagCopyWithoutOffsets_decl.hpp:148
Tpetra::CrsMatrix::getRowMapMultiVector
Teuchos::RCP< MV > getRowMapMultiVector(const MV &Y_rangeMap, const bool force=false) const
Create a (or fetch a cached) row Map MultiVector.
Definition: Tpetra_CrsMatrix_def.hpp:7354
Tpetra::CrsMatrix::description
std::string description() const override
A one-line description of this object.
Definition: Tpetra_CrsMatrix_def.hpp:5220
Tpetra::CrsMatrix::getLocalDiagOffsets
void getLocalDiagOffsets(Teuchos::ArrayRCP< size_t > &offsets) const
Get offsets of the diagonal entries in the matrix.
Definition: Tpetra_CrsMatrix_def.hpp:3526
Tpetra::CrsMatrix::apply
void apply(const MultiVector< Scalar, LocalOrdinal, GlobalOrdinal, Node > &X, MultiVector< Scalar, LocalOrdinal, GlobalOrdinal, Node > &Y, Teuchos::ETransp mode=Teuchos::NO_TRANS, Scalar alpha=Teuchos::ScalarTraits< Scalar >::one(), Scalar beta=Teuchos::ScalarTraits< Scalar >::zero()) const override
Compute a sparse matrix-MultiVector multiply.
Definition: Tpetra_CrsMatrix_def.hpp:5123
Tpetra::CrsMatrix::getLocalNumEntries
size_t getLocalNumEntries() const override
The local number of entries in this matrix.
Definition: Tpetra_CrsMatrix_def.hpp:836
Tpetra::REPLACE
Replace existing values with new values.
Definition: Tpetra_CombineMode.hpp:68
Tpetra::CrsMatrix::fillComplete
void fillComplete(const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null)
Tell the matrix that you are done changing its structure or values, and that you are ready to do comp...
Definition: Tpetra_CrsMatrix_def.hpp:4317
Tpetra::CrsGraph::getRangeMap
Teuchos::RCP< const map_type > getRangeMap() const override
Returns the Map associated with the domain of this graph.
Definition: Tpetra_CrsGraph_def.hpp:902
Tpetra::MultiVector< Scalar, LocalOrdinal, GlobalOrdinal, Node >::getLocalViewDevice
dual_view_type::t_dev::const_type getLocalViewDevice(Access::ReadOnlyStruct) const
Return a read-only, up-to-date view of this MultiVector&#39;s local data on device. This requires that th...
Tpetra::ZERO
Replace old values with zero.
Definition: Tpetra_CombineMode.hpp:70
Tpetra::CrsGraph::getRowPtrsUnpackedHost
const row_ptrs_host_view_type & getRowPtrsUnpackedHost() const
Get the unpacked row pointers on host. Lazily make a copy from device.
Definition: Tpetra_CrsGraph_decl.hpp:2194
Tpetra::combineModeToString
std::string combineModeToString(const CombineMode combineMode)
Human-readable string representation of the given CombineMode.
Definition: Tpetra_CombineMode.cpp:61
Tpetra::CrsMatrix::insertGlobalValues
void insertGlobalValues(const GlobalOrdinal globalRow, const Teuchos::ArrayView< const GlobalOrdinal > &cols, const Teuchos::ArrayView< const Scalar > &vals)
Insert one or more entries into the matrix, using global column indices.
Definition: Tpetra_CrsMatrix_def.hpp:1977
Tpetra::Details::Behavior::rowImbalanceThreshold
static size_t rowImbalanceThreshold()
Threshold for deciding if a local matrix is &quot;imbalanced&quot; in the number of entries per row...
Definition: Tpetra_Details_Behavior.cpp:514
Tpetra::Details::Transfer::isLocallyComplete
bool isLocallyComplete() const
Is this Export or Import locally complete?
Definition: Tpetra_Details_Transfer_def.hpp:266
Tpetra::CrsMatrix::replaceLocalValuesImpl
virtual LocalOrdinal replaceLocalValuesImpl(impl_scalar_type rowVals[], const crs_graph_type &graph, const RowInfo &rowInfo, const LocalOrdinal inds[], const impl_scalar_type newVals[], const LocalOrdinal numElts)
Implementation detail of replaceLocalValues.
Definition: Tpetra_CrsMatrix_def.hpp:2281
Tpetra_Details_leftScaleLocalCrsMatrix.hpp
Declaration and definition of Tpetra::Details::leftScaleLocalCrsMatrix.
Tpetra::CrsGraph::getRowInfo
RowInfo getRowInfo(const local_ordinal_type myRow) const
Get information about the locally owned row with local index myRow.
Definition: Tpetra_CrsGraph_def.hpp:1437
Tpetra::CrsMatrix::getValuesViewDeviceNonConst
values_dualv_type::t_dev getValuesViewDeviceNonConst(const RowInfo &rowinfo)
Get a non-const Device view of the locally owned values row myRow, such that rowinfo = getRowInfo(myR...
Definition: Tpetra_CrsMatrix_def.hpp:3145
Tpetra::Details::dualViewStatusToString
std::string dualViewStatusToString(const DualViewType &dv, const char name[])
Return the status of the given Kokkos::DualView, as a human-readable string.
Definition: Tpetra_Util.hpp:1045
Tpetra::CrsMatrix::removeEmptyProcessesInPlace
virtual void removeEmptyProcessesInPlace(const Teuchos::RCP< const map_type > &newMap) override
Remove processes owning zero rows from the Maps and their communicator.
Definition: Tpetra_CrsMatrix_def.hpp:7400
Tpetra::CrsMatrix::checkSizes
virtual bool checkSizes(const SrcDistObject &source) override
Compare the source and target (this) objects for compatibility.
Definition: Tpetra_CrsMatrix_def.hpp:5482
Tpetra::Map::getLocalMap
local_map_type getLocalMap() const
Get the LocalMap for Kokkos-Kernels.
Definition: Tpetra_Map_def.hpp:1311
Tpetra::CrsGraph
A distributed graph accessed by rows (adjacency lists) and stored sparsely.
Definition: Tpetra_CrsGraph_decl.hpp:190
Tpetra::CrsMatrix::impl_scalar_type
typename row_matrix_type::impl_scalar_type impl_scalar_type
The type used internally in place of Scalar.
Definition: Tpetra_CrsMatrix_decl.hpp:459
Tpetra::Details::unpackAndCombineWithOwningPIDsCount
size_t unpackAndCombineWithOwningPIDsCount(const CrsGraph< LO, GO, NT > &sourceGraph, const Teuchos::ArrayView< const LO > &importLIDs, const Teuchos::ArrayView< const typename CrsGraph< LO, GO, NT >::packet_type > &imports, const Teuchos::ArrayView< const size_t > &numPacketsPerLID, size_t constantNumPackets, CombineMode combineMode, size_t numSameIDs, const Teuchos::ArrayView< const LO > &permuteToLIDs, const Teuchos::ArrayView< const LO > &permuteFromLIDs)
Special version of Tpetra::Details::unpackCrsGraphAndCombine that also unpacks owning process ranks...
Tpetra::Map
A parallel distribution of indices over processes.
Definition: Tpetra_Map_decl.hpp:198
Tpetra::CrsMatrix::getLocalRowCopy
void getLocalRowCopy(LocalOrdinal LocalRow, nonconst_local_inds_host_view_type &Indices, nonconst_values_host_view_type &Values, size_t &NumEntries) const override
Fill given arrays with a deep copy of the locally owned entries of the matrix in a given row...
Definition: Tpetra_CrsMatrix_def.hpp:3159
Tpetra::DistObject::doExport
void doExport(const SrcDistObject &source, const Export< LocalOrdinal, GlobalOrdinal, Node > &exporter, const CombineMode CM, const bool restrictedMode=false)
Export data into this object using an Export object (&quot;forward mode&quot;).
Definition: Tpetra_DistObject_def.hpp:279
Tpetra::CrsMatrix::getDomainMap
Teuchos::RCP< const map_type > getDomainMap() const override
The domain Map of this matrix.
Definition: Tpetra_CrsMatrix_def.hpp:922
Tpetra::Details::getArrayViewFromDualView
Teuchos::ArrayView< typename DualViewType::t_dev::value_type > getArrayViewFromDualView(const DualViewType &x)
Get a Teuchos::ArrayView which views the host Kokkos::View of the input 1-D Kokkos::DualView.
Definition: Tpetra_Util.hpp:974
Tpetra::Details::PackTraits< LO >::packValueCount
static KOKKOS_INLINE_FUNCTION size_t packValueCount(const LO &)
Number of bytes required to pack or unpack the given value of type value_type.
Definition: Tpetra_Details_PackTraits.hpp:204
Tpetra::CrsMatrix::insertLocalValues
void insertLocalValues(const LocalOrdinal localRow, const Teuchos::ArrayView< const LocalOrdinal > &cols, const Teuchos::ArrayView< const Scalar > &vals, const CombineMode CM=ADD)
Insert one or more entries into the matrix, using local column indices.
Definition: Tpetra_CrsMatrix_def.hpp:1745
Tpetra_Import_Util.hpp
Internal functions and macros designed for use with Tpetra::Import and Tpetra::Export objects...
Tpetra::CrsMatrix::storageStatus_
Details::EStorageStatus storageStatus_
Status of the matrix&#39;s storage, when not in a fill-complete state.
Definition: Tpetra_CrsMatrix_decl.hpp:4028
Tpetra::RowMatrix
A read-only, row-oriented interface to a sparse matrix.
Definition: Tpetra_RowMatrix_decl.hpp:53
Tpetra::CrsMatrix::rightScale
void rightScale(const Vector< Scalar, LocalOrdinal, GlobalOrdinal, Node > &x) override
Scale the matrix on the right with the given Vector.
Definition: Tpetra_CrsMatrix_def.hpp:3772
Tpetra::CrsMatrix::replaceDomainMap
void replaceDomainMap(const Teuchos::RCP< const map_type > &newDomainMap)
Replace the current domain Map with the given objects.
Definition: Tpetra_CrsMatrix_def.hpp:3932
Tpetra::Details::AbsMax::operator()
Scalar operator()(const Scalar &x, const Scalar &y)
Return the maximum of the magnitudes (absolute values) of x and y.
Definition: Tpetra_CrsMatrix_def.hpp:101
Tpetra::Map::getLocalElement
local_ordinal_type getLocalElement(global_ordinal_type globalIndex) const
The local index corresponding to the given global index.
Definition: Tpetra_Map_def.hpp:1234
Tpetra::CrsMatrix::getLocalRowView
void getLocalRowView(LocalOrdinal LocalRow, local_inds_host_view_type &indices, values_host_view_type &values) const override
Get a constant view of a row of this matrix, using local row and column indices.
Definition: Tpetra_CrsMatrix_def.hpp:3260
Tpetra::CrsMatrix::getValuesViewHost
values_dualv_type::t_host::const_type getValuesViewHost(const RowInfo &rowinfo) const
Get a const Host view of the locally owned values row myRow, such that rowinfo = getRowInfo(myRow).
Definition: Tpetra_CrsMatrix_def.hpp:3103
Tpetra::CrsGraph::isLocallyIndexed
bool isLocallyIndexed() const override
Whether the graph&#39;s column indices are stored as local indices.
Definition: Tpetra_CrsGraph_def.hpp:1067
Tpetra::Vector
A distributed dense vector.
Definition: Tpetra_Vector_decl.hpp:44
Tpetra_Details_iallreduce.hpp
Declaration of Tpetra::Details::iallreduce.
Tpetra::MultiVector::reduce
void reduce()
Sum values of a locally replicated multivector across all processes.
Definition: Tpetra_MultiVector_def.hpp:4322
Tpetra_Details_castAwayConstDualView.hpp
Declaration and definition of Tpetra::Details::castAwayConstDualView, an implementation detail of Tpe...
Tpetra::Details::iallreduce
std::shared_ptr< CommRequest > iallreduce(const InputViewType &sendbuf, const OutputViewType &recvbuf, const ::Teuchos::EReductionType op, const ::Teuchos::Comm< int > &comm)
Nonblocking all-reduce, for either rank-1 or rank-0 Kokkos::View objects.
Definition: Tpetra_Details_iallreduce.hpp:271
Tpetra::CrsMatrix::applyTranspose
void applyTranspose(const MV &X_in, MV &Y_in, const Teuchos::ETransp mode, Scalar alpha, Scalar beta) const
Special case of apply() for mode != Teuchos::NO_TRANS.
Definition: Tpetra_CrsMatrix_def.hpp:4842
Tpetra::Details::computeOffsetsFromCounts
OffsetsViewType::non_const_value_type computeOffsetsFromCounts(const ExecutionSpace &execSpace, const OffsetsViewType &ptr, const CountsViewType &counts)
Compute offsets from counts.
Definition: Tpetra_Details_computeOffsets.hpp:216
Tpetra::CrsMatrix::allocateValues
void allocateValues(ELocalGlobal lg, GraphAllocationStatus gas, const bool verbose)
Allocate values (and optionally indices) using the Node.
Definition: Tpetra_CrsMatrix_def.hpp:1063
Tpetra::Details::castAwayConstDualView
Kokkos::DualView< ValueType *, DeviceType > castAwayConstDualView(const Kokkos::DualView< const ValueType *, DeviceType > &input_dv)
Cast away const-ness of a 1-D Kokkos::DualView.
Definition: Tpetra_Details_castAwayConstDualView.hpp:32
Tpetra::CrsMatrix::expertStaticFillComplete
void expertStaticFillComplete(const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap, const Teuchos::RCP< const import_type > &importer=Teuchos::null, const Teuchos::RCP< const export_type > &exporter=Teuchos::null, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null)
Perform a fillComplete on a matrix that already has data.
Definition: Tpetra_CrsMatrix_def.hpp:4516
Tpetra::DistObject< Scalar, LocalOrdinal, GlobalOrdinal, Node >::getMap
virtual Teuchos::RCP< const map_type > getMap() const
The Map describing the parallel distribution of this object.
Definition: Tpetra_DistObject_decl.hpp:559
Tpetra::CrsMatrix::getNumEntriesInGlobalRow
size_t getNumEntriesInGlobalRow(GlobalOrdinal globalRow) const override
Number of entries in the sparse matrix in the given global row, on the calling (MPI) process...
Definition: Tpetra_CrsMatrix_def.hpp:873
Tpetra::Details::Behavior::verbosePrintCountThreshold
static size_t verbosePrintCountThreshold()
Number of entries below which arrays, lists, etc. will be printed in debug mode.
Definition: Tpetra_Details_Behavior.cpp:504
Tpetra::CrsMatrix::getLocalValuesDevice
local_matrix_device_type::values_type::const_type getLocalValuesDevice(Access::ReadOnlyStruct s) const
Get the Kokkos local values on device, read only.
Definition: Tpetra_CrsMatrix_decl.hpp:3334
Tpetra::CrsMatrix::setAllValues
void setAllValues(const typename local_graph_device_type::row_map_type &ptr, const typename local_graph_device_type::entries_type::non_const_type &ind, const typename local_matrix_device_type::values_type &val)
Set the local matrix using three (compressed sparse row) arrays.
Definition: Tpetra_CrsMatrix_def.hpp:3421
Tpetra::CrsMatrix::isLocallyIndexed
bool isLocallyIndexed() const override
Whether the matrix is locally indexed on the calling process.
Definition: Tpetra_CrsMatrix_def.hpp:808
Tpetra::CrsMatrix::getColMap
Teuchos::RCP< const map_type > getColMap() const override
The Map that describes the column distribution in this matrix.
Definition: Tpetra_CrsMatrix_def.hpp:915
Tpetra::CrsMatrix::replaceLocalValues
local_ordinal_type replaceLocalValues(const local_ordinal_type localRow, const Kokkos::View< const local_ordinal_type *, Kokkos::AnonymousSpace > &inputInds, const Kokkos::View< const impl_scalar_type *, Kokkos::AnonymousSpace > &inputVals)
Replace one or more entries&#39; values, using local row and column indices.
Definition: Tpetra_CrsMatrix_def.hpp:2371
Tpetra_Details_rightScaleLocalCrsMatrix.hpp
Declaration and definition of Tpetra::Details::rightScaleLocalCrsMatrix.
Tpetra_Details_getEntryOnHost.hpp
Declaration and definition of Tpetra::Details::getEntryOnHost.
Tpetra::Details::packCrsMatrixWithOwningPIDs
void packCrsMatrixWithOwningPIDs(const CrsMatrix< ST, LO, GO, NT > &sourceMatrix, Kokkos::DualView< char *, typename DistObject< char, LO, GO, NT >::buffer_device_type > &exports_dv, const Teuchos::ArrayView< size_t > &numPacketsPerLID, const Teuchos::ArrayView< const LO > &exportLIDs, const Teuchos::ArrayView< const int > &sourcePIDs, size_t &constantNumPackets)
Pack specified entries of the given local sparse matrix for communication.
Definition: Tpetra_Details_packCrsMatrix_def.hpp:932
Tpetra::CrsMatrix::replaceRangeMap
void replaceRangeMap(const Teuchos::RCP< const map_type > &newRangeMap)
Replace the current range Map with the given objects.
Definition: Tpetra_CrsMatrix_def.hpp:3963
Tpetra::CrsMatrix::getLocalMultiplyOperator
std::shared_ptr< local_multiply_op_type > getLocalMultiplyOperator() const
The local sparse matrix operator (a wrapper of getLocalMatrixDevice() that supports local matrix-vect...
Definition: Tpetra_CrsMatrix_def.hpp:1007
Tpetra::CrsGraph::colMap_
Teuchos::RCP< const map_type > colMap_
The Map describing the distribution of columns of the graph.
Definition: Tpetra_CrsGraph_decl.hpp:2114
Tpetra::CrsMatrix::local_ordinal_type
LocalOrdinal local_ordinal_type
The type of each local index in the matrix.
Definition: Tpetra_CrsMatrix_decl.hpp:422
Tpetra::CrsMatrix::getFrobeniusNorm
mag_type getFrobeniusNorm() const override
Compute and return the Frobenius norm of the matrix.
Definition: Tpetra_CrsMatrix_def.hpp:3826
Tpetra::Details::createPrefix
std::unique_ptr< std::string > createPrefix(const int myRank, const char prefix[])
Create string prefix for each line of verbose output.
Definition: Tpetra_Util.cpp:71
Tpetra::RowMatrix::getRowMap
virtual Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getRowMap() const =0
The Map that describes the distribution of rows over processes.
Tpetra::ADD_ASSIGN
Accumulate new values into existing values (may not be supported in all classes)
Definition: Tpetra_CombineMode.hpp:71
Tpetra::CrsMatrix::localApply
void localApply(const MultiVector< Scalar, LocalOrdinal, GlobalOrdinal, Node > &X, MultiVector< Scalar, LocalOrdinal, GlobalOrdinal, Node > &Y, const Teuchos::ETransp mode=Teuchos::NO_TRANS, const Scalar &alpha=Teuchos::ScalarTraits< Scalar >::one(), const Scalar &beta=Teuchos::ScalarTraits< Scalar >::zero()) const
Compute the local part of a sparse matrix-(Multi)Vector multiply.
Definition: Tpetra_CrsMatrix_def.hpp:4985
Tpetra::Details::rightScaleLocalCrsMatrix
void rightScaleLocalCrsMatrix(const LocalSparseMatrixType &A_lcl, const ScalingFactorsViewType &scalingFactors, const bool assumeSymmetric, const bool divide=true)
Right-scale a KokkosSparse::CrsMatrix.
Definition: Tpetra_Details_rightScaleLocalCrsMatrix.hpp:113
Tpetra::CrsMatrix::isStorageOptimized
bool isStorageOptimized() const
Returns true if storage has been optimized.
Definition: Tpetra_CrsMatrix_def.hpp:801
Tpetra::Details::Behavior
Description of Tpetra&#39;s behavior.
Definition: Tpetra_Details_Behavior.hpp:91
Tpetra::CrsMatrix::copyAndPermute
virtual void copyAndPermute(const SrcDistObject &source, const size_t numSameIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &permuteToLIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &permuteFromLIDs, const CombineMode CM) override
Definition: Tpetra_CrsMatrix_def.hpp:6040
Tpetra::CrsMatrix::getValuesViewHostNonConst
values_dualv_type::t_host getValuesViewHostNonConst(const RowInfo &rowinfo)
Get a non-const Host view of the locally owned values row myRow, such that rowinfo = getRowInfo(myRow...
Definition: Tpetra_CrsMatrix_def.hpp:3117
Tpetra_Details_createMirrorView.hpp
Functions that wrap Kokkos::create_mirror_view, in order to avoid deep copies when not necessary...
Tpetra_Details_Behavior.hpp
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra&#39;s behavior.
Tpetra::CrsMatrix::getLocalNumRows
size_t getLocalNumRows() const override
The number of matrix rows owned by the calling process.
Definition: Tpetra_CrsMatrix_def.hpp:857
Tpetra::CrsMatrix::exportMV_
Teuchos::RCP< MV > exportMV_
Row Map MultiVector used in apply().
Definition: Tpetra_CrsMatrix_decl.hpp:3839
Tpetra::CrsMatrix::getRowMap
Teuchos::RCP< const map_type > getRowMap() const override
The Map that describes the row distribution in this matrix.
Definition: Tpetra_CrsMatrix_def.hpp:908
Tpetra::CrsMatrix::getGlobalNumEntries
global_size_t getGlobalNumEntries() const override
The global number of entries in this matrix.
Definition: Tpetra_CrsMatrix_def.hpp:829
Generated on Mon Jun 16 2025 09:19:05 for Tpetra parallel linear algebra by   doxygen 1.8.5