Tpetra parallel linear algebra  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
MatrixMarket_Tpetra.hpp
Go to the documentation of this file.
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Tpetra: Templated Linear Algebra Services Package
5 // Copyright (2008) Sandia Corporation
6 //
7 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8 // the U.S. Government retains certain rights in this software.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ************************************************************************
40 // @HEADER
41 
42 #ifndef __MatrixMarket_Tpetra_hpp
43 #define __MatrixMarket_Tpetra_hpp
44 
56 #include "Tpetra_Details_gathervPrint.hpp"
57 #include "Tpetra_CrsMatrix.hpp"
58 #include "Tpetra_Operator.hpp"
59 #include "Tpetra_Vector.hpp"
61 #include "Teuchos_MatrixMarket_Raw_Adder.hpp"
62 #include "Teuchos_MatrixMarket_Raw_Graph_Adder.hpp"
63 #include "Teuchos_MatrixMarket_SymmetrizingAdder.hpp"
64 #include "Teuchos_MatrixMarket_SymmetrizingGraphAdder.hpp"
65 #include "Teuchos_MatrixMarket_assignScalar.hpp"
66 #include "Teuchos_MatrixMarket_Banner.hpp"
67 #include "Teuchos_MatrixMarket_CoordDataReader.hpp"
68 #include "Teuchos_SetScientific.hpp"
69 
70 #include <algorithm>
71 #include <fstream>
72 #include <iostream>
73 #include <iterator>
74 #include <vector>
75 #include <stdexcept>
76 #include <numeric>
77 
78 namespace Tpetra {
108  namespace MatrixMarket {
164  template<class SparseMatrixType>
165  class Reader {
166  public:
168  typedef SparseMatrixType sparse_matrix_type;
169  typedef Teuchos::RCP<sparse_matrix_type> sparse_matrix_ptr;
170 
173  typedef typename SparseMatrixType::scalar_type scalar_type;
176  typedef typename SparseMatrixType::local_ordinal_type local_ordinal_type;
184  typedef typename SparseMatrixType::global_ordinal_type
187  typedef typename SparseMatrixType::node_type node_type;
188 
193 
195  typedef MultiVector<scalar_type,
199 
201  typedef Vector<scalar_type,
205 
206  typedef Teuchos::Comm<int> comm_type;
208 
209 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
210  // DEPRECATED typedefs for backwards compatibility.
211  typedef Teuchos::RCP<const comm_type> comm_ptr TPETRA_DEPRECATED;
212  typedef Teuchos::RCP<const map_type> map_ptr TPETRA_DEPRECATED;
213  typedef Teuchos::RCP<node_type> node_ptr TPETRA_DEPRECATED;
214 #endif // TPETRA_ENABLE_DEPRECATED_CODE
215 
216  private:
222  typedef Teuchos::ArrayRCP<int>::size_type size_type;
223 
234  static Teuchos::RCP<const map_type>
235  makeRangeMap (const Teuchos::RCP<const comm_type>& pComm,
236  const global_ordinal_type numRows)
237  {
238  // Return a conventional, uniformly partitioned, contiguous map.
239  return rcp (new map_type (static_cast<global_size_t> (numRows),
240  static_cast<global_ordinal_type> (0),
241  pComm, GloballyDistributed));
242  }
243 
271  static Teuchos::RCP<const map_type>
272  makeRowMap (const Teuchos::RCP<const map_type>& pRowMap,
273  const Teuchos::RCP<const comm_type>& pComm,
274  const global_ordinal_type numRows)
275  {
276  // If the caller didn't provide a map, return a conventional,
277  // uniformly partitioned, contiguous map.
278  if (pRowMap.is_null ()) {
279  return rcp (new map_type (static_cast<global_size_t> (numRows),
280  static_cast<global_ordinal_type> (0),
281  pComm, GloballyDistributed));
282  }
283  else {
284  TEUCHOS_TEST_FOR_EXCEPTION
285  (! pRowMap->isDistributed () && pComm->getSize () > 1,
286  std::invalid_argument, "The specified row map is not distributed, "
287  "but the given communicator includes more than one process (in "
288  "fact, there are " << pComm->getSize () << " processes).");
289  TEUCHOS_TEST_FOR_EXCEPTION
290  (pRowMap->getComm () != pComm, std::invalid_argument,
291  "The specified row Map's communicator (pRowMap->getComm()) "
292  "differs from the given separately supplied communicator pComm.");
293  return pRowMap;
294  }
295  }
296 
311  static Teuchos::RCP<const map_type>
312  makeDomainMap (const Teuchos::RCP<const map_type>& pRangeMap,
313  const global_ordinal_type numRows,
314  const global_ordinal_type numCols)
315  {
316  // Abbreviations so that the map creation call isn't too long.
317  typedef local_ordinal_type LO;
318  typedef global_ordinal_type GO;
319  typedef node_type NT;
320 
321  if (numRows == numCols) {
322  return pRangeMap;
323  } else {
324  return createUniformContigMapWithNode<LO,GO,NT> (numCols,
325  pRangeMap->getComm ());
326  }
327  }
328 
401  static void
402  distribute (Teuchos::ArrayRCP<size_t>& myNumEntriesPerRow,
403  Teuchos::ArrayRCP<size_t>& myRowPtr,
404  Teuchos::ArrayRCP<global_ordinal_type>& myColInd,
405  Teuchos::ArrayRCP<scalar_type>& myValues,
406  const Teuchos::RCP<const map_type>& pRowMap,
407  Teuchos::ArrayRCP<size_t>& numEntriesPerRow,
408  Teuchos::ArrayRCP<size_t>& rowPtr,
409  Teuchos::ArrayRCP<global_ordinal_type>& colInd,
410  Teuchos::ArrayRCP<scalar_type>& values,
411  const bool debug=false)
412  {
413  using Teuchos::arcp;
414  using Teuchos::ArrayRCP;
415  using Teuchos::ArrayView;
416  using Teuchos::as;
417  using Teuchos::Comm;
418  using Teuchos::CommRequest;
419  using Teuchos::null;
420  using Teuchos::RCP;
421  using Teuchos::receive;
422  using Teuchos::send;
423  using std::cerr;
424  using std::endl;
425 
426  const bool extraDebug = false;
427  RCP<const comm_type> pComm = pRowMap->getComm ();
428  const int numProcs = pComm->getSize ();
429  const int myRank = pComm->getRank ();
430  const int rootRank = 0;
431 
432  // Type abbreviations to make the code more concise.
433  typedef global_ordinal_type GO;
434 
435  // List of the global indices of my rows. They may or may
436  // not be contiguous, and the row map need not be one-to-one.
437  ArrayView<const GO> myRows = pRowMap->getNodeElementList();
438  const size_type myNumRows = myRows.size();
439  TEUCHOS_TEST_FOR_EXCEPTION(static_cast<size_t>(myNumRows) !=
440  pRowMap->getNodeNumElements(),
441  std::logic_error,
442  "pRowMap->getNodeElementList().size() = "
443  << myNumRows
444  << " != pRowMap->getNodeNumElements() = "
445  << pRowMap->getNodeNumElements() << ". "
446  "Please report this bug to the Tpetra developers.");
447  TEUCHOS_TEST_FOR_EXCEPTION(myRank == 0 && numEntriesPerRow.size() < myNumRows,
448  std::logic_error,
449  "On Proc 0: numEntriesPerRow.size() = "
450  << numEntriesPerRow.size()
451  << " != pRowMap->getNodeElementList().size() = "
452  << myNumRows << ". Please report this bug to the "
453  "Tpetra developers.");
454 
455  // Space for my proc's number of entries per row. Will be
456  // filled in below.
457  myNumEntriesPerRow = arcp<size_t> (myNumRows);
458 
459  if (myRank != rootRank) {
460  // Tell the root how many rows we have. If we're sending
461  // none, then we don't have anything else to send, nor does
462  // the root have to receive anything else.
463  send (*pComm, myNumRows, rootRank);
464  if (myNumRows != 0) {
465  // Now send my rows' global indices. Hopefully the cast
466  // to int doesn't overflow. This is unlikely, since it
467  // should fit in a LO, even though it is a GO.
468  send (*pComm, static_cast<int> (myNumRows),
469  myRows.getRawPtr(), rootRank);
470 
471  // I (this proc) don't care if my global row indices are
472  // contiguous, though the root proc does (since otherwise
473  // it needs to pack noncontiguous data into contiguous
474  // storage before sending). That's why we don't check
475  // for contiguousness here.
476 
477  // Ask the root process for my part of the array of the
478  // number of entries per row.
479  receive (*pComm, rootRank,
480  static_cast<int> (myNumRows),
481  myNumEntriesPerRow.getRawPtr());
482 
483  // Use the resulting array to figure out how many column
484  // indices and values I should ask from the root process.
485  const local_ordinal_type myNumEntries =
486  std::accumulate (myNumEntriesPerRow.begin(),
487  myNumEntriesPerRow.end(), 0);
488 
489  // Make space for my entries of the sparse matrix. Note
490  // that they don't have to be sorted by row index.
491  // Iterating through all my rows requires computing a
492  // running sum over myNumEntriesPerRow.
493  myColInd = arcp<GO> (myNumEntries);
494  myValues = arcp<scalar_type> (myNumEntries);
495  if (myNumEntries > 0) {
496  // Ask for that many column indices and values, if
497  // there are any.
498  receive (*pComm, rootRank,
499  static_cast<int> (myNumEntries),
500  myColInd.getRawPtr());
501  receive (*pComm, rootRank,
502  static_cast<int> (myNumEntries),
503  myValues.getRawPtr());
504  }
505  } // If I own at least one row
506  } // If I am not the root processor
507  else { // I _am_ the root processor
508  if (debug) {
509  cerr << "-- Proc 0: Copying my data from global arrays" << endl;
510  }
511  // Proc 0 still needs to (allocate, if not done already)
512  // and fill its part of the matrix (my*).
513  for (size_type k = 0; k < myNumRows; ++k) {
514  const GO myCurRow = myRows[k];
515  const local_ordinal_type numEntriesInThisRow = numEntriesPerRow[myCurRow];
516  myNumEntriesPerRow[k] = numEntriesInThisRow;
517  }
518  if (extraDebug && debug) {
519  cerr << "Proc " << pRowMap->getComm ()->getRank ()
520  << ": myNumEntriesPerRow[0.." << (myNumRows-1) << "] = [";
521  for (size_type k = 0; k < myNumRows; ++k) {
522  cerr << myNumEntriesPerRow[k];
523  if (k < myNumRows-1) {
524  cerr << " ";
525  }
526  }
527  cerr << "]" << endl;
528  }
529  // The total number of matrix entries that my proc owns.
530  const local_ordinal_type myNumEntries =
531  std::accumulate (myNumEntriesPerRow.begin(),
532  myNumEntriesPerRow.end(), 0);
533  if (debug) {
534  cerr << "-- Proc 0: I own " << myNumRows << " rows and "
535  << myNumEntries << " entries" << endl;
536  }
537  myColInd = arcp<GO> (myNumEntries);
538  myValues = arcp<scalar_type> (myNumEntries);
539 
540  // Copy Proc 0's part of the matrix into the my* arrays.
541  // It's important that myCurPos be updated _before_ k,
542  // otherwise myCurPos will get the wrong number of entries
543  // per row (it should be for the row in the just-completed
544  // iteration, not for the next iteration's row).
545  local_ordinal_type myCurPos = 0;
546  for (size_type k = 0; k < myNumRows;
547  myCurPos += myNumEntriesPerRow[k], ++k) {
548  const local_ordinal_type curNumEntries = myNumEntriesPerRow[k];
549  const GO myRow = myRows[k];
550  const size_t curPos = rowPtr[myRow];
551  // Only copy if there are entries to copy, in order not
552  // to construct empty ranges for the ArrayRCP views.
553  if (curNumEntries > 0) {
554  ArrayView<GO> colIndView = colInd (curPos, curNumEntries);
555  ArrayView<GO> myColIndView = myColInd (myCurPos, curNumEntries);
556  std::copy (colIndView.begin(), colIndView.end(),
557  myColIndView.begin());
558 
559  ArrayView<scalar_type> valuesView =
560  values (curPos, curNumEntries);
561  ArrayView<scalar_type> myValuesView =
562  myValues (myCurPos, curNumEntries);
563  std::copy (valuesView.begin(), valuesView.end(),
564  myValuesView.begin());
565  }
566  }
567 
568  // Proc 0 processes each other proc p in turn.
569  for (int p = 1; p < numProcs; ++p) {
570  if (debug) {
571  cerr << "-- Proc 0: Processing proc " << p << endl;
572  }
573 
574  size_type theirNumRows = 0;
575  // Ask Proc p how many rows it has. If it doesn't
576  // have any, we can move on to the next proc. This
577  // has to be a standard receive so that we can avoid
578  // the degenerate case of sending zero data.
579  receive (*pComm, p, &theirNumRows);
580  if (debug) {
581  cerr << "-- Proc 0: Proc " << p << " owns "
582  << theirNumRows << " rows" << endl;
583  }
584  if (theirNumRows != 0) {
585  // Ask Proc p which rows it owns. The resulting global
586  // row indices are not guaranteed to be contiguous or
587  // sorted. Global row indices are themselves indices
588  // into the numEntriesPerRow array.
589  ArrayRCP<GO> theirRows = arcp<GO> (theirNumRows);
590  receive (*pComm, p, as<int> (theirNumRows),
591  theirRows.getRawPtr ());
592  // Extra test to make sure that the rows we received
593  // are all sensible. This is a good idea since we are
594  // going to use the global row indices we've received
595  // to index into the numEntriesPerRow array. Better to
596  // catch any bugs here and print a sensible error
597  // message, rather than segfault and print a cryptic
598  // error message.
599  {
600  const global_size_t numRows = pRowMap->getGlobalNumElements ();
601  const GO indexBase = pRowMap->getIndexBase ();
602  bool theirRowsValid = true;
603  for (size_type k = 0; k < theirNumRows; ++k) {
604  if (theirRows[k] < indexBase ||
605  as<global_size_t> (theirRows[k] - indexBase) >= numRows) {
606  theirRowsValid = false;
607  }
608  }
609  if (! theirRowsValid) {
610  TEUCHOS_TEST_FOR_EXCEPTION(
611  ! theirRowsValid, std::logic_error,
612  "Proc " << p << " has at least one invalid row index. "
613  "Here are all of them: " <<
614  Teuchos::toString (theirRows ()) << ". Valid row index "
615  "range (zero-based): [0, " << (numRows - 1) << "].");
616  }
617  }
618 
619  // Perhaps we could save a little work if we check
620  // whether Proc p's row indices are contiguous. That
621  // would make lookups in the global data arrays
622  // faster. For now, we just implement the general
623  // case and don't prematurely optimize. (Remember
624  // that you're making Proc 0 read the whole file, so
625  // you've already lost scalability.)
626 
627  // Compute the number of entries in each of Proc p's
628  // rows. (Proc p will compute its row pointer array
629  // on its own, after it gets the data from Proc 0.)
630  ArrayRCP<size_t> theirNumEntriesPerRow;
631  theirNumEntriesPerRow = arcp<size_t> (theirNumRows);
632  for (size_type k = 0; k < theirNumRows; ++k) {
633  theirNumEntriesPerRow[k] = numEntriesPerRow[theirRows[k]];
634  }
635 
636  // Tell Proc p the number of entries in each of its
637  // rows. Hopefully the cast to int doesn't overflow.
638  // This is unlikely, since it should fit in a LO,
639  // even though it is a GO.
640  send (*pComm, static_cast<int> (theirNumRows),
641  theirNumEntriesPerRow.getRawPtr(), p);
642 
643  // Figure out how many entries Proc p owns.
644  const local_ordinal_type theirNumEntries =
645  std::accumulate (theirNumEntriesPerRow.begin(),
646  theirNumEntriesPerRow.end(), 0);
647 
648  if (debug) {
649  cerr << "-- Proc 0: Proc " << p << " owns "
650  << theirNumEntries << " entries" << endl;
651  }
652 
653  // If there are no entries to send, then we're done
654  // with Proc p.
655  if (theirNumEntries == 0) {
656  continue;
657  }
658 
659  // Construct (views of) proc p's column indices and
660  // values. Later, we might like to optimize for the
661  // (common) contiguous case, for which we don't need to
662  // copy data into separate "their*" arrays (we can just
663  // use contiguous views of the global arrays).
664  ArrayRCP<GO> theirColInd (theirNumEntries);
665  ArrayRCP<scalar_type> theirValues (theirNumEntries);
666  // Copy Proc p's part of the matrix into the their*
667  // arrays. It's important that theirCurPos be updated
668  // _before_ k, otherwise theirCurPos will get the wrong
669  // number of entries per row (it should be for the row
670  // in the just-completed iteration, not for the next
671  // iteration's row).
672  local_ordinal_type theirCurPos = 0;
673  for (size_type k = 0; k < theirNumRows;
674  theirCurPos += theirNumEntriesPerRow[k], k++) {
675  const local_ordinal_type curNumEntries = theirNumEntriesPerRow[k];
676  const GO theirRow = theirRows[k];
677  const local_ordinal_type curPos = rowPtr[theirRow];
678 
679  // Only copy if there are entries to copy, in order
680  // not to construct empty ranges for the ArrayRCP
681  // views.
682  if (curNumEntries > 0) {
683  ArrayView<GO> colIndView =
684  colInd (curPos, curNumEntries);
685  ArrayView<GO> theirColIndView =
686  theirColInd (theirCurPos, curNumEntries);
687  std::copy (colIndView.begin(), colIndView.end(),
688  theirColIndView.begin());
689 
690  ArrayView<scalar_type> valuesView =
691  values (curPos, curNumEntries);
692  ArrayView<scalar_type> theirValuesView =
693  theirValues (theirCurPos, curNumEntries);
694  std::copy (valuesView.begin(), valuesView.end(),
695  theirValuesView.begin());
696  }
697  }
698  // Send Proc p its column indices and values.
699  // Hopefully the cast to int doesn't overflow. This
700  // is unlikely, since it should fit in a LO, even
701  // though it is a GO.
702  send (*pComm, static_cast<int> (theirNumEntries),
703  theirColInd.getRawPtr(), p);
704  send (*pComm, static_cast<int> (theirNumEntries),
705  theirValues.getRawPtr(), p);
706 
707  if (debug) {
708  cerr << "-- Proc 0: Finished with proc " << p << endl;
709  }
710  } // If proc p owns at least one row
711  } // For each proc p not the root proc 0
712  } // If I'm (not) the root proc 0
713 
714  // Invalidate the input data to save space, since we don't
715  // need it anymore.
716  numEntriesPerRow = null;
717  rowPtr = null;
718  colInd = null;
719  values = null;
720 
721  if (debug && myRank == 0) {
722  cerr << "-- Proc 0: About to fill in myRowPtr" << endl;
723  }
724 
725  // Allocate and fill in myRowPtr (the row pointer array for
726  // my rank's rows). We delay this until the end because we
727  // don't need it to compute anything else in distribute().
728  // Each proc can do this work for itself, since it only needs
729  // myNumEntriesPerRow to do so.
730  myRowPtr = arcp<size_t> (myNumRows+1);
731  myRowPtr[0] = 0;
732  for (size_type k = 1; k < myNumRows+1; ++k) {
733  myRowPtr[k] = myRowPtr[k-1] + myNumEntriesPerRow[k-1];
734  }
735  if (extraDebug && debug) {
736  cerr << "Proc " << Teuchos::rank (*(pRowMap->getComm()))
737  << ": myRowPtr[0.." << myNumRows << "] = [";
738  for (size_type k = 0; k < myNumRows+1; ++k) {
739  cerr << myRowPtr[k];
740  if (k < myNumRows) {
741  cerr << " ";
742  }
743  }
744  cerr << "]" << endl << endl;
745  }
746 
747  if (debug && myRank == 0) {
748  cerr << "-- Proc 0: Done with distribute" << endl;
749  }
750  }
751 
765  static Teuchos::RCP<sparse_matrix_type>
766  makeMatrix (Teuchos::ArrayRCP<size_t>& myNumEntriesPerRow,
767  Teuchos::ArrayRCP<size_t>& myRowPtr,
768  Teuchos::ArrayRCP<global_ordinal_type>& myColInd,
769  Teuchos::ArrayRCP<scalar_type>& myValues,
770  const Teuchos::RCP<const map_type>& pRowMap,
771  const Teuchos::RCP<const map_type>& pRangeMap,
772  const Teuchos::RCP<const map_type>& pDomainMap,
773  const bool callFillComplete = true)
774  {
775  using Teuchos::ArrayView;
776  using Teuchos::null;
777  using Teuchos::RCP;
778  using Teuchos::rcp;
779  using std::cerr;
780  using std::endl;
781  // Typedef to make certain type declarations shorter.
782  typedef global_ordinal_type GO;
783 
784  // The row pointer array always has at least one entry, even
785  // if the matrix has zero rows. myNumEntriesPerRow, myColInd,
786  // and myValues would all be empty arrays in that degenerate
787  // case, but the row and domain maps would still be nonnull
788  // (though they would be trivial maps).
789  TEUCHOS_TEST_FOR_EXCEPTION(myRowPtr.is_null(), std::logic_error,
790  "makeMatrix: myRowPtr array is null. "
791  "Please report this bug to the Tpetra developers.");
792  TEUCHOS_TEST_FOR_EXCEPTION(pDomainMap.is_null(), std::logic_error,
793  "makeMatrix: domain map is null. "
794  "Please report this bug to the Tpetra developers.");
795  TEUCHOS_TEST_FOR_EXCEPTION(pRangeMap.is_null(), std::logic_error,
796  "makeMatrix: range map is null. "
797  "Please report this bug to the Tpetra developers.");
798  TEUCHOS_TEST_FOR_EXCEPTION(pRowMap.is_null(), std::logic_error,
799  "makeMatrix: row map is null. "
800  "Please report this bug to the Tpetra developers.");
801 
802  // Construct the CrsMatrix, using the row map, with the
803  // constructor specifying the number of nonzeros for each row.
804  RCP<sparse_matrix_type> A =
805  rcp (new sparse_matrix_type (pRowMap, myNumEntriesPerRow (),
806  StaticProfile));
807 
808  // List of the global indices of my rows.
809  // They may or may not be contiguous.
810  ArrayView<const GO> myRows = pRowMap->getNodeElementList ();
811  const size_type myNumRows = myRows.size ();
812 
813  // Add this processor's matrix entries to the CrsMatrix.
814  const GO indexBase = pRowMap->getIndexBase ();
815  for (size_type i = 0; i < myNumRows; ++i) {
816  const size_type myCurPos = myRowPtr[i];
817  const local_ordinal_type curNumEntries = myNumEntriesPerRow[i];
818  ArrayView<GO> curColInd = myColInd.view (myCurPos, curNumEntries);
819  ArrayView<scalar_type> curValues = myValues.view (myCurPos, curNumEntries);
820 
821  // Modify the column indices in place to have the right index base.
822  for (size_type k = 0; k < curNumEntries; ++k) {
823  curColInd[k] += indexBase;
824  }
825  // Avoid constructing empty views of ArrayRCP objects.
826  if (curNumEntries > 0) {
827  A->insertGlobalValues (myRows[i], curColInd, curValues);
828  }
829  }
830  // We've entered in all our matrix entries, so we can delete
831  // the original data. This will save memory when we call
832  // fillComplete(), so that we never keep more than two copies
833  // of the matrix's data in memory at once.
834  myNumEntriesPerRow = null;
835  myRowPtr = null;
836  myColInd = null;
837  myValues = null;
838 
839  if (callFillComplete) {
840  A->fillComplete (pDomainMap, pRangeMap);
841  }
842  return A;
843  }
844 
850  static Teuchos::RCP<sparse_matrix_type>
851  makeMatrix (Teuchos::ArrayRCP<size_t>& myNumEntriesPerRow,
852  Teuchos::ArrayRCP<size_t>& myRowPtr,
853  Teuchos::ArrayRCP<global_ordinal_type>& myColInd,
854  Teuchos::ArrayRCP<scalar_type>& myValues,
855  const Teuchos::RCP<const map_type>& pRowMap,
856  const Teuchos::RCP<const map_type>& pRangeMap,
857  const Teuchos::RCP<const map_type>& pDomainMap,
858  const Teuchos::RCP<Teuchos::ParameterList>& constructorParams,
859  const Teuchos::RCP<Teuchos::ParameterList>& fillCompleteParams)
860  {
861  using Teuchos::ArrayView;
862  using Teuchos::null;
863  using Teuchos::RCP;
864  using Teuchos::rcp;
865  using std::cerr;
866  using std::endl;
867  // Typedef to make certain type declarations shorter.
868  typedef global_ordinal_type GO;
869 
870  // The row pointer array always has at least one entry, even
871  // if the matrix has zero rows. myNumEntriesPerRow, myColInd,
872  // and myValues would all be empty arrays in that degenerate
873  // case, but the row and domain maps would still be nonnull
874  // (though they would be trivial maps).
875  TEUCHOS_TEST_FOR_EXCEPTION(
876  myRowPtr.is_null(), std::logic_error,
877  "makeMatrix: myRowPtr array is null. "
878  "Please report this bug to the Tpetra developers.");
879  TEUCHOS_TEST_FOR_EXCEPTION(
880  pDomainMap.is_null(), std::logic_error,
881  "makeMatrix: domain map is null. "
882  "Please report this bug to the Tpetra developers.");
883  TEUCHOS_TEST_FOR_EXCEPTION(
884  pRangeMap.is_null(), std::logic_error,
885  "makeMatrix: range map is null. "
886  "Please report this bug to the Tpetra developers.");
887  TEUCHOS_TEST_FOR_EXCEPTION(
888  pRowMap.is_null(), std::logic_error,
889  "makeMatrix: row map is null. "
890  "Please report this bug to the Tpetra developers.");
891 
892  // Construct the CrsMatrix, using the row map, with the
893  // constructor specifying the number of nonzeros for each row.
894  RCP<sparse_matrix_type> A =
895  rcp (new sparse_matrix_type (pRowMap, myNumEntriesPerRow,
896  StaticProfile, constructorParams));
897 
898  // List of the global indices of my rows.
899  // They may or may not be contiguous.
900  ArrayView<const GO> myRows = pRowMap->getNodeElementList();
901  const size_type myNumRows = myRows.size();
902 
903  // Add this processor's matrix entries to the CrsMatrix.
904  const GO indexBase = pRowMap->getIndexBase ();
905  for (size_type i = 0; i < myNumRows; ++i) {
906  const size_type myCurPos = myRowPtr[i];
907  const local_ordinal_type curNumEntries = myNumEntriesPerRow[i];
908  ArrayView<GO> curColInd = myColInd.view (myCurPos, curNumEntries);
909  ArrayView<scalar_type> curValues = myValues.view (myCurPos, curNumEntries);
910 
911  // Modify the column indices in place to have the right index base.
912  for (size_type k = 0; k < curNumEntries; ++k) {
913  curColInd[k] += indexBase;
914  }
915  if (curNumEntries > 0) {
916  A->insertGlobalValues (myRows[i], curColInd, curValues);
917  }
918  }
919  // We've entered in all our matrix entries, so we can delete
920  // the original data. This will save memory when we call
921  // fillComplete(), so that we never keep more than two copies
922  // of the matrix's data in memory at once.
923  myNumEntriesPerRow = null;
924  myRowPtr = null;
925  myColInd = null;
926  myValues = null;
927 
928  A->fillComplete (pDomainMap, pRangeMap, fillCompleteParams);
929  return A;
930  }
931 
936  static Teuchos::RCP<sparse_matrix_type>
937  makeMatrix (Teuchos::ArrayRCP<size_t>& myNumEntriesPerRow,
938  Teuchos::ArrayRCP<size_t>& myRowPtr,
939  Teuchos::ArrayRCP<global_ordinal_type>& myColInd,
940  Teuchos::ArrayRCP<scalar_type>& myValues,
941  const Teuchos::RCP<const map_type>& rowMap,
942  Teuchos::RCP<const map_type>& colMap,
943  const Teuchos::RCP<const map_type>& domainMap,
944  const Teuchos::RCP<const map_type>& rangeMap,
945  const bool callFillComplete = true)
946  {
947  using Teuchos::ArrayView;
948  using Teuchos::as;
949  using Teuchos::null;
950  using Teuchos::RCP;
951  using Teuchos::rcp;
952  typedef global_ordinal_type GO;
953 
954  // Construct the CrsMatrix.
955 
956  RCP<sparse_matrix_type> A; // the matrix to return.
957  if (colMap.is_null ()) { // the user didn't provide a column Map
958  A = rcp (new sparse_matrix_type (rowMap, myNumEntriesPerRow, StaticProfile));
959  } else { // the user provided a column Map
960  A = rcp (new sparse_matrix_type (rowMap, colMap, myNumEntriesPerRow, StaticProfile));
961  }
962 
963  // List of the global indices of my rows.
964  // They may or may not be contiguous.
965  ArrayView<const GO> myRows = rowMap->getNodeElementList ();
966  const size_type myNumRows = myRows.size ();
967 
968  // Add this process' matrix entries to the CrsMatrix.
969  const GO indexBase = rowMap->getIndexBase ();
970  for (size_type i = 0; i < myNumRows; ++i) {
971  const size_type myCurPos = myRowPtr[i];
972  const size_type curNumEntries = as<size_type> (myNumEntriesPerRow[i]);
973  ArrayView<GO> curColInd = myColInd.view (myCurPos, curNumEntries);
974  ArrayView<scalar_type> curValues = myValues.view (myCurPos, curNumEntries);
975 
976  // Modify the column indices in place to have the right index base.
977  for (size_type k = 0; k < curNumEntries; ++k) {
978  curColInd[k] += indexBase;
979  }
980  if (curNumEntries > 0) {
981  A->insertGlobalValues (myRows[i], curColInd, curValues);
982  }
983  }
984  // We've entered in all our matrix entries, so we can delete
985  // the original data. This will save memory when we call
986  // fillComplete(), so that we never keep more than two copies
987  // of the matrix's data in memory at once.
988  myNumEntriesPerRow = null;
989  myRowPtr = null;
990  myColInd = null;
991  myValues = null;
992 
993  if (callFillComplete) {
994  A->fillComplete (domainMap, rangeMap);
995  if (colMap.is_null ()) {
996  colMap = A->getColMap ();
997  }
998  }
999  return A;
1000  }
1001 
1002  private:
1003 
1020  static Teuchos::RCP<const Teuchos::MatrixMarket::Banner>
1021  readBanner (std::istream& in,
1022  size_t& lineNumber,
1023  const bool tolerant=false,
1024  const bool /* debug */=false,
1025  const bool isGraph=false)
1026  {
1027  using Teuchos::MatrixMarket::Banner;
1028  using Teuchos::RCP;
1029  using Teuchos::rcp;
1030  using std::cerr;
1031  using std::endl;
1032  typedef Teuchos::ScalarTraits<scalar_type> STS;
1033 
1034  RCP<Banner> pBanner; // On output, if successful: the read-in Banner.
1035  std::string line; // If read from stream successful: the Banner line
1036 
1037  // Try to read a line from the input stream.
1038  const bool readFailed = ! getline(in, line);
1039  TEUCHOS_TEST_FOR_EXCEPTION(readFailed, std::invalid_argument,
1040  "Failed to get Matrix Market banner line from input.");
1041 
1042  // We read a line from the input stream.
1043  lineNumber++;
1044 
1045  // Assume that the line we found is the Banner line.
1046  try {
1047  pBanner = rcp (new Banner (line, tolerant));
1048  } catch (std::exception& e) {
1049  TEUCHOS_TEST_FOR_EXCEPTION(true, std::invalid_argument,
1050  "Matrix Market banner line contains syntax error(s): "
1051  << e.what());
1052  }
1053  TEUCHOS_TEST_FOR_EXCEPTION(pBanner->objectType() != "matrix",
1054  std::invalid_argument, "The Matrix Market file does not contain "
1055  "matrix data. Its Banner (first) line says that its object type is \""
1056  << pBanner->matrixType() << "\", rather than the required \"matrix\".");
1057 
1058  // Validate the data type of the matrix, with respect to the
1059  // Scalar type of the CrsMatrix entries.
1060  TEUCHOS_TEST_FOR_EXCEPTION(
1061  ! STS::isComplex && pBanner->dataType() == "complex",
1062  std::invalid_argument,
1063  "The Matrix Market file contains complex-valued data, but you are "
1064  "trying to read it into a matrix containing entries of the real-"
1065  "valued Scalar type \""
1066  << Teuchos::TypeNameTraits<scalar_type>::name() << "\".");
1067  TEUCHOS_TEST_FOR_EXCEPTION(
1068  !isGraph &&
1069  pBanner->dataType() != "real" &&
1070  pBanner->dataType() != "complex" &&
1071  pBanner->dataType() != "integer",
1072  std::invalid_argument,
1073  "When reading Matrix Market data into a Tpetra::CrsMatrix, the "
1074  "Matrix Market file may not contain a \"pattern\" matrix. A "
1075  "pattern matrix is really just a graph with no weights. It "
1076  "should be stored in a CrsGraph, not a CrsMatrix.");
1077 
1078  TEUCHOS_TEST_FOR_EXCEPTION(
1079  isGraph &&
1080  pBanner->dataType() != "pattern",
1081  std::invalid_argument,
1082  "When reading Matrix Market data into a Tpetra::CrsGraph, the "
1083  "Matrix Market file must contain a \"pattern\" matrix.");
1084 
1085  return pBanner;
1086  }
1087 
1110  static Teuchos::Tuple<global_ordinal_type, 3>
1111  readCoordDims (std::istream& in,
1112  size_t& lineNumber,
1113  const Teuchos::RCP<const Teuchos::MatrixMarket::Banner>& pBanner,
1114  const Teuchos::RCP<const comm_type>& pComm,
1115  const bool tolerant = false,
1116  const bool /* debug */ = false)
1117  {
1118  using Teuchos::MatrixMarket::readCoordinateDimensions;
1119  using Teuchos::Tuple;
1120 
1121  // Packed coordinate matrix dimensions (numRows, numCols,
1122  // numNonzeros); computed on Rank 0 and broadcasted to all MPI
1123  // ranks.
1124  Tuple<global_ordinal_type, 3> dims;
1125 
1126  // Read in the coordinate matrix dimensions from the input
1127  // stream. "success" tells us whether reading in the
1128  // coordinate matrix dimensions succeeded ("Guilty unless
1129  // proven innocent").
1130  bool success = false;
1131  if (pComm->getRank() == 0) {
1132  TEUCHOS_TEST_FOR_EXCEPTION(pBanner->matrixType() != "coordinate",
1133  std::invalid_argument, "The Tpetra::CrsMatrix Matrix Market reader "
1134  "only accepts \"coordinate\" (sparse) matrix data.");
1135  // Unpacked coordinate matrix dimensions
1136  global_ordinal_type numRows, numCols, numNonzeros;
1137  // Only MPI Rank 0 reads from the input stream
1138  success = readCoordinateDimensions (in, numRows, numCols,
1139  numNonzeros, lineNumber,
1140  tolerant);
1141  // Pack up the data into a Tuple so we can send them with
1142  // one broadcast instead of three.
1143  dims[0] = numRows;
1144  dims[1] = numCols;
1145  dims[2] = numNonzeros;
1146  }
1147  // Only Rank 0 did the reading, so it decides success.
1148  //
1149  // FIXME (mfh 02 Feb 2011) Teuchos::broadcast doesn't know how
1150  // to send bools. For now, we convert to/from int instead,
1151  // using the usual "true is 1, false is 0" encoding.
1152  {
1153  int the_success = success ? 1 : 0; // only matters on MPI Rank 0
1154  Teuchos::broadcast (*pComm, 0, &the_success);
1155  success = (the_success == 1);
1156  }
1157  if (success) {
1158  // Broadcast (numRows, numCols, numNonzeros) from Rank 0
1159  // to all the other MPI ranks.
1160  Teuchos::broadcast (*pComm, 0, dims);
1161  }
1162  else {
1163  // Perhaps in tolerant mode, we could set all the
1164  // dimensions to zero for now, and deduce correct
1165  // dimensions by reading all of the file's entries and
1166  // computing the max(row index) and max(column index).
1167  // However, for now we just error out in that case.
1168  TEUCHOS_TEST_FOR_EXCEPTION(true, std::invalid_argument,
1169  "Error reading Matrix Market sparse matrix: failed to read "
1170  "coordinate matrix dimensions.");
1171  }
1172  return dims;
1173  }
1174 
1185  typedef Teuchos::MatrixMarket::SymmetrizingAdder<Teuchos::MatrixMarket::Raw::Adder<scalar_type, global_ordinal_type> > adder_type;
1186 
1187  typedef Teuchos::MatrixMarket::SymmetrizingGraphAdder<Teuchos::MatrixMarket::Raw::GraphAdder<global_ordinal_type> > graph_adder_type;
1188 
1214  static Teuchos::RCP<adder_type>
1215  makeAdder (const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
1216  Teuchos::RCP<const Teuchos::MatrixMarket::Banner>& pBanner,
1217  const Teuchos::Tuple<global_ordinal_type, 3>& dims,
1218  const bool tolerant=false,
1219  const bool debug=false)
1220  {
1221  if (pComm->getRank () == 0) {
1222  typedef Teuchos::MatrixMarket::Raw::Adder<scalar_type,
1224  raw_adder_type;
1225  Teuchos::RCP<raw_adder_type> pRaw =
1226  Teuchos::rcp (new raw_adder_type (dims[0], dims[1], dims[2],
1227  tolerant, debug));
1228  return Teuchos::rcp (new adder_type (pRaw, pBanner->symmType ()));
1229  }
1230  else {
1231  return Teuchos::null;
1232  }
1233  }
1234 
1260  static Teuchos::RCP<graph_adder_type>
1261  makeGraphAdder (const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
1262  Teuchos::RCP<const Teuchos::MatrixMarket::Banner>& pBanner,
1263  const Teuchos::Tuple<global_ordinal_type, 3>& dims,
1264  const bool tolerant=false,
1265  const bool debug=false)
1266  {
1267  if (pComm->getRank () == 0) {
1268  typedef Teuchos::MatrixMarket::Raw::GraphAdder<global_ordinal_type> raw_adder_type;
1269  Teuchos::RCP<raw_adder_type> pRaw =
1270  Teuchos::rcp (new raw_adder_type (dims[0], dims[1], dims[2],
1271  tolerant, debug));
1272  return Teuchos::rcp (new graph_adder_type (pRaw, pBanner->symmType ()));
1273  }
1274  else {
1275  return Teuchos::null;
1276  }
1277  }
1278 
1280  static Teuchos::RCP<sparse_graph_type>
1281  readSparseGraphHelper (std::istream& in,
1282  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
1283  const Teuchos::RCP<const map_type>& rowMap,
1284  Teuchos::RCP<const map_type>& colMap,
1285  const Teuchos::RCP<Teuchos::ParameterList>& constructorParams,
1286  const bool tolerant,
1287  const bool debug)
1288  {
1289  using Teuchos::MatrixMarket::Banner;
1290  using Teuchos::RCP;
1291  using Teuchos::ptr;
1292  using Teuchos::Tuple;
1293  using std::cerr;
1294  using std::endl;
1295 
1296  const int myRank = pComm->getRank ();
1297  const int rootRank = 0;
1298 
1299  // Current line number in the input stream. Various calls
1300  // will modify this depending on the number of lines that are
1301  // read from the input stream. Only Rank 0 modifies this.
1302  size_t lineNumber = 1;
1303 
1304  if (debug && myRank == rootRank) {
1305  cerr << "Matrix Market reader: readGraph:" << endl
1306  << "-- Reading banner line" << endl;
1307  }
1308 
1309  // The "Banner" tells you whether the input stream represents
1310  // a sparse matrix, the symmetry type of the matrix, and the
1311  // type of the data it contains.
1312  //
1313  // pBanner will only be nonnull on MPI Rank 0. It will be
1314  // null on all other MPI processes.
1315  RCP<const Banner> pBanner;
1316  {
1317  // We read and validate the Banner on Proc 0, but broadcast
1318  // the validation result to all processes.
1319  // Teuchos::broadcast doesn't currently work with bool, so
1320  // we use int (true -> 1, false -> 0).
1321  int bannerIsCorrect = 1;
1322  std::ostringstream errMsg;
1323 
1324  if (myRank == rootRank) {
1325  // Read the Banner line from the input stream.
1326  try {
1327  pBanner = readBanner (in, lineNumber, tolerant, debug, true);
1328  }
1329  catch (std::exception& e) {
1330  errMsg << "Attempt to read the Matrix Market file's Banner line "
1331  "threw an exception: " << e.what();
1332  bannerIsCorrect = 0;
1333  }
1334 
1335  if (bannerIsCorrect) {
1336  // Validate the Banner for the case of a sparse graph.
1337  // We validate on Proc 0, since it reads the Banner.
1338 
1339  // In intolerant mode, the matrix type must be "coordinate".
1340  if (! tolerant && pBanner->matrixType() != "coordinate") {
1341  bannerIsCorrect = 0;
1342  errMsg << "The Matrix Market input file must contain a "
1343  "\"coordinate\"-format sparse graph in order to create a "
1344  "Tpetra::CrsGraph object from it, but the file's matrix "
1345  "type is \"" << pBanner->matrixType() << "\" instead.";
1346  }
1347  // In tolerant mode, we allow the matrix type to be
1348  // anything other than "array" (which would mean that
1349  // the file contains a dense matrix).
1350  if (tolerant && pBanner->matrixType() == "array") {
1351  bannerIsCorrect = 0;
1352  errMsg << "Matrix Market file must contain a \"coordinate\"-"
1353  "format sparse graph in order to create a Tpetra::CrsGraph "
1354  "object from it, but the file's matrix type is \"array\" "
1355  "instead. That probably means the file contains dense matrix "
1356  "data.";
1357  }
1358  }
1359  } // Proc 0: Done reading the Banner, hopefully successfully.
1360 
1361  // Broadcast from Proc 0 whether the Banner was read correctly.
1362  broadcast (*pComm, rootRank, ptr (&bannerIsCorrect));
1363 
1364  // If the Banner is invalid, all processes throw an
1365  // exception. Only Proc 0 gets the exception message, but
1366  // that's OK, since the main point is to "stop the world"
1367  // (rather than throw an exception on one process and leave
1368  // the others hanging).
1369  TEUCHOS_TEST_FOR_EXCEPTION(bannerIsCorrect == 0,
1370  std::invalid_argument, errMsg.str ());
1371  } // Done reading the Banner line and broadcasting success.
1372  if (debug && myRank == rootRank) {
1373  cerr << "-- Reading dimensions line" << endl;
1374  }
1375 
1376  // Read the graph dimensions from the Matrix Market metadata.
1377  // dims = (numRows, numCols, numEntries). Proc 0 does the
1378  // reading, but it broadcasts the results to all MPI
1379  // processes. Thus, readCoordDims() is a collective
1380  // operation. It does a collective check for correctness too.
1381  Tuple<global_ordinal_type, 3> dims =
1382  readCoordDims (in, lineNumber, pBanner, pComm, tolerant, debug);
1383 
1384  if (debug && myRank == rootRank) {
1385  cerr << "-- Making Adder for collecting graph data" << endl;
1386  }
1387 
1388  // "Adder" object for collecting all the sparse graph entries
1389  // from the input stream. This is only nonnull on Proc 0.
1390  // The Adder internally converts the one-based indices (native
1391  // Matrix Market format) into zero-based indices.
1392  RCP<graph_adder_type> pAdder =
1393  makeGraphAdder (pComm, pBanner, dims, tolerant, debug);
1394 
1395  if (debug && myRank == rootRank) {
1396  cerr << "-- Reading graph data" << endl;
1397  }
1398  //
1399  // Read the graph entries from the input stream on Proc 0.
1400  //
1401  {
1402  // We use readSuccess to broadcast the results of the read
1403  // (succeeded or not) to all MPI processes. Since
1404  // Teuchos::broadcast doesn't currently know how to send
1405  // bools, we convert to int (true -> 1, false -> 0).
1406  int readSuccess = 1;
1407  std::ostringstream errMsg; // Exception message (only valid on Proc 0)
1408  if (myRank == rootRank) {
1409  try {
1410  // Reader for "coordinate" format sparse graph data.
1411  typedef Teuchos::MatrixMarket::CoordPatternReader<graph_adder_type,
1412  global_ordinal_type> reader_type;
1413  reader_type reader (pAdder);
1414 
1415  // Read the sparse graph entries.
1416  std::pair<bool, std::vector<size_t> > results =
1417  reader.read (in, lineNumber, tolerant, debug);
1418  readSuccess = results.first ? 1 : 0;
1419  }
1420  catch (std::exception& e) {
1421  readSuccess = 0;
1422  errMsg << e.what();
1423  }
1424  }
1425  broadcast (*pComm, rootRank, ptr (&readSuccess));
1426 
1427  // It would be nice to add a "verbose" flag, so that in
1428  // tolerant mode, we could log any bad line number(s) on
1429  // Proc 0. For now, we just throw if the read fails to
1430  // succeed.
1431  //
1432  // Question: If we're in tolerant mode, and if the read did
1433  // not succeed, should we attempt to call fillComplete()?
1434  TEUCHOS_TEST_FOR_EXCEPTION(readSuccess == 0, std::runtime_error,
1435  "Failed to read the Matrix Market sparse graph file: "
1436  << errMsg.str());
1437  } // Done reading the graph entries (stored on Proc 0 for now)
1438 
1439  if (debug && myRank == rootRank) {
1440  cerr << "-- Successfully read the Matrix Market data" << endl;
1441  }
1442 
1443  // In tolerant mode, we need to rebroadcast the graph
1444  // dimensions, since they may be different after reading the
1445  // actual graph data. We only need to broadcast the number
1446  // of rows and columns. Only Rank 0 needs to know the actual
1447  // global number of entries, since (a) we need to merge
1448  // duplicates on Rank 0 first anyway, and (b) when we
1449  // distribute the entries, each rank other than Rank 0 will
1450  // only need to know how many entries it owns, not the total
1451  // number of entries.
1452  if (tolerant) {
1453  if (debug && myRank == rootRank) {
1454  cerr << "-- Tolerant mode: rebroadcasting graph dimensions"
1455  << endl
1456  << "----- Dimensions before: "
1457  << dims[0] << " x " << dims[1]
1458  << endl;
1459  }
1460  // Packed coordinate graph dimensions (numRows, numCols).
1461  Tuple<global_ordinal_type, 2> updatedDims;
1462  if (myRank == rootRank) {
1463  // If one or more bottom rows of the graph contain no
1464  // entries, then the Adder will report that the number
1465  // of rows is less than that specified in the
1466  // metadata. We allow this case, and favor the
1467  // metadata so that the zero row(s) will be included.
1468  updatedDims[0] =
1469  std::max (dims[0], pAdder->getAdder()->numRows());
1470  updatedDims[1] = pAdder->getAdder()->numCols();
1471  }
1472  broadcast (*pComm, rootRank, updatedDims);
1473  dims[0] = updatedDims[0];
1474  dims[1] = updatedDims[1];
1475  if (debug && myRank == rootRank) {
1476  cerr << "----- Dimensions after: " << dims[0] << " x "
1477  << dims[1] << endl;
1478  }
1479  }
1480  else {
1481  // In strict mode, we require that the graph's metadata and
1482  // its actual data agree, at least somewhat. In particular,
1483  // the number of rows must agree, since otherwise we cannot
1484  // distribute the graph correctly.
1485 
1486  // Teuchos::broadcast() doesn't know how to broadcast bools,
1487  // so we use an int with the standard 1 == true, 0 == false
1488  // encoding.
1489  int dimsMatch = 1;
1490  if (myRank == rootRank) {
1491  // If one or more bottom rows of the graph contain no
1492  // entries, then the Adder will report that the number of
1493  // rows is less than that specified in the metadata. We
1494  // allow this case, and favor the metadata, but do not
1495  // allow the Adder to think there are more rows in the
1496  // graph than the metadata says.
1497  if (dims[0] < pAdder->getAdder ()->numRows ()) {
1498  dimsMatch = 0;
1499  }
1500  }
1501  broadcast (*pComm, 0, ptr (&dimsMatch));
1502  if (dimsMatch == 0) {
1503  // We're in an error state anyway, so we might as well
1504  // work a little harder to print an informative error
1505  // message.
1506  //
1507  // Broadcast the Adder's idea of the graph dimensions
1508  // from Proc 0 to all processes.
1509  Tuple<global_ordinal_type, 2> addersDims;
1510  if (myRank == rootRank) {
1511  addersDims[0] = pAdder->getAdder()->numRows();
1512  addersDims[1] = pAdder->getAdder()->numCols();
1513  }
1514  broadcast (*pComm, 0, addersDims);
1515  TEUCHOS_TEST_FOR_EXCEPTION(
1516  dimsMatch == 0, std::runtime_error,
1517  "The graph metadata says that the graph is " << dims[0] << " x "
1518  << dims[1] << ", but the actual data says that the graph is "
1519  << addersDims[0] << " x " << addersDims[1] << ". That means the "
1520  "data includes more rows than reported in the metadata. This "
1521  "is not allowed when parsing in strict mode. Parse the graph in "
1522  "tolerant mode to ignore the metadata when it disagrees with the "
1523  "data.");
1524  }
1525  } // Matrix dimensions (# rows, # cols, # entries) agree.
1526 
1527  // Create a map describing a distribution where the root owns EVERYTHING
1528  RCP<map_type> proc0Map;
1529  global_ordinal_type indexBase;
1530  if(Teuchos::is_null(rowMap)) {
1531  indexBase = 0;
1532  }
1533  else {
1534  indexBase = rowMap->getIndexBase();
1535  }
1536  if(myRank == rootRank) {
1537  proc0Map = rcp(new map_type(dims[0],dims[0],indexBase,pComm));
1538  }
1539  else {
1540  proc0Map = rcp(new map_type(dims[0],0,indexBase,pComm));
1541  }
1542 
1543  // Create the graph where the root owns EVERYTHING
1544  std::map<global_ordinal_type, size_t> numEntriesPerRow_map;
1545  if (myRank == rootRank) {
1546  const auto& entries = pAdder()->getAdder()->getEntries();
1547  // This will count duplicates, but it's better than dense.
1548  // An even better approach would use a classic algorithm,
1549  // likely in Saad's old textbook, for converting COO (entries)
1550  // to CSR (the local part of the sparse matrix data structure).
1551  for (const auto& entry : entries) {
1552  const global_ordinal_type gblRow = entry.rowIndex () + indexBase;
1553  ++numEntriesPerRow_map[gblRow];
1554  }
1555  }
1556 
1557  Teuchos::Array<size_t> numEntriesPerRow (proc0Map->getNodeNumElements ());
1558  for (const auto& ent : numEntriesPerRow_map) {
1559  const local_ordinal_type lclRow = proc0Map->getLocalElement (ent.first);
1560  numEntriesPerRow[lclRow] = ent.second;
1561  }
1562  // Free anything we don't need before allocating the graph.
1563  // Swapping with an empty data structure is the standard idiom
1564  // for freeing memory used by Standard Library containers.
1565  // (Just resizing to 0 doesn't promise to free memory.)
1566  {
1567  std::map<global_ordinal_type, size_t> empty_map;
1568  std::swap (numEntriesPerRow_map, empty_map);
1569  }
1570 
1571  RCP<sparse_graph_type> proc0Graph =
1572  rcp(new sparse_graph_type(proc0Map,numEntriesPerRow (),
1573  StaticProfile,constructorParams));
1574  if(myRank == rootRank) {
1575  typedef Teuchos::MatrixMarket::Raw::GraphElement<global_ordinal_type> element_type;
1576 
1577  // Get the entries
1578  const std::vector<element_type>& entries =
1579  pAdder->getAdder()->getEntries();
1580 
1581  // Insert them one at a time
1582  for(size_t curPos=0; curPos<entries.size(); curPos++) {
1583  const element_type& curEntry = entries[curPos];
1584  const global_ordinal_type curRow = curEntry.rowIndex()+indexBase;
1585  const global_ordinal_type curCol = curEntry.colIndex()+indexBase;
1586  Teuchos::ArrayView<const global_ordinal_type> colView(&curCol,1);
1587  proc0Graph->insertGlobalIndices(curRow,colView);
1588  }
1589  }
1590  proc0Graph->fillComplete();
1591 
1592  RCP<sparse_graph_type> distGraph;
1593  if(Teuchos::is_null(rowMap))
1594  {
1595  // Create a map describing the distribution we actually want
1596  RCP<map_type> distMap =
1597  rcp(new map_type(dims[0],0,pComm,GloballyDistributed));
1598 
1599  // Create the graph with that distribution too
1600  distGraph = rcp(new sparse_graph_type(distMap,colMap,0,StaticProfile,constructorParams));
1601 
1602  // Create an importer/exporter/vandelay to redistribute the graph
1603  typedef Import<local_ordinal_type, global_ordinal_type, node_type> import_type;
1604  import_type importer (proc0Map, distMap);
1605 
1606  // Import the data
1607  distGraph->doImport(*proc0Graph,importer,INSERT);
1608  }
1609  else {
1610  distGraph = rcp(new sparse_graph_type(rowMap,colMap,0,StaticProfile,constructorParams));
1611 
1612  // Create an importer/exporter/vandelay to redistribute the graph
1613  typedef Import<local_ordinal_type, global_ordinal_type, node_type> import_type;
1614  import_type importer (proc0Map, rowMap);
1615 
1616  // Import the data
1617  distGraph->doImport(*proc0Graph,importer,INSERT);
1618  }
1619 
1620  return distGraph;
1621  }
1622 
1623  public:
1647  static Teuchos::RCP<sparse_graph_type>
1648  readSparseGraphFile (const std::string& filename,
1649  const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
1650  const bool callFillComplete=true,
1651  const bool tolerant=false,
1652  const bool debug=false)
1653  {
1654  using Teuchos::broadcast;
1655  using Teuchos::outArg;
1656 
1657  // Only open the file on Process 0. Test carefully to make
1658  // sure that the file opened successfully (and broadcast that
1659  // result to all processes to prevent a hang on exception
1660  // throw), since it's a common mistake to misspell a filename.
1661  std::ifstream in;
1662  int opened = 0;
1663  if (comm->getRank () == 0) {
1664  try {
1665  in.open (filename.c_str ());
1666  opened = 1;
1667  }
1668  catch (...) {
1669  opened = 0;
1670  }
1671  }
1672  broadcast<int, int> (*comm, 0, outArg (opened));
1673  TEUCHOS_TEST_FOR_EXCEPTION
1674  (opened == 0, std::runtime_error, "readSparseGraphFile: "
1675  "Failed to open file \"" << filename << "\" on Process 0.");
1676  return readSparseGraph (in, comm,
1677  callFillComplete,
1678  tolerant, debug);
1679  // We can rely on the destructor of the input stream to close
1680  // the file on scope exit, even if readSparseGraph() throws an
1681  // exception.
1682  }
1683 
1684 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
1685  static Teuchos::RCP<sparse_graph_type> TPETRA_DEPRECATED
1692  readSparseGraphFile (const std::string& filename,
1693  const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
1694  const Teuchos::RCP<node_type>& /* pNode*/,
1695  const bool callFillComplete=true,
1696  const bool tolerant=false,
1697  const bool debug=false)
1698  {
1699  // Call the overload below.
1700  return readSparseGraphFile (filename, comm,
1701  callFillComplete, tolerant, debug);
1702  }
1703 #endif // TPETRA_ENABLE_DEPRECATED_CODE
1704 
1733  static Teuchos::RCP<sparse_graph_type>
1734  readSparseGraphFile (const std::string& filename,
1735  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
1736  const Teuchos::RCP<Teuchos::ParameterList>& constructorParams,
1737  const Teuchos::RCP<Teuchos::ParameterList>& fillCompleteParams,
1738  const bool tolerant=false,
1739  const bool debug=false)
1740  {
1741  using Teuchos::broadcast;
1742  using Teuchos::outArg;
1743 
1744  // Only open the file on Process 0. Test carefully to make
1745  // sure that the file opened successfully (and broadcast that
1746  // result to all processes to prevent a hang on exception
1747  // throw), since it's a common mistake to misspell a filename.
1748  std::ifstream in;
1749  int opened = 0;
1750  if (pComm->getRank () == 0) {
1751  try {
1752  in.open (filename.c_str ());
1753  opened = 1;
1754  }
1755  catch (...) {
1756  opened = 0;
1757  }
1758  }
1759  broadcast<int, int> (*pComm, 0, outArg (opened));
1760  TEUCHOS_TEST_FOR_EXCEPTION
1761  (opened == 0, std::runtime_error, "readSparseGraphFile: "
1762  "Failed to open file \"" << filename << "\" on Process 0.");
1763  if (pComm->getRank () == 0) { // only open the input file on Process 0
1764  in.open (filename.c_str ());
1765  }
1766  return readSparseGraph (in, pComm,
1767  constructorParams,
1768  fillCompleteParams, tolerant, debug);
1769  // We can rely on the destructor of the input stream to close
1770  // the file on scope exit, even if readSparseGraph() throws an
1771  // exception.
1772  }
1773 
1774 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
1775  static Teuchos::RCP<sparse_graph_type> TPETRA_DEPRECATED
1782  readSparseGraphFile (const std::string& filename,
1783  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
1784  const Teuchos::RCP<node_type>& /*pNode*/,
1785  const Teuchos::RCP<Teuchos::ParameterList>& constructorParams,
1786  const Teuchos::RCP<Teuchos::ParameterList>& fillCompleteParams,
1787  const bool tolerant=false,
1788  const bool debug=false)
1789  {
1790  // Call the overload below.
1791  return readSparseGraphFile (filename, pComm,
1792  constructorParams, fillCompleteParams,
1793  tolerant, debug);
1794  }
1795 #endif // TPETRA_ENABLE_DEPRECATED_CODE
1796 
1834  static Teuchos::RCP<sparse_graph_type>
1835  readSparseGraphFile (const std::string& filename,
1836  const Teuchos::RCP<const map_type>& rowMap,
1837  Teuchos::RCP<const map_type>& colMap,
1838  const Teuchos::RCP<const map_type>& domainMap,
1839  const Teuchos::RCP<const map_type>& rangeMap,
1840  const bool callFillComplete=true,
1841  const bool tolerant=false,
1842  const bool debug=false)
1843  {
1844  using Teuchos::broadcast;
1845  using Teuchos::Comm;
1846  using Teuchos::outArg;
1847  using Teuchos::RCP;
1848 
1849  TEUCHOS_TEST_FOR_EXCEPTION
1850  (rowMap.is_null (), std::invalid_argument,
1851  "Input rowMap must be nonnull.");
1852  RCP<const Comm<int> > comm = rowMap->getComm ();
1853  if (comm.is_null ()) {
1854  // If the input communicator is null on some process, then
1855  // that process does not participate in the collective.
1856  return Teuchos::null;
1857  }
1858 
1859  // Only open the file on Process 0. Test carefully to make
1860  // sure that the file opened successfully (and broadcast that
1861  // result to all processes to prevent a hang on exception
1862  // throw), since it's a common mistake to misspell a filename.
1863  std::ifstream in;
1864  int opened = 0;
1865  if (comm->getRank () == 0) {
1866  try {
1867  in.open (filename.c_str ());
1868  opened = 1;
1869  }
1870  catch (...) {
1871  opened = 0;
1872  }
1873  }
1874  broadcast<int, int> (*comm, 0, outArg (opened));
1875  TEUCHOS_TEST_FOR_EXCEPTION
1876  (opened == 0, std::runtime_error, "readSparseGraphFile: "
1877  "Failed to open file \"" << filename << "\" on Process 0.");
1878  return readSparseGraph (in, rowMap, colMap, domainMap, rangeMap,
1879  callFillComplete, tolerant, debug);
1880  }
1881 
1907  static Teuchos::RCP<sparse_graph_type>
1908  readSparseGraph (std::istream& in,
1909  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
1910  const bool callFillComplete=true,
1911  const bool tolerant=false,
1912  const bool debug=false)
1913  {
1914  Teuchos::RCP<const map_type> fakeRowMap;
1915  Teuchos::RCP<const map_type> fakeColMap;
1916  Teuchos::RCP<Teuchos::ParameterList> fakeCtorParams;
1917 
1918  Teuchos::RCP<sparse_graph_type> graph =
1919  readSparseGraphHelper (in, pComm,
1920  fakeRowMap, fakeColMap,
1921  fakeCtorParams, tolerant, debug);
1922  if (callFillComplete) {
1923  graph->fillComplete ();
1924  }
1925  return graph;
1926  }
1927 
1928 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
1929  static Teuchos::RCP<sparse_graph_type> TPETRA_DEPRECATED
1931  readSparseGraph (std::istream& in,
1932  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
1933  const Teuchos::RCP<node_type>& /* pNode */,
1934  const bool callFillComplete=true,
1935  const bool tolerant=false,
1936  const bool debug=false)
1937  {
1938  // Call the overload below.
1939  return readSparseGraph (in, pComm, callFillComplete,
1940  tolerant, debug);
1941  }
1942 #endif // TPETRA_ENABLE_DEPRECATED_CODE
1943 
1973  static Teuchos::RCP<sparse_graph_type>
1974  readSparseGraph (std::istream& in,
1975  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
1976  const Teuchos::RCP<Teuchos::ParameterList>& constructorParams,
1977  const Teuchos::RCP<Teuchos::ParameterList>& fillCompleteParams,
1978  const bool tolerant=false,
1979  const bool debug=false)
1980  {
1981  Teuchos::RCP<const map_type> fakeRowMap;
1982  Teuchos::RCP<const map_type> fakeColMap;
1983  Teuchos::RCP<sparse_graph_type> graph =
1984  readSparseGraphHelper (in, pComm,
1985  fakeRowMap, fakeColMap,
1986  constructorParams, tolerant, debug);
1987  graph->fillComplete (fillCompleteParams);
1988  return graph;
1989  }
1990 
1991 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
1992  static Teuchos::RCP<sparse_graph_type> TPETRA_DEPRECATED
1994  readSparseGraph (std::istream& in,
1995  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
1996  const Teuchos::RCP<node_type>& /* pNode */,
1997  const Teuchos::RCP<Teuchos::ParameterList>& constructorParams,
1998  const Teuchos::RCP<Teuchos::ParameterList>& fillCompleteParams,
1999  const bool tolerant=false,
2000  const bool debug=false)
2001  {
2002  // Call the overload below.
2003  return readSparseGraph (in, pComm, constructorParams,
2004  fillCompleteParams, tolerant, debug);
2005  }
2006 #endif // TPETRA_ENABLE_DEPRECATED_CODE
2007 
2048  static Teuchos::RCP<sparse_graph_type>
2049  readSparseGraph (std::istream& in,
2050  const Teuchos::RCP<const map_type>& rowMap,
2051  Teuchos::RCP<const map_type>& colMap,
2052  const Teuchos::RCP<const map_type>& domainMap,
2053  const Teuchos::RCP<const map_type>& rangeMap,
2054  const bool callFillComplete=true,
2055  const bool tolerant=false,
2056  const bool debug=false)
2057  {
2058  Teuchos::RCP<sparse_graph_type> graph =
2059  readSparseGraphHelper (in, rowMap->getComm (),
2060  rowMap, colMap, Teuchos::null, tolerant,
2061  debug);
2062  if (callFillComplete) {
2063  graph->fillComplete (domainMap, rangeMap);
2064  }
2065  return graph;
2066  }
2067 
2091  static Teuchos::RCP<sparse_matrix_type>
2092  readSparseFile (const std::string& filename,
2093  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
2094  const bool callFillComplete=true,
2095  const bool tolerant=false,
2096  const bool debug=false)
2097  {
2098  const int myRank = pComm->getRank ();
2099  std::ifstream in;
2100 
2101  // Only open the file on Rank 0.
2102  if (myRank == 0) {
2103  in.open (filename.c_str ());
2104  }
2105  // FIXME (mfh 16 Jun 2015) Do a broadcast to make sure that
2106  // opening the file succeeded, before continuing. That will
2107  // avoid hangs if the read doesn't work. On the other hand,
2108  // readSparse could do that too, by checking the status of the
2109  // std::ostream.
2110 
2111  return readSparse (in, pComm, callFillComplete, tolerant, debug);
2112  // We can rely on the destructor of the input stream to close
2113  // the file on scope exit, even if readSparse() throws an
2114  // exception.
2115  }
2116 
2117 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
2118  static Teuchos::RCP<sparse_matrix_type> TPETRA_DEPRECATED
2120  readSparseFile (const std::string& filename,
2121  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
2122  const Teuchos::RCP<node_type>& /* pNode */,
2123  const bool callFillComplete=true,
2124  const bool tolerant=false,
2125  const bool debug=false)
2126  {
2127  return readSparseFile (filename, pComm, callFillComplete, tolerant, debug);
2128  }
2129 #endif // TPETRA_ENABLE_DEPRECATED_CODE
2130 
2159  static Teuchos::RCP<sparse_matrix_type>
2160  readSparseFile (const std::string& filename,
2161  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
2162  const Teuchos::RCP<Teuchos::ParameterList>& constructorParams,
2163  const Teuchos::RCP<Teuchos::ParameterList>& fillCompleteParams,
2164  const bool tolerant=false,
2165  const bool debug=false)
2166  {
2167  std::ifstream in;
2168  if (pComm->getRank () == 0) { // only open on Process 0
2169  in.open (filename.c_str ());
2170  }
2171  return readSparse (in, pComm, constructorParams,
2172  fillCompleteParams, tolerant, debug);
2173  }
2174 
2175 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
2176  static Teuchos::RCP<sparse_matrix_type> TPETRA_DEPRECATED
2178  readSparseFile (const std::string& filename,
2179  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
2180  const Teuchos::RCP<node_type>& /* pNode */,
2181  const Teuchos::RCP<Teuchos::ParameterList>& constructorParams,
2182  const Teuchos::RCP<Teuchos::ParameterList>& fillCompleteParams,
2183  const bool tolerant=false,
2184  const bool debug=false)
2185  {
2186  return readSparseFile (filename, pComm,
2187  constructorParams, fillCompleteParams,
2188  tolerant, debug);
2189  }
2190 #endif // TPETRA_ENABLE_DEPRECATED_CODE
2191 
2229  static Teuchos::RCP<sparse_matrix_type>
2230  readSparseFile (const std::string& filename,
2231  const Teuchos::RCP<const map_type>& rowMap,
2232  Teuchos::RCP<const map_type>& colMap,
2233  const Teuchos::RCP<const map_type>& domainMap,
2234  const Teuchos::RCP<const map_type>& rangeMap,
2235  const bool callFillComplete=true,
2236  const bool tolerant=false,
2237  const bool debug=false)
2238  {
2239  using Teuchos::broadcast;
2240  using Teuchos::Comm;
2241  using Teuchos::outArg;
2242  using Teuchos::RCP;
2243 
2244  TEUCHOS_TEST_FOR_EXCEPTION(
2245  rowMap.is_null (), std::invalid_argument,
2246  "Row Map must be nonnull.");
2247 
2248  RCP<const Comm<int> > comm = rowMap->getComm ();
2249  const int myRank = comm->getRank ();
2250 
2251  // Only open the file on Process 0. Test carefully to make
2252  // sure that the file opened successfully (and broadcast that
2253  // result to all processes to prevent a hang on exception
2254  // throw), since it's a common mistake to misspell a filename.
2255  std::ifstream in;
2256  int opened = 0;
2257  if (myRank == 0) {
2258  try {
2259  in.open (filename.c_str ());
2260  opened = 1;
2261  }
2262  catch (...) {
2263  opened = 0;
2264  }
2265  }
2266  broadcast<int, int> (*comm, 0, outArg (opened));
2267  TEUCHOS_TEST_FOR_EXCEPTION(
2268  opened == 0, std::runtime_error,
2269  "readSparseFile: Failed to open file \"" << filename << "\" on "
2270  "Process 0.");
2271  return readSparse (in, rowMap, colMap, domainMap, rangeMap,
2272  callFillComplete, tolerant, debug);
2273  }
2274 
2300  static Teuchos::RCP<sparse_matrix_type>
2301  readSparse (std::istream& in,
2302  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
2303  const bool callFillComplete=true,
2304  const bool tolerant=false,
2305  const bool debug=false)
2306  {
2307  using Teuchos::MatrixMarket::Banner;
2308  using Teuchos::arcp;
2309  using Teuchos::ArrayRCP;
2310  using Teuchos::broadcast;
2311  using Teuchos::null;
2312  using Teuchos::ptr;
2313  using Teuchos::RCP;
2314  using Teuchos::REDUCE_MAX;
2315  using Teuchos::reduceAll;
2316  using Teuchos::Tuple;
2317  using std::cerr;
2318  using std::endl;
2319  typedef Teuchos::ScalarTraits<scalar_type> STS;
2320 
2321  const bool extraDebug = false;
2322  const int myRank = pComm->getRank ();
2323  const int rootRank = 0;
2324 
2325  // Current line number in the input stream. Various calls
2326  // will modify this depending on the number of lines that are
2327  // read from the input stream. Only Rank 0 modifies this.
2328  size_t lineNumber = 1;
2329 
2330  if (debug && myRank == rootRank) {
2331  cerr << "Matrix Market reader: readSparse:" << endl
2332  << "-- Reading banner line" << endl;
2333  }
2334 
2335  // The "Banner" tells you whether the input stream represents
2336  // a sparse matrix, the symmetry type of the matrix, and the
2337  // type of the data it contains.
2338  //
2339  // pBanner will only be nonnull on MPI Rank 0. It will be
2340  // null on all other MPI processes.
2341  RCP<const Banner> pBanner;
2342  {
2343  // We read and validate the Banner on Proc 0, but broadcast
2344  // the validation result to all processes.
2345  // Teuchos::broadcast doesn't currently work with bool, so
2346  // we use int (true -> 1, false -> 0).
2347  int bannerIsCorrect = 1;
2348  std::ostringstream errMsg;
2349 
2350  if (myRank == rootRank) {
2351  // Read the Banner line from the input stream.
2352  try {
2353  pBanner = readBanner (in, lineNumber, tolerant, debug);
2354  }
2355  catch (std::exception& e) {
2356  errMsg << "Attempt to read the Matrix Market file's Banner line "
2357  "threw an exception: " << e.what();
2358  bannerIsCorrect = 0;
2359  }
2360 
2361  if (bannerIsCorrect) {
2362  // Validate the Banner for the case of a sparse matrix.
2363  // We validate on Proc 0, since it reads the Banner.
2364 
2365  // In intolerant mode, the matrix type must be "coordinate".
2366  if (! tolerant && pBanner->matrixType() != "coordinate") {
2367  bannerIsCorrect = 0;
2368  errMsg << "The Matrix Market input file must contain a "
2369  "\"coordinate\"-format sparse matrix in order to create a "
2370  "Tpetra::CrsMatrix object from it, but the file's matrix "
2371  "type is \"" << pBanner->matrixType() << "\" instead.";
2372  }
2373  // In tolerant mode, we allow the matrix type to be
2374  // anything other than "array" (which would mean that
2375  // the file contains a dense matrix).
2376  if (tolerant && pBanner->matrixType() == "array") {
2377  bannerIsCorrect = 0;
2378  errMsg << "Matrix Market file must contain a \"coordinate\"-"
2379  "format sparse matrix in order to create a Tpetra::CrsMatrix "
2380  "object from it, but the file's matrix type is \"array\" "
2381  "instead. That probably means the file contains dense matrix "
2382  "data.";
2383  }
2384  }
2385  } // Proc 0: Done reading the Banner, hopefully successfully.
2386 
2387  // Broadcast from Proc 0 whether the Banner was read correctly.
2388  broadcast (*pComm, rootRank, ptr (&bannerIsCorrect));
2389 
2390  // If the Banner is invalid, all processes throw an
2391  // exception. Only Proc 0 gets the exception message, but
2392  // that's OK, since the main point is to "stop the world"
2393  // (rather than throw an exception on one process and leave
2394  // the others hanging).
2395  TEUCHOS_TEST_FOR_EXCEPTION(bannerIsCorrect == 0,
2396  std::invalid_argument, errMsg.str ());
2397  } // Done reading the Banner line and broadcasting success.
2398  if (debug && myRank == rootRank) {
2399  cerr << "-- Reading dimensions line" << endl;
2400  }
2401 
2402  // Read the matrix dimensions from the Matrix Market metadata.
2403  // dims = (numRows, numCols, numEntries). Proc 0 does the
2404  // reading, but it broadcasts the results to all MPI
2405  // processes. Thus, readCoordDims() is a collective
2406  // operation. It does a collective check for correctness too.
2407  Tuple<global_ordinal_type, 3> dims =
2408  readCoordDims (in, lineNumber, pBanner, pComm, tolerant, debug);
2409 
2410  if (debug && myRank == rootRank) {
2411  cerr << "-- Making Adder for collecting matrix data" << endl;
2412  }
2413 
2414  // "Adder" object for collecting all the sparse matrix entries
2415  // from the input stream. This is only nonnull on Proc 0.
2416  RCP<adder_type> pAdder =
2417  makeAdder (pComm, pBanner, dims, tolerant, debug);
2418 
2419  if (debug && myRank == rootRank) {
2420  cerr << "-- Reading matrix data" << endl;
2421  }
2422  //
2423  // Read the matrix entries from the input stream on Proc 0.
2424  //
2425  {
2426  // We use readSuccess to broadcast the results of the read
2427  // (succeeded or not) to all MPI processes. Since
2428  // Teuchos::broadcast doesn't currently know how to send
2429  // bools, we convert to int (true -> 1, false -> 0).
2430  int readSuccess = 1;
2431  std::ostringstream errMsg; // Exception message (only valid on Proc 0)
2432  if (myRank == rootRank) {
2433  try {
2434  // Reader for "coordinate" format sparse matrix data.
2435  typedef Teuchos::MatrixMarket::CoordDataReader<adder_type,
2436  global_ordinal_type, scalar_type, STS::isComplex> reader_type;
2437  reader_type reader (pAdder);
2438 
2439  // Read the sparse matrix entries.
2440  std::pair<bool, std::vector<size_t> > results =
2441  reader.read (in, lineNumber, tolerant, debug);
2442  readSuccess = results.first ? 1 : 0;
2443  }
2444  catch (std::exception& e) {
2445  readSuccess = 0;
2446  errMsg << e.what();
2447  }
2448  }
2449  broadcast (*pComm, rootRank, ptr (&readSuccess));
2450 
2451  // It would be nice to add a "verbose" flag, so that in
2452  // tolerant mode, we could log any bad line number(s) on
2453  // Proc 0. For now, we just throw if the read fails to
2454  // succeed.
2455  //
2456  // Question: If we're in tolerant mode, and if the read did
2457  // not succeed, should we attempt to call fillComplete()?
2458  TEUCHOS_TEST_FOR_EXCEPTION(readSuccess == 0, std::runtime_error,
2459  "Failed to read the Matrix Market sparse matrix file: "
2460  << errMsg.str());
2461  } // Done reading the matrix entries (stored on Proc 0 for now)
2462 
2463  if (debug && myRank == rootRank) {
2464  cerr << "-- Successfully read the Matrix Market data" << endl;
2465  }
2466 
2467  // In tolerant mode, we need to rebroadcast the matrix
2468  // dimensions, since they may be different after reading the
2469  // actual matrix data. We only need to broadcast the number
2470  // of rows and columns. Only Rank 0 needs to know the actual
2471  // global number of entries, since (a) we need to merge
2472  // duplicates on Rank 0 first anyway, and (b) when we
2473  // distribute the entries, each rank other than Rank 0 will
2474  // only need to know how many entries it owns, not the total
2475  // number of entries.
2476  if (tolerant) {
2477  if (debug && myRank == rootRank) {
2478  cerr << "-- Tolerant mode: rebroadcasting matrix dimensions"
2479  << endl
2480  << "----- Dimensions before: "
2481  << dims[0] << " x " << dims[1]
2482  << endl;
2483  }
2484  // Packed coordinate matrix dimensions (numRows, numCols).
2485  Tuple<global_ordinal_type, 2> updatedDims;
2486  if (myRank == rootRank) {
2487  // If one or more bottom rows of the matrix contain no
2488  // entries, then the Adder will report that the number
2489  // of rows is less than that specified in the
2490  // metadata. We allow this case, and favor the
2491  // metadata so that the zero row(s) will be included.
2492  updatedDims[0] =
2493  std::max (dims[0], pAdder->getAdder()->numRows());
2494  updatedDims[1] = pAdder->getAdder()->numCols();
2495  }
2496  broadcast (*pComm, rootRank, updatedDims);
2497  dims[0] = updatedDims[0];
2498  dims[1] = updatedDims[1];
2499  if (debug && myRank == rootRank) {
2500  cerr << "----- Dimensions after: " << dims[0] << " x "
2501  << dims[1] << endl;
2502  }
2503  }
2504  else {
2505  // In strict mode, we require that the matrix's metadata and
2506  // its actual data agree, at least somewhat. In particular,
2507  // the number of rows must agree, since otherwise we cannot
2508  // distribute the matrix correctly.
2509 
2510  // Teuchos::broadcast() doesn't know how to broadcast bools,
2511  // so we use an int with the standard 1 == true, 0 == false
2512  // encoding.
2513  int dimsMatch = 1;
2514  if (myRank == rootRank) {
2515  // If one or more bottom rows of the matrix contain no
2516  // entries, then the Adder will report that the number of
2517  // rows is less than that specified in the metadata. We
2518  // allow this case, and favor the metadata, but do not
2519  // allow the Adder to think there are more rows in the
2520  // matrix than the metadata says.
2521  if (dims[0] < pAdder->getAdder ()->numRows ()) {
2522  dimsMatch = 0;
2523  }
2524  }
2525  broadcast (*pComm, 0, ptr (&dimsMatch));
2526  if (dimsMatch == 0) {
2527  // We're in an error state anyway, so we might as well
2528  // work a little harder to print an informative error
2529  // message.
2530  //
2531  // Broadcast the Adder's idea of the matrix dimensions
2532  // from Proc 0 to all processes.
2533  Tuple<global_ordinal_type, 2> addersDims;
2534  if (myRank == rootRank) {
2535  addersDims[0] = pAdder->getAdder()->numRows();
2536  addersDims[1] = pAdder->getAdder()->numCols();
2537  }
2538  broadcast (*pComm, 0, addersDims);
2539  TEUCHOS_TEST_FOR_EXCEPTION(
2540  dimsMatch == 0, std::runtime_error,
2541  "The matrix metadata says that the matrix is " << dims[0] << " x "
2542  << dims[1] << ", but the actual data says that the matrix is "
2543  << addersDims[0] << " x " << addersDims[1] << ". That means the "
2544  "data includes more rows than reported in the metadata. This "
2545  "is not allowed when parsing in strict mode. Parse the matrix in "
2546  "tolerant mode to ignore the metadata when it disagrees with the "
2547  "data.");
2548  }
2549  } // Matrix dimensions (# rows, # cols, # entries) agree.
2550 
2551  if (debug && myRank == rootRank) {
2552  cerr << "-- Converting matrix data into CSR format on Proc 0" << endl;
2553  }
2554 
2555  // Now that we've read in all the matrix entries from the
2556  // input stream into the adder on Proc 0, post-process them
2557  // into CSR format (still on Proc 0). This will facilitate
2558  // distributing them to all the processors.
2559  //
2560  // These arrays represent the global matrix data as a CSR
2561  // matrix (with numEntriesPerRow as redundant but convenient
2562  // metadata, since it's computable from rowPtr and vice
2563  // versa). They are valid only on Proc 0.
2564  ArrayRCP<size_t> numEntriesPerRow;
2565  ArrayRCP<size_t> rowPtr;
2566  ArrayRCP<global_ordinal_type> colInd;
2567  ArrayRCP<scalar_type> values;
2568 
2569  // Proc 0 first merges duplicate entries, and then converts
2570  // the coordinate-format matrix data to CSR.
2571  {
2572  int mergeAndConvertSucceeded = 1;
2573  std::ostringstream errMsg;
2574 
2575  if (myRank == rootRank) {
2576  try {
2577  typedef Teuchos::MatrixMarket::Raw::Element<scalar_type,
2578  global_ordinal_type> element_type;
2579 
2580  // Number of rows in the matrix. If we are in tolerant
2581  // mode, we've already synchronized dims with the actual
2582  // matrix data. If in strict mode, we should use dims
2583  // (as read from the file's metadata) rather than the
2584  // matrix data to determine the dimensions. (The matrix
2585  // data will claim fewer rows than the metadata, if one
2586  // or more rows have no entries stored in the file.)
2587  const size_type numRows = dims[0];
2588 
2589  // Additively merge duplicate matrix entries.
2590  pAdder->getAdder()->merge ();
2591 
2592  // Get a temporary const view of the merged matrix entries.
2593  const std::vector<element_type>& entries =
2594  pAdder->getAdder()->getEntries();
2595 
2596  // Number of matrix entries (after merging).
2597  const size_t numEntries = (size_t)entries.size();
2598 
2599  if (debug) {
2600  cerr << "----- Proc 0: Matrix has numRows=" << numRows
2601  << " rows and numEntries=" << numEntries
2602  << " entries." << endl;
2603  }
2604 
2605  // Make space for the CSR matrix data. Converting to
2606  // CSR is easier if we fill numEntriesPerRow with zeros
2607  // at first.
2608  numEntriesPerRow = arcp<size_t> (numRows);
2609  std::fill (numEntriesPerRow.begin(), numEntriesPerRow.end(), 0);
2610  rowPtr = arcp<size_t> (numRows+1);
2611  std::fill (rowPtr.begin(), rowPtr.end(), 0);
2612  colInd = arcp<global_ordinal_type> (numEntries);
2613  values = arcp<scalar_type> (numEntries);
2614 
2615  // Convert from array-of-structs coordinate format to CSR
2616  // (compressed sparse row) format.
2617  global_ordinal_type prvRow = 0;
2618  size_t curPos = 0;
2619  rowPtr[0] = 0;
2620  for (curPos = 0; curPos < numEntries; ++curPos) {
2621  const element_type& curEntry = entries[curPos];
2622  const global_ordinal_type curRow = curEntry.rowIndex();
2623  TEUCHOS_TEST_FOR_EXCEPTION(
2624  curRow < prvRow, std::logic_error,
2625  "Row indices are out of order, even though they are supposed "
2626  "to be sorted. curRow = " << curRow << ", prvRow = "
2627  << prvRow << ", at curPos = " << curPos << ". Please report "
2628  "this bug to the Tpetra developers.");
2629  if (curRow > prvRow) {
2630  for (global_ordinal_type r = prvRow+1; r <= curRow; ++r) {
2631  rowPtr[r] = curPos;
2632  }
2633  prvRow = curRow;
2634  }
2635  numEntriesPerRow[curRow]++;
2636  colInd[curPos] = curEntry.colIndex();
2637  values[curPos] = curEntry.value();
2638  }
2639  // rowPtr has one more entry than numEntriesPerRow. The
2640  // last entry of rowPtr is the number of entries in
2641  // colInd and values.
2642  rowPtr[numRows] = numEntries;
2643  } // Finished conversion to CSR format
2644  catch (std::exception& e) {
2645  mergeAndConvertSucceeded = 0;
2646  errMsg << "Failed to merge sparse matrix entries and convert to "
2647  "CSR format: " << e.what();
2648  }
2649 
2650  if (debug && mergeAndConvertSucceeded) {
2651  // Number of rows in the matrix.
2652  const size_type numRows = dims[0];
2653  const size_type maxToDisplay = 100;
2654 
2655  cerr << "----- Proc 0: numEntriesPerRow[0.."
2656  << (numEntriesPerRow.size()-1) << "] ";
2657  if (numRows > maxToDisplay) {
2658  cerr << "(only showing first and last few entries) ";
2659  }
2660  cerr << "= [";
2661  if (numRows > 0) {
2662  if (numRows > maxToDisplay) {
2663  for (size_type k = 0; k < 2; ++k) {
2664  cerr << numEntriesPerRow[k] << " ";
2665  }
2666  cerr << "... ";
2667  for (size_type k = numRows-2; k < numRows-1; ++k) {
2668  cerr << numEntriesPerRow[k] << " ";
2669  }
2670  }
2671  else {
2672  for (size_type k = 0; k < numRows-1; ++k) {
2673  cerr << numEntriesPerRow[k] << " ";
2674  }
2675  }
2676  cerr << numEntriesPerRow[numRows-1];
2677  } // numRows > 0
2678  cerr << "]" << endl;
2679 
2680  cerr << "----- Proc 0: rowPtr ";
2681  if (numRows > maxToDisplay) {
2682  cerr << "(only showing first and last few entries) ";
2683  }
2684  cerr << "= [";
2685  if (numRows > maxToDisplay) {
2686  for (size_type k = 0; k < 2; ++k) {
2687  cerr << rowPtr[k] << " ";
2688  }
2689  cerr << "... ";
2690  for (size_type k = numRows-2; k < numRows; ++k) {
2691  cerr << rowPtr[k] << " ";
2692  }
2693  }
2694  else {
2695  for (size_type k = 0; k < numRows; ++k) {
2696  cerr << rowPtr[k] << " ";
2697  }
2698  }
2699  cerr << rowPtr[numRows] << "]" << endl;
2700  }
2701  } // if myRank == rootRank
2702  } // Done converting sparse matrix data to CSR format
2703 
2704  // Now we're done with the Adder, so we can release the
2705  // reference ("free" it) to save space. This only actually
2706  // does anything on Rank 0, since pAdder is null on all the
2707  // other MPI processes.
2708  pAdder = null;
2709 
2710  if (debug && myRank == rootRank) {
2711  cerr << "-- Making range, domain, and row maps" << endl;
2712  }
2713 
2714  // Make the maps that describe the matrix's range and domain,
2715  // and the distribution of its rows. Creating a Map is a
2716  // collective operation, so we don't have to do a broadcast of
2717  // a success Boolean.
2718  RCP<const map_type> pRangeMap = makeRangeMap (pComm, dims[0]);
2719  RCP<const map_type> pDomainMap =
2720  makeDomainMap (pRangeMap, dims[0], dims[1]);
2721  RCP<const map_type> pRowMap = makeRowMap (null, pComm, dims[0]);
2722 
2723  if (debug && myRank == rootRank) {
2724  cerr << "-- Distributing the matrix data" << endl;
2725  }
2726 
2727  // Distribute the matrix data. Each processor has to add the
2728  // rows that it owns. If you try to make Proc 0 call
2729  // insertGlobalValues() for _all_ the rows, not just those it
2730  // owns, then fillComplete() will compute the number of
2731  // columns incorrectly. That's why Proc 0 has to distribute
2732  // the matrix data and why we make all the processors (not
2733  // just Proc 0) call insertGlobalValues() on their own data.
2734  //
2735  // These arrays represent each processor's part of the matrix
2736  // data, in "CSR" format (sort of, since the row indices might
2737  // not be contiguous).
2738  ArrayRCP<size_t> myNumEntriesPerRow;
2739  ArrayRCP<size_t> myRowPtr;
2740  ArrayRCP<global_ordinal_type> myColInd;
2741  ArrayRCP<scalar_type> myValues;
2742  // Distribute the matrix data. This is a collective operation.
2743  distribute (myNumEntriesPerRow, myRowPtr, myColInd, myValues, pRowMap,
2744  numEntriesPerRow, rowPtr, colInd, values, debug);
2745 
2746  if (debug && myRank == rootRank) {
2747  cerr << "-- Inserting matrix entries on each processor";
2748  if (callFillComplete) {
2749  cerr << " and calling fillComplete()";
2750  }
2751  cerr << endl;
2752  }
2753  // Each processor inserts its part of the matrix data, and
2754  // then they all call fillComplete(). This method invalidates
2755  // the my* distributed matrix data before calling
2756  // fillComplete(), in order to save space. In general, we
2757  // never store more than two copies of the matrix's entries in
2758  // memory at once, which is no worse than what Tpetra
2759  // promises.
2760  RCP<sparse_matrix_type> pMatrix =
2761  makeMatrix (myNumEntriesPerRow, myRowPtr, myColInd, myValues,
2762  pRowMap, pRangeMap, pDomainMap, callFillComplete);
2763  // Only use a reduce-all in debug mode to check if pMatrix is
2764  // null. Otherwise, just throw an exception. We never expect
2765  // a null pointer here, so we can save a communication.
2766  if (debug) {
2767  int localIsNull = pMatrix.is_null () ? 1 : 0;
2768  int globalIsNull = 0;
2769  reduceAll (*pComm, REDUCE_MAX, localIsNull, ptr (&globalIsNull));
2770  TEUCHOS_TEST_FOR_EXCEPTION(globalIsNull != 0, std::logic_error,
2771  "Reader::makeMatrix() returned a null pointer on at least one "
2772  "process. Please report this bug to the Tpetra developers.");
2773  }
2774  else {
2775  TEUCHOS_TEST_FOR_EXCEPTION(pMatrix.is_null(), std::logic_error,
2776  "Reader::makeMatrix() returned a null pointer. "
2777  "Please report this bug to the Tpetra developers.");
2778  }
2779 
2780  // We can't get the dimensions of the matrix until after
2781  // fillComplete() is called. Thus, we can't do the sanity
2782  // check (dimensions read from the Matrix Market data,
2783  // vs. dimensions reported by the CrsMatrix) unless the user
2784  // asked makeMatrix() to call fillComplete().
2785  //
2786  // Note that pMatrix->getGlobalNum{Rows,Cols}() does _not_ do
2787  // what one might think it does, so you have to ask the range
2788  // resp. domain map for the number of rows resp. columns.
2789  if (callFillComplete) {
2790  const int numProcs = pComm->getSize ();
2791 
2792  if (extraDebug && debug) {
2793  const global_size_t globalNumRows =
2794  pRangeMap->getGlobalNumElements ();
2795  const global_size_t globalNumCols =
2796  pDomainMap->getGlobalNumElements ();
2797  if (myRank == rootRank) {
2798  cerr << "-- Matrix is "
2799  << globalNumRows << " x " << globalNumCols
2800  << " with " << pMatrix->getGlobalNumEntries()
2801  << " entries, and index base "
2802  << pMatrix->getIndexBase() << "." << endl;
2803  }
2804  pComm->barrier ();
2805  for (int p = 0; p < numProcs; ++p) {
2806  if (myRank == p) {
2807  cerr << "-- Proc " << p << " owns "
2808  << pMatrix->getNodeNumCols() << " columns, and "
2809  << pMatrix->getNodeNumEntries() << " entries." << endl;
2810  }
2811  pComm->barrier ();
2812  }
2813  } // if (extraDebug && debug)
2814  } // if (callFillComplete)
2815 
2816  if (debug && myRank == rootRank) {
2817  cerr << "-- Done creating the CrsMatrix from the Matrix Market data"
2818  << endl;
2819  }
2820  return pMatrix;
2821  }
2822 
2823 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
2824  static Teuchos::RCP<sparse_matrix_type> TPETRA_DEPRECATED
2826  readSparse (std::istream& in,
2827  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
2828  const Teuchos::RCP<node_type>& /* pNode */,
2829  const bool callFillComplete=true,
2830  const bool tolerant=false,
2831  const bool debug=false)
2832  {
2833  return readSparse (in, pComm, callFillComplete, tolerant, debug);
2834  }
2835 #endif // TPETRA_ENABLE_DEPRECATED_CODE
2836 
2865  static Teuchos::RCP<sparse_matrix_type>
2866  readSparse (std::istream& in,
2867  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
2868  const Teuchos::RCP<Teuchos::ParameterList>& constructorParams,
2869  const Teuchos::RCP<Teuchos::ParameterList>& fillCompleteParams,
2870  const bool tolerant=false,
2871  const bool debug=false)
2872  {
2873  using Teuchos::MatrixMarket::Banner;
2874  using Teuchos::arcp;
2875  using Teuchos::ArrayRCP;
2876  using Teuchos::broadcast;
2877  using Teuchos::null;
2878  using Teuchos::ptr;
2879  using Teuchos::RCP;
2880  using Teuchos::reduceAll;
2881  using Teuchos::Tuple;
2882  using std::cerr;
2883  using std::endl;
2884  typedef Teuchos::ScalarTraits<scalar_type> STS;
2885 
2886  const bool extraDebug = false;
2887  const int myRank = pComm->getRank ();
2888  const int rootRank = 0;
2889 
2890  // Current line number in the input stream. Various calls
2891  // will modify this depending on the number of lines that are
2892  // read from the input stream. Only Rank 0 modifies this.
2893  size_t lineNumber = 1;
2894 
2895  if (debug && myRank == rootRank) {
2896  cerr << "Matrix Market reader: readSparse:" << endl
2897  << "-- Reading banner line" << endl;
2898  }
2899 
2900  // The "Banner" tells you whether the input stream represents
2901  // a sparse matrix, the symmetry type of the matrix, and the
2902  // type of the data it contains.
2903  //
2904  // pBanner will only be nonnull on MPI Rank 0. It will be
2905  // null on all other MPI processes.
2906  RCP<const Banner> pBanner;
2907  {
2908  // We read and validate the Banner on Proc 0, but broadcast
2909  // the validation result to all processes.
2910  // Teuchos::broadcast doesn't currently work with bool, so
2911  // we use int (true -> 1, false -> 0).
2912  int bannerIsCorrect = 1;
2913  std::ostringstream errMsg;
2914 
2915  if (myRank == rootRank) {
2916  // Read the Banner line from the input stream.
2917  try {
2918  pBanner = readBanner (in, lineNumber, tolerant, debug);
2919  }
2920  catch (std::exception& e) {
2921  errMsg << "Attempt to read the Matrix Market file's Banner line "
2922  "threw an exception: " << e.what();
2923  bannerIsCorrect = 0;
2924  }
2925 
2926  if (bannerIsCorrect) {
2927  // Validate the Banner for the case of a sparse matrix.
2928  // We validate on Proc 0, since it reads the Banner.
2929 
2930  // In intolerant mode, the matrix type must be "coordinate".
2931  if (! tolerant && pBanner->matrixType() != "coordinate") {
2932  bannerIsCorrect = 0;
2933  errMsg << "The Matrix Market input file must contain a "
2934  "\"coordinate\"-format sparse matrix in order to create a "
2935  "Tpetra::CrsMatrix object from it, but the file's matrix "
2936  "type is \"" << pBanner->matrixType() << "\" instead.";
2937  }
2938  // In tolerant mode, we allow the matrix type to be
2939  // anything other than "array" (which would mean that
2940  // the file contains a dense matrix).
2941  if (tolerant && pBanner->matrixType() == "array") {
2942  bannerIsCorrect = 0;
2943  errMsg << "Matrix Market file must contain a \"coordinate\"-"
2944  "format sparse matrix in order to create a Tpetra::CrsMatrix "
2945  "object from it, but the file's matrix type is \"array\" "
2946  "instead. That probably means the file contains dense matrix "
2947  "data.";
2948  }
2949  }
2950  } // Proc 0: Done reading the Banner, hopefully successfully.
2951 
2952  // Broadcast from Proc 0 whether the Banner was read correctly.
2953  broadcast (*pComm, rootRank, ptr (&bannerIsCorrect));
2954 
2955  // If the Banner is invalid, all processes throw an
2956  // exception. Only Proc 0 gets the exception message, but
2957  // that's OK, since the main point is to "stop the world"
2958  // (rather than throw an exception on one process and leave
2959  // the others hanging).
2960  TEUCHOS_TEST_FOR_EXCEPTION(bannerIsCorrect == 0,
2961  std::invalid_argument, errMsg.str ());
2962  } // Done reading the Banner line and broadcasting success.
2963  if (debug && myRank == rootRank) {
2964  cerr << "-- Reading dimensions line" << endl;
2965  }
2966 
2967  // Read the matrix dimensions from the Matrix Market metadata.
2968  // dims = (numRows, numCols, numEntries). Proc 0 does the
2969  // reading, but it broadcasts the results to all MPI
2970  // processes. Thus, readCoordDims() is a collective
2971  // operation. It does a collective check for correctness too.
2972  Tuple<global_ordinal_type, 3> dims =
2973  readCoordDims (in, lineNumber, pBanner, pComm, tolerant, debug);
2974 
2975  if (debug && myRank == rootRank) {
2976  cerr << "-- Making Adder for collecting matrix data" << endl;
2977  }
2978 
2979  // "Adder" object for collecting all the sparse matrix entries
2980  // from the input stream. This is only nonnull on Proc 0.
2981  RCP<adder_type> pAdder =
2982  makeAdder (pComm, pBanner, dims, tolerant, debug);
2983 
2984  if (debug && myRank == rootRank) {
2985  cerr << "-- Reading matrix data" << endl;
2986  }
2987  //
2988  // Read the matrix entries from the input stream on Proc 0.
2989  //
2990  {
2991  // We use readSuccess to broadcast the results of the read
2992  // (succeeded or not) to all MPI processes. Since
2993  // Teuchos::broadcast doesn't currently know how to send
2994  // bools, we convert to int (true -> 1, false -> 0).
2995  int readSuccess = 1;
2996  std::ostringstream errMsg; // Exception message (only valid on Proc 0)
2997  if (myRank == rootRank) {
2998  try {
2999  // Reader for "coordinate" format sparse matrix data.
3000  typedef Teuchos::MatrixMarket::CoordDataReader<adder_type,
3001  global_ordinal_type, scalar_type, STS::isComplex> reader_type;
3002  reader_type reader (pAdder);
3003 
3004  // Read the sparse matrix entries.
3005  std::pair<bool, std::vector<size_t> > results =
3006  reader.read (in, lineNumber, tolerant, debug);
3007  readSuccess = results.first ? 1 : 0;
3008  }
3009  catch (std::exception& e) {
3010  readSuccess = 0;
3011  errMsg << e.what();
3012  }
3013  }
3014  broadcast (*pComm, rootRank, ptr (&readSuccess));
3015 
3016  // It would be nice to add a "verbose" flag, so that in
3017  // tolerant mode, we could log any bad line number(s) on
3018  // Proc 0. For now, we just throw if the read fails to
3019  // succeed.
3020  //
3021  // Question: If we're in tolerant mode, and if the read did
3022  // not succeed, should we attempt to call fillComplete()?
3023  TEUCHOS_TEST_FOR_EXCEPTION(readSuccess == 0, std::runtime_error,
3024  "Failed to read the Matrix Market sparse matrix file: "
3025  << errMsg.str());
3026  } // Done reading the matrix entries (stored on Proc 0 for now)
3027 
3028  if (debug && myRank == rootRank) {
3029  cerr << "-- Successfully read the Matrix Market data" << endl;
3030  }
3031 
3032  // In tolerant mode, we need to rebroadcast the matrix
3033  // dimensions, since they may be different after reading the
3034  // actual matrix data. We only need to broadcast the number
3035  // of rows and columns. Only Rank 0 needs to know the actual
3036  // global number of entries, since (a) we need to merge
3037  // duplicates on Rank 0 first anyway, and (b) when we
3038  // distribute the entries, each rank other than Rank 0 will
3039  // only need to know how many entries it owns, not the total
3040  // number of entries.
3041  if (tolerant) {
3042  if (debug && myRank == rootRank) {
3043  cerr << "-- Tolerant mode: rebroadcasting matrix dimensions"
3044  << endl
3045  << "----- Dimensions before: "
3046  << dims[0] << " x " << dims[1]
3047  << endl;
3048  }
3049  // Packed coordinate matrix dimensions (numRows, numCols).
3050  Tuple<global_ordinal_type, 2> updatedDims;
3051  if (myRank == rootRank) {
3052  // If one or more bottom rows of the matrix contain no
3053  // entries, then the Adder will report that the number
3054  // of rows is less than that specified in the
3055  // metadata. We allow this case, and favor the
3056  // metadata so that the zero row(s) will be included.
3057  updatedDims[0] =
3058  std::max (dims[0], pAdder->getAdder()->numRows());
3059  updatedDims[1] = pAdder->getAdder()->numCols();
3060  }
3061  broadcast (*pComm, rootRank, updatedDims);
3062  dims[0] = updatedDims[0];
3063  dims[1] = updatedDims[1];
3064  if (debug && myRank == rootRank) {
3065  cerr << "----- Dimensions after: " << dims[0] << " x "
3066  << dims[1] << endl;
3067  }
3068  }
3069  else {
3070  // In strict mode, we require that the matrix's metadata and
3071  // its actual data agree, at least somewhat. In particular,
3072  // the number of rows must agree, since otherwise we cannot
3073  // distribute the matrix correctly.
3074 
3075  // Teuchos::broadcast() doesn't know how to broadcast bools,
3076  // so we use an int with the standard 1 == true, 0 == false
3077  // encoding.
3078  int dimsMatch = 1;
3079  if (myRank == rootRank) {
3080  // If one or more bottom rows of the matrix contain no
3081  // entries, then the Adder will report that the number of
3082  // rows is less than that specified in the metadata. We
3083  // allow this case, and favor the metadata, but do not
3084  // allow the Adder to think there are more rows in the
3085  // matrix than the metadata says.
3086  if (dims[0] < pAdder->getAdder ()->numRows ()) {
3087  dimsMatch = 0;
3088  }
3089  }
3090  broadcast (*pComm, 0, ptr (&dimsMatch));
3091  if (dimsMatch == 0) {
3092  // We're in an error state anyway, so we might as well
3093  // work a little harder to print an informative error
3094  // message.
3095  //
3096  // Broadcast the Adder's idea of the matrix dimensions
3097  // from Proc 0 to all processes.
3098  Tuple<global_ordinal_type, 2> addersDims;
3099  if (myRank == rootRank) {
3100  addersDims[0] = pAdder->getAdder()->numRows();
3101  addersDims[1] = pAdder->getAdder()->numCols();
3102  }
3103  broadcast (*pComm, 0, addersDims);
3104  TEUCHOS_TEST_FOR_EXCEPTION(
3105  dimsMatch == 0, std::runtime_error,
3106  "The matrix metadata says that the matrix is " << dims[0] << " x "
3107  << dims[1] << ", but the actual data says that the matrix is "
3108  << addersDims[0] << " x " << addersDims[1] << ". That means the "
3109  "data includes more rows than reported in the metadata. This "
3110  "is not allowed when parsing in strict mode. Parse the matrix in "
3111  "tolerant mode to ignore the metadata when it disagrees with the "
3112  "data.");
3113  }
3114  } // Matrix dimensions (# rows, # cols, # entries) agree.
3115 
3116  if (debug && myRank == rootRank) {
3117  cerr << "-- Converting matrix data into CSR format on Proc 0" << endl;
3118  }
3119 
3120  // Now that we've read in all the matrix entries from the
3121  // input stream into the adder on Proc 0, post-process them
3122  // into CSR format (still on Proc 0). This will facilitate
3123  // distributing them to all the processors.
3124  //
3125  // These arrays represent the global matrix data as a CSR
3126  // matrix (with numEntriesPerRow as redundant but convenient
3127  // metadata, since it's computable from rowPtr and vice
3128  // versa). They are valid only on Proc 0.
3129  ArrayRCP<size_t> numEntriesPerRow;
3130  ArrayRCP<size_t> rowPtr;
3131  ArrayRCP<global_ordinal_type> colInd;
3132  ArrayRCP<scalar_type> values;
3133 
3134  // Proc 0 first merges duplicate entries, and then converts
3135  // the coordinate-format matrix data to CSR.
3136  {
3137  int mergeAndConvertSucceeded = 1;
3138  std::ostringstream errMsg;
3139 
3140  if (myRank == rootRank) {
3141  try {
3142  typedef Teuchos::MatrixMarket::Raw::Element<scalar_type,
3143  global_ordinal_type> element_type;
3144 
3145  // Number of rows in the matrix. If we are in tolerant
3146  // mode, we've already synchronized dims with the actual
3147  // matrix data. If in strict mode, we should use dims
3148  // (as read from the file's metadata) rather than the
3149  // matrix data to determine the dimensions. (The matrix
3150  // data will claim fewer rows than the metadata, if one
3151  // or more rows have no entries stored in the file.)
3152  const size_type numRows = dims[0];
3153 
3154  // Additively merge duplicate matrix entries.
3155  pAdder->getAdder()->merge ();
3156 
3157  // Get a temporary const view of the merged matrix entries.
3158  const std::vector<element_type>& entries =
3159  pAdder->getAdder()->getEntries();
3160 
3161  // Number of matrix entries (after merging).
3162  const size_t numEntries = (size_t)entries.size();
3163 
3164  if (debug) {
3165  cerr << "----- Proc 0: Matrix has numRows=" << numRows
3166  << " rows and numEntries=" << numEntries
3167  << " entries." << endl;
3168  }
3169 
3170  // Make space for the CSR matrix data. Converting to
3171  // CSR is easier if we fill numEntriesPerRow with zeros
3172  // at first.
3173  numEntriesPerRow = arcp<size_t> (numRows);
3174  std::fill (numEntriesPerRow.begin(), numEntriesPerRow.end(), 0);
3175  rowPtr = arcp<size_t> (numRows+1);
3176  std::fill (rowPtr.begin(), rowPtr.end(), 0);
3177  colInd = arcp<global_ordinal_type> (numEntries);
3178  values = arcp<scalar_type> (numEntries);
3179 
3180  // Convert from array-of-structs coordinate format to CSR
3181  // (compressed sparse row) format.
3182  global_ordinal_type prvRow = 0;
3183  size_t curPos = 0;
3184  rowPtr[0] = 0;
3185  for (curPos = 0; curPos < numEntries; ++curPos) {
3186  const element_type& curEntry = entries[curPos];
3187  const global_ordinal_type curRow = curEntry.rowIndex();
3188  TEUCHOS_TEST_FOR_EXCEPTION(
3189  curRow < prvRow, std::logic_error,
3190  "Row indices are out of order, even though they are supposed "
3191  "to be sorted. curRow = " << curRow << ", prvRow = "
3192  << prvRow << ", at curPos = " << curPos << ". Please report "
3193  "this bug to the Tpetra developers.");
3194  if (curRow > prvRow) {
3195  for (global_ordinal_type r = prvRow+1; r <= curRow; ++r) {
3196  rowPtr[r] = curPos;
3197  }
3198  prvRow = curRow;
3199  }
3200  numEntriesPerRow[curRow]++;
3201  colInd[curPos] = curEntry.colIndex();
3202  values[curPos] = curEntry.value();
3203  }
3204  // rowPtr has one more entry than numEntriesPerRow. The
3205  // last entry of rowPtr is the number of entries in
3206  // colInd and values.
3207  rowPtr[numRows] = numEntries;
3208  } // Finished conversion to CSR format
3209  catch (std::exception& e) {
3210  mergeAndConvertSucceeded = 0;
3211  errMsg << "Failed to merge sparse matrix entries and convert to "
3212  "CSR format: " << e.what();
3213  }
3214 
3215  if (debug && mergeAndConvertSucceeded) {
3216  // Number of rows in the matrix.
3217  const size_type numRows = dims[0];
3218  const size_type maxToDisplay = 100;
3219 
3220  cerr << "----- Proc 0: numEntriesPerRow[0.."
3221  << (numEntriesPerRow.size()-1) << "] ";
3222  if (numRows > maxToDisplay) {
3223  cerr << "(only showing first and last few entries) ";
3224  }
3225  cerr << "= [";
3226  if (numRows > 0) {
3227  if (numRows > maxToDisplay) {
3228  for (size_type k = 0; k < 2; ++k) {
3229  cerr << numEntriesPerRow[k] << " ";
3230  }
3231  cerr << "... ";
3232  for (size_type k = numRows-2; k < numRows-1; ++k) {
3233  cerr << numEntriesPerRow[k] << " ";
3234  }
3235  }
3236  else {
3237  for (size_type k = 0; k < numRows-1; ++k) {
3238  cerr << numEntriesPerRow[k] << " ";
3239  }
3240  }
3241  cerr << numEntriesPerRow[numRows-1];
3242  } // numRows > 0
3243  cerr << "]" << endl;
3244 
3245  cerr << "----- Proc 0: rowPtr ";
3246  if (numRows > maxToDisplay) {
3247  cerr << "(only showing first and last few entries) ";
3248  }
3249  cerr << "= [";
3250  if (numRows > maxToDisplay) {
3251  for (size_type k = 0; k < 2; ++k) {
3252  cerr << rowPtr[k] << " ";
3253  }
3254  cerr << "... ";
3255  for (size_type k = numRows-2; k < numRows; ++k) {
3256  cerr << rowPtr[k] << " ";
3257  }
3258  }
3259  else {
3260  for (size_type k = 0; k < numRows; ++k) {
3261  cerr << rowPtr[k] << " ";
3262  }
3263  }
3264  cerr << rowPtr[numRows] << "]" << endl;
3265  }
3266  } // if myRank == rootRank
3267  } // Done converting sparse matrix data to CSR format
3268 
3269  // Now we're done with the Adder, so we can release the
3270  // reference ("free" it) to save space. This only actually
3271  // does anything on Rank 0, since pAdder is null on all the
3272  // other MPI processes.
3273  pAdder = null;
3274 
3275  if (debug && myRank == rootRank) {
3276  cerr << "-- Making range, domain, and row maps" << endl;
3277  }
3278 
3279  // Make the maps that describe the matrix's range and domain,
3280  // and the distribution of its rows. Creating a Map is a
3281  // collective operation, so we don't have to do a broadcast of
3282  // a success Boolean.
3283  RCP<const map_type> pRangeMap = makeRangeMap (pComm, dims[0]);
3284  RCP<const map_type> pDomainMap =
3285  makeDomainMap (pRangeMap, dims[0], dims[1]);
3286  RCP<const map_type> pRowMap = makeRowMap (null, pComm, dims[0]);
3287 
3288  if (debug && myRank == rootRank) {
3289  cerr << "-- Distributing the matrix data" << endl;
3290  }
3291 
3292  // Distribute the matrix data. Each processor has to add the
3293  // rows that it owns. If you try to make Proc 0 call
3294  // insertGlobalValues() for _all_ the rows, not just those it
3295  // owns, then fillComplete() will compute the number of
3296  // columns incorrectly. That's why Proc 0 has to distribute
3297  // the matrix data and why we make all the processors (not
3298  // just Proc 0) call insertGlobalValues() on their own data.
3299  //
3300  // These arrays represent each processor's part of the matrix
3301  // data, in "CSR" format (sort of, since the row indices might
3302  // not be contiguous).
3303  ArrayRCP<size_t> myNumEntriesPerRow;
3304  ArrayRCP<size_t> myRowPtr;
3305  ArrayRCP<global_ordinal_type> myColInd;
3306  ArrayRCP<scalar_type> myValues;
3307  // Distribute the matrix data. This is a collective operation.
3308  distribute (myNumEntriesPerRow, myRowPtr, myColInd, myValues, pRowMap,
3309  numEntriesPerRow, rowPtr, colInd, values, debug);
3310 
3311  if (debug && myRank == rootRank) {
3312  cerr << "-- Inserting matrix entries on each process "
3313  "and calling fillComplete()" << endl;
3314  }
3315  // Each processor inserts its part of the matrix data, and
3316  // then they all call fillComplete(). This method invalidates
3317  // the my* distributed matrix data before calling
3318  // fillComplete(), in order to save space. In general, we
3319  // never store more than two copies of the matrix's entries in
3320  // memory at once, which is no worse than what Tpetra
3321  // promises.
3322  Teuchos::RCP<sparse_matrix_type> pMatrix =
3323  makeMatrix (myNumEntriesPerRow, myRowPtr, myColInd, myValues,
3324  pRowMap, pRangeMap, pDomainMap, constructorParams,
3325  fillCompleteParams);
3326  // Only use a reduce-all in debug mode to check if pMatrix is
3327  // null. Otherwise, just throw an exception. We never expect
3328  // a null pointer here, so we can save a communication.
3329  if (debug) {
3330  int localIsNull = pMatrix.is_null () ? 1 : 0;
3331  int globalIsNull = 0;
3332  reduceAll (*pComm, Teuchos::REDUCE_MAX, localIsNull, ptr (&globalIsNull));
3333  TEUCHOS_TEST_FOR_EXCEPTION(globalIsNull != 0, std::logic_error,
3334  "Reader::makeMatrix() returned a null pointer on at least one "
3335  "process. Please report this bug to the Tpetra developers.");
3336  }
3337  else {
3338  TEUCHOS_TEST_FOR_EXCEPTION(pMatrix.is_null(), std::logic_error,
3339  "Reader::makeMatrix() returned a null pointer. "
3340  "Please report this bug to the Tpetra developers.");
3341  }
3342 
3343  // Sanity check for dimensions (read from the Matrix Market
3344  // data, vs. dimensions reported by the CrsMatrix).
3345  //
3346  // Note that pMatrix->getGlobalNum{Rows,Cols}() does _not_ do
3347  // what one might think it does, so you have to ask the range
3348  // resp. domain map for the number of rows resp. columns.
3349  if (extraDebug && debug) {
3350  const int numProcs = pComm->getSize ();
3351  const global_size_t globalNumRows =
3352  pRangeMap->getGlobalNumElements();
3353  const global_size_t globalNumCols =
3354  pDomainMap->getGlobalNumElements();
3355  if (myRank == rootRank) {
3356  cerr << "-- Matrix is "
3357  << globalNumRows << " x " << globalNumCols
3358  << " with " << pMatrix->getGlobalNumEntries()
3359  << " entries, and index base "
3360  << pMatrix->getIndexBase() << "." << endl;
3361  }
3362  pComm->barrier ();
3363  for (int p = 0; p < numProcs; ++p) {
3364  if (myRank == p) {
3365  cerr << "-- Proc " << p << " owns "
3366  << pMatrix->getNodeNumCols() << " columns, and "
3367  << pMatrix->getNodeNumEntries() << " entries." << endl;
3368  }
3369  pComm->barrier ();
3370  }
3371  } // if (extraDebug && debug)
3372 
3373  if (debug && myRank == rootRank) {
3374  cerr << "-- Done creating the CrsMatrix from the Matrix Market data"
3375  << endl;
3376  }
3377  return pMatrix;
3378  }
3379 
3380 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
3381  static Teuchos::RCP<sparse_matrix_type> TPETRA_DEPRECATED
3383  readSparse (std::istream& in,
3384  const Teuchos::RCP<const Teuchos::Comm<int> >& pComm,
3385  const Teuchos::RCP<node_type>& /* pNode */,
3386  const Teuchos::RCP<Teuchos::ParameterList>& constructorParams,
3387  const Teuchos::RCP<Teuchos::ParameterList>& fillCompleteParams,
3388  const bool tolerant=false,
3389  const bool debug=false)
3390  {
3391  return readSparse (in, pComm, constructorParams,
3392  fillCompleteParams, tolerant, debug);
3393  }
3394 #endif // TPETRA_ENABLE_DEPRECATED_CODE
3395 
3436  static Teuchos::RCP<sparse_matrix_type>
3437  readSparse (std::istream& in,
3438  const Teuchos::RCP<const map_type>& rowMap,
3439  Teuchos::RCP<const map_type>& colMap,
3440  const Teuchos::RCP<const map_type>& domainMap,
3441  const Teuchos::RCP<const map_type>& rangeMap,
3442  const bool callFillComplete=true,
3443  const bool tolerant=false,
3444  const bool debug=false)
3445  {
3446  using Teuchos::MatrixMarket::Banner;
3447  using Teuchos::arcp;
3448  using Teuchos::ArrayRCP;
3449  using Teuchos::ArrayView;
3450  using Teuchos::as;
3451  using Teuchos::broadcast;
3452  using Teuchos::Comm;
3453  using Teuchos::null;
3454  using Teuchos::ptr;
3455  using Teuchos::RCP;
3456  using Teuchos::reduceAll;
3457  using Teuchos::Tuple;
3458  using std::cerr;
3459  using std::endl;
3460  typedef Teuchos::ScalarTraits<scalar_type> STS;
3461 
3462  RCP<const Comm<int> > pComm = rowMap->getComm ();
3463  const int myRank = pComm->getRank ();
3464  const int rootRank = 0;
3465  const bool extraDebug = false;
3466 
3467  // Fast checks for invalid input. We can't check other
3468  // attributes of the Maps until we've read in the matrix
3469  // dimensions.
3470  TEUCHOS_TEST_FOR_EXCEPTION(
3471  rowMap.is_null (), std::invalid_argument,
3472  "Row Map must be nonnull.");
3473  TEUCHOS_TEST_FOR_EXCEPTION(
3474  rangeMap.is_null (), std::invalid_argument,
3475  "Range Map must be nonnull.");
3476  TEUCHOS_TEST_FOR_EXCEPTION(
3477  domainMap.is_null (), std::invalid_argument,
3478  "Domain Map must be nonnull.");
3479  TEUCHOS_TEST_FOR_EXCEPTION(
3480  rowMap->getComm().getRawPtr() != pComm.getRawPtr(),
3481  std::invalid_argument,
3482  "The specified row Map's communicator (rowMap->getComm())"
3483  "differs from the given separately supplied communicator pComm.");
3484  TEUCHOS_TEST_FOR_EXCEPTION(
3485  domainMap->getComm().getRawPtr() != pComm.getRawPtr(),
3486  std::invalid_argument,
3487  "The specified domain Map's communicator (domainMap->getComm())"
3488  "differs from the given separately supplied communicator pComm.");
3489  TEUCHOS_TEST_FOR_EXCEPTION(
3490  rangeMap->getComm().getRawPtr() != pComm.getRawPtr(),
3491  std::invalid_argument,
3492  "The specified range Map's communicator (rangeMap->getComm())"
3493  "differs from the given separately supplied communicator pComm.");
3494 
3495  // Current line number in the input stream. Various calls
3496  // will modify this depending on the number of lines that are
3497  // read from the input stream. Only Rank 0 modifies this.
3498  size_t lineNumber = 1;
3499 
3500  if (debug && myRank == rootRank) {
3501  cerr << "Matrix Market reader: readSparse:" << endl
3502  << "-- Reading banner line" << endl;
3503  }
3504 
3505  // The "Banner" tells you whether the input stream represents
3506  // a sparse matrix, the symmetry type of the matrix, and the
3507  // type of the data it contains.
3508  //
3509  // pBanner will only be nonnull on MPI Rank 0. It will be
3510  // null on all other MPI processes.
3511  RCP<const Banner> pBanner;
3512  {
3513  // We read and validate the Banner on Proc 0, but broadcast
3514  // the validation result to all processes.
3515  // Teuchos::broadcast doesn't currently work with bool, so
3516  // we use int (true -> 1, false -> 0).
3517  int bannerIsCorrect = 1;
3518  std::ostringstream errMsg;
3519 
3520  if (myRank == rootRank) {
3521  // Read the Banner line from the input stream.
3522  try {
3523  pBanner = readBanner (in, lineNumber, tolerant, debug);
3524  }
3525  catch (std::exception& e) {
3526  errMsg << "Attempt to read the Matrix Market file's Banner line "
3527  "threw an exception: " << e.what();
3528  bannerIsCorrect = 0;
3529  }
3530 
3531  if (bannerIsCorrect) {
3532  // Validate the Banner for the case of a sparse matrix.
3533  // We validate on Proc 0, since it reads the Banner.
3534 
3535  // In intolerant mode, the matrix type must be "coordinate".
3536  if (! tolerant && pBanner->matrixType() != "coordinate") {
3537  bannerIsCorrect = 0;
3538  errMsg << "The Matrix Market input file must contain a "
3539  "\"coordinate\"-format sparse matrix in order to create a "
3540  "Tpetra::CrsMatrix object from it, but the file's matrix "
3541  "type is \"" << pBanner->matrixType() << "\" instead.";
3542  }
3543  // In tolerant mode, we allow the matrix type to be
3544  // anything other than "array" (which would mean that
3545  // the file contains a dense matrix).
3546  if (tolerant && pBanner->matrixType() == "array") {
3547  bannerIsCorrect = 0;
3548  errMsg << "Matrix Market file must contain a \"coordinate\"-"
3549  "format sparse matrix in order to create a Tpetra::CrsMatrix "
3550  "object from it, but the file's matrix type is \"array\" "
3551  "instead. That probably means the file contains dense matrix "
3552  "data.";
3553  }
3554  }
3555  } // Proc 0: Done reading the Banner, hopefully successfully.
3556 
3557  // Broadcast from Proc 0 whether the Banner was read correctly.
3558  broadcast (*pComm, rootRank, ptr (&bannerIsCorrect));
3559 
3560  // If the Banner is invalid, all processes throw an
3561  // exception. Only Proc 0 gets the exception message, but
3562  // that's OK, since the main point is to "stop the world"
3563  // (rather than throw an exception on one process and leave
3564  // the others hanging).
3565  TEUCHOS_TEST_FOR_EXCEPTION(bannerIsCorrect == 0,
3566  std::invalid_argument, errMsg.str ());
3567  } // Done reading the Banner line and broadcasting success.
3568  if (debug && myRank == rootRank) {
3569  cerr << "-- Reading dimensions line" << endl;
3570  }
3571 
3572  // Read the matrix dimensions from the Matrix Market metadata.
3573  // dims = (numRows, numCols, numEntries). Proc 0 does the
3574  // reading, but it broadcasts the results to all MPI
3575  // processes. Thus, readCoordDims() is a collective
3576  // operation. It does a collective check for correctness too.
3577  Tuple<global_ordinal_type, 3> dims =
3578  readCoordDims (in, lineNumber, pBanner, pComm, tolerant, debug);
3579 
3580  if (debug && myRank == rootRank) {
3581  cerr << "-- Making Adder for collecting matrix data" << endl;
3582  }
3583 
3584  // "Adder" object for collecting all the sparse matrix entries
3585  // from the input stream. This is only nonnull on Proc 0.
3586  // The Adder internally converts the one-based indices (native
3587  // Matrix Market format) into zero-based indices.
3588  RCP<adder_type> pAdder =
3589  makeAdder (pComm, pBanner, dims, tolerant, debug);
3590 
3591  if (debug && myRank == rootRank) {
3592  cerr << "-- Reading matrix data" << endl;
3593  }
3594  //
3595  // Read the matrix entries from the input stream on Proc 0.
3596  //
3597  {
3598  // We use readSuccess to broadcast the results of the read
3599  // (succeeded or not) to all MPI processes. Since
3600  // Teuchos::broadcast doesn't currently know how to send
3601  // bools, we convert to int (true -> 1, false -> 0).
3602  int readSuccess = 1;
3603  std::ostringstream errMsg; // Exception message (only valid on Proc 0)
3604  if (myRank == rootRank) {
3605  try {
3606  // Reader for "coordinate" format sparse matrix data.
3607  typedef Teuchos::MatrixMarket::CoordDataReader<adder_type,
3608  global_ordinal_type, scalar_type, STS::isComplex> reader_type;
3609  reader_type reader (pAdder);
3610 
3611  // Read the sparse matrix entries.
3612  std::pair<bool, std::vector<size_t> > results =
3613  reader.read (in, lineNumber, tolerant, debug);
3614  readSuccess = results.first ? 1 : 0;
3615  }
3616  catch (std::exception& e) {
3617  readSuccess = 0;
3618  errMsg << e.what();
3619  }
3620  }
3621  broadcast (*pComm, rootRank, ptr (&readSuccess));
3622 
3623  // It would be nice to add a "verbose" flag, so that in
3624  // tolerant mode, we could log any bad line number(s) on
3625  // Proc 0. For now, we just throw if the read fails to
3626  // succeed.
3627  //
3628  // Question: If we're in tolerant mode, and if the read did
3629  // not succeed, should we attempt to call fillComplete()?
3630  TEUCHOS_TEST_FOR_EXCEPTION(readSuccess == 0, std::runtime_error,
3631  "Failed to read the Matrix Market sparse matrix file: "
3632  << errMsg.str());
3633  } // Done reading the matrix entries (stored on Proc 0 for now)
3634 
3635  if (debug && myRank == rootRank) {
3636  cerr << "-- Successfully read the Matrix Market data" << endl;
3637  }
3638 
3639  // In tolerant mode, we need to rebroadcast the matrix
3640  // dimensions, since they may be different after reading the
3641  // actual matrix data. We only need to broadcast the number
3642  // of rows and columns. Only Rank 0 needs to know the actual
3643  // global number of entries, since (a) we need to merge
3644  // duplicates on Rank 0 first anyway, and (b) when we
3645  // distribute the entries, each rank other than Rank 0 will
3646  // only need to know how many entries it owns, not the total
3647  // number of entries.
3648  if (tolerant) {
3649  if (debug && myRank == rootRank) {
3650  cerr << "-- Tolerant mode: rebroadcasting matrix dimensions"
3651  << endl
3652  << "----- Dimensions before: "
3653  << dims[0] << " x " << dims[1]
3654  << endl;
3655  }
3656  // Packed coordinate matrix dimensions (numRows, numCols).
3657  Tuple<global_ordinal_type, 2> updatedDims;
3658  if (myRank == rootRank) {
3659  // If one or more bottom rows of the matrix contain no
3660  // entries, then the Adder will report that the number
3661  // of rows is less than that specified in the
3662  // metadata. We allow this case, and favor the
3663  // metadata so that the zero row(s) will be included.
3664  updatedDims[0] =
3665  std::max (dims[0], pAdder->getAdder()->numRows());
3666  updatedDims[1] = pAdder->getAdder()->numCols();
3667  }
3668  broadcast (*pComm, rootRank, updatedDims);
3669  dims[0] = updatedDims[0];
3670  dims[1] = updatedDims[1];
3671  if (debug && myRank == rootRank) {
3672  cerr << "----- Dimensions after: " << dims[0] << " x "
3673  << dims[1] << endl;
3674  }
3675  }
3676  else {
3677  // In strict mode, we require that the matrix's metadata and
3678  // its actual data agree, at least somewhat. In particular,
3679  // the number of rows must agree, since otherwise we cannot
3680  // distribute the matrix correctly.
3681 
3682  // Teuchos::broadcast() doesn't know how to broadcast bools,
3683  // so we use an int with the standard 1 == true, 0 == false
3684  // encoding.
3685  int dimsMatch = 1;
3686  if (myRank == rootRank) {
3687  // If one or more bottom rows of the matrix contain no
3688  // entries, then the Adder will report that the number of
3689  // rows is less than that specified in the metadata. We
3690  // allow this case, and favor the metadata, but do not
3691  // allow the Adder to think there are more rows in the
3692  // matrix than the metadata says.
3693  if (dims[0] < pAdder->getAdder ()->numRows ()) {
3694  dimsMatch = 0;
3695  }
3696  }
3697  broadcast (*pComm, 0, ptr (&dimsMatch));
3698  if (dimsMatch == 0) {
3699  // We're in an error state anyway, so we might as well
3700  // work a little harder to print an informative error
3701  // message.
3702  //
3703  // Broadcast the Adder's idea of the matrix dimensions
3704  // from Proc 0 to all processes.
3705  Tuple<global_ordinal_type, 2> addersDims;
3706  if (myRank == rootRank) {
3707  addersDims[0] = pAdder->getAdder()->numRows();
3708  addersDims[1] = pAdder->getAdder()->numCols();
3709  }
3710  broadcast (*pComm, 0, addersDims);
3711  TEUCHOS_TEST_FOR_EXCEPTION(
3712  dimsMatch == 0, std::runtime_error,
3713  "The matrix metadata says that the matrix is " << dims[0] << " x "
3714  << dims[1] << ", but the actual data says that the matrix is "
3715  << addersDims[0] << " x " << addersDims[1] << ". That means the "
3716  "data includes more rows than reported in the metadata. This "
3717  "is not allowed when parsing in strict mode. Parse the matrix in "
3718  "tolerant mode to ignore the metadata when it disagrees with the "
3719  "data.");
3720  }
3721  } // Matrix dimensions (# rows, # cols, # entries) agree.
3722 
3723  if (debug && myRank == rootRank) {
3724  cerr << "-- Converting matrix data into CSR format on Proc 0" << endl;
3725  }
3726 
3727  // Now that we've read in all the matrix entries from the
3728  // input stream into the adder on Proc 0, post-process them
3729  // into CSR format (still on Proc 0). This will facilitate
3730  // distributing them to all the processors.
3731  //
3732  // These arrays represent the global matrix data as a CSR
3733  // matrix (with numEntriesPerRow as redundant but convenient
3734  // metadata, since it's computable from rowPtr and vice
3735  // versa). They are valid only on Proc 0.
3736  ArrayRCP<size_t> numEntriesPerRow;
3737  ArrayRCP<size_t> rowPtr;
3738  ArrayRCP<global_ordinal_type> colInd;
3739  ArrayRCP<scalar_type> values;
3740 
3741  // Proc 0 first merges duplicate entries, and then converts
3742  // the coordinate-format matrix data to CSR.
3743  {
3744  int mergeAndConvertSucceeded = 1;
3745  std::ostringstream errMsg;
3746 
3747  if (myRank == rootRank) {
3748  try {
3749  typedef Teuchos::MatrixMarket::Raw::Element<scalar_type,
3750  global_ordinal_type> element_type;
3751 
3752  // Number of rows in the matrix. If we are in tolerant
3753  // mode, we've already synchronized dims with the actual
3754  // matrix data. If in strict mode, we should use dims
3755  // (as read from the file's metadata) rather than the
3756  // matrix data to determine the dimensions. (The matrix
3757  // data will claim fewer rows than the metadata, if one
3758  // or more rows have no entries stored in the file.)
3759  const size_type numRows = dims[0];
3760 
3761  // Additively merge duplicate matrix entries.
3762  pAdder->getAdder()->merge ();
3763 
3764  // Get a temporary const view of the merged matrix entries.
3765  const std::vector<element_type>& entries =
3766  pAdder->getAdder()->getEntries();
3767 
3768  // Number of matrix entries (after merging).
3769  const size_t numEntries = (size_t)entries.size();
3770 
3771  if (debug) {
3772  cerr << "----- Proc 0: Matrix has numRows=" << numRows
3773  << " rows and numEntries=" << numEntries
3774  << " entries." << endl;
3775  }
3776 
3777  // Make space for the CSR matrix data. Converting to
3778  // CSR is easier if we fill numEntriesPerRow with zeros
3779  // at first.
3780  numEntriesPerRow = arcp<size_t> (numRows);
3781  std::fill (numEntriesPerRow.begin(), numEntriesPerRow.end(), 0);
3782  rowPtr = arcp<size_t> (numRows+1);
3783  std::fill (rowPtr.begin(), rowPtr.end(), 0);
3784  colInd = arcp<global_ordinal_type> (numEntries);
3785  values = arcp<scalar_type> (numEntries);
3786 
3787  // Convert from array-of-structs coordinate format to CSR
3788  // (compressed sparse row) format.
3789  global_ordinal_type prvRow = 0;
3790  size_t curPos = 0;
3791  rowPtr[0] = 0;
3792  for (curPos = 0; curPos < numEntries; ++curPos) {
3793  const element_type& curEntry = entries[curPos];
3794  const global_ordinal_type curRow = curEntry.rowIndex();
3795  TEUCHOS_TEST_FOR_EXCEPTION(
3796  curRow < prvRow, std::logic_error,
3797  "Row indices are out of order, even though they are supposed "
3798  "to be sorted. curRow = " << curRow << ", prvRow = "
3799  << prvRow << ", at curPos = " << curPos << ". Please report "
3800  "this bug to the Tpetra developers.");
3801  if (curRow > prvRow) {
3802  for (global_ordinal_type r = prvRow+1; r <= curRow; ++r) {
3803  rowPtr[r] = curPos;
3804  }
3805  prvRow = curRow;
3806  }
3807  numEntriesPerRow[curRow]++;
3808  colInd[curPos] = curEntry.colIndex();
3809  values[curPos] = curEntry.value();
3810  }
3811  // rowPtr has one more entry than numEntriesPerRow. The
3812  // last entry of rowPtr is the number of entries in
3813  // colInd and values.
3814  rowPtr[numRows] = numEntries;
3815  } // Finished conversion to CSR format
3816  catch (std::exception& e) {
3817  mergeAndConvertSucceeded = 0;
3818  errMsg << "Failed to merge sparse matrix entries and convert to "
3819  "CSR format: " << e.what();
3820  }
3821 
3822  if (debug && mergeAndConvertSucceeded) {
3823  // Number of rows in the matrix.
3824  const size_type numRows = dims[0];
3825  const size_type maxToDisplay = 100;
3826 
3827  cerr << "----- Proc 0: numEntriesPerRow[0.."
3828  << (numEntriesPerRow.size()-1) << "] ";
3829  if (numRows > maxToDisplay) {
3830  cerr << "(only showing first and last few entries) ";
3831  }
3832  cerr << "= [";
3833  if (numRows > 0) {
3834  if (numRows > maxToDisplay) {
3835  for (size_type k = 0; k < 2; ++k) {
3836  cerr << numEntriesPerRow[k] << " ";
3837  }
3838  cerr << "... ";
3839  for (size_type k = numRows-2; k < numRows-1; ++k) {
3840  cerr << numEntriesPerRow[k] << " ";
3841  }
3842  }
3843  else {
3844  for (size_type k = 0; k < numRows-1; ++k) {
3845  cerr << numEntriesPerRow[k] << " ";
3846  }
3847  }
3848  cerr << numEntriesPerRow[numRows-1];
3849  } // numRows > 0
3850  cerr << "]" << endl;
3851 
3852  cerr << "----- Proc 0: rowPtr ";
3853  if (numRows > maxToDisplay) {
3854  cerr << "(only showing first and last few entries) ";
3855  }
3856  cerr << "= [";
3857  if (numRows > maxToDisplay) {
3858  for (size_type k = 0; k < 2; ++k) {
3859  cerr << rowPtr[k] << " ";
3860  }
3861  cerr << "... ";
3862  for (size_type k = numRows-2; k < numRows; ++k) {
3863  cerr << rowPtr[k] << " ";
3864  }
3865  }
3866  else {
3867  for (size_type k = 0; k < numRows; ++k) {
3868  cerr << rowPtr[k] << " ";
3869  }
3870  }
3871  cerr << rowPtr[numRows] << "]" << endl;
3872 
3873  cerr << "----- Proc 0: colInd = [";
3874  for (size_t k = 0; k < rowPtr[numRows]; ++k) {
3875  cerr << colInd[k] << " ";
3876  }
3877  cerr << "]" << endl;
3878  }
3879  } // if myRank == rootRank
3880  } // Done converting sparse matrix data to CSR format
3881 
3882  // Now we're done with the Adder, so we can release the
3883  // reference ("free" it) to save space. This only actually
3884  // does anything on Rank 0, since pAdder is null on all the
3885  // other MPI processes.
3886  pAdder = null;
3887 
3888  // Verify details of the Maps. Don't count the global number
3889  // of entries in the row Map, since that number doesn't
3890  // correctly count overlap.
3891  if (debug && myRank == rootRank) {
3892  cerr << "-- Verifying Maps" << endl;
3893  }
3894  TEUCHOS_TEST_FOR_EXCEPTION(
3895  as<global_size_t> (dims[0]) != rangeMap->getGlobalNumElements(),
3896  std::invalid_argument,
3897  "The range Map has " << rangeMap->getGlobalNumElements ()
3898  << " entries, but the matrix has a global number of rows " << dims[0]
3899  << ".");
3900  TEUCHOS_TEST_FOR_EXCEPTION(
3901  as<global_size_t> (dims[1]) != domainMap->getGlobalNumElements (),
3902  std::invalid_argument,
3903  "The domain Map has " << domainMap->getGlobalNumElements ()
3904  << " entries, but the matrix has a global number of columns "
3905  << dims[1] << ".");
3906 
3907  // Create a row Map which is entirely owned on Proc 0.
3908  RCP<Teuchos::FancyOStream> err = debug ?
3909  Teuchos::getFancyOStream (Teuchos::rcpFromRef (cerr)) : null;
3910 
3911  RCP<const map_type> gatherRowMap = Details::computeGatherMap (rowMap, err, debug);
3912  ArrayView<const global_ordinal_type> myRows =
3913  gatherRowMap->getNodeElementList ();
3914  const size_type myNumRows = myRows.size ();
3915  const global_ordinal_type indexBase = gatherRowMap->getIndexBase ();
3916 
3917  ArrayRCP<size_t> gatherNumEntriesPerRow = arcp<size_t>(myNumRows);
3918  for (size_type i_ = 0; i_ < myNumRows; i_++) {
3919  gatherNumEntriesPerRow[i_] = numEntriesPerRow[myRows[i_]-indexBase];
3920  }
3921 
3922  // Create a matrix using this Map, and fill in on Proc 0. We
3923  // know how many entries there are in each row, so we can use
3924  // static profile.
3925  RCP<sparse_matrix_type> A_proc0 =
3926  rcp (new sparse_matrix_type (gatherRowMap, gatherNumEntriesPerRow (),
3927  Tpetra::StaticProfile));
3928  if (myRank == rootRank) {
3929  if (debug) {
3930  cerr << "-- Proc 0: Filling gather matrix" << endl;
3931  }
3932  if (debug) {
3933  cerr << "---- Rows: " << Teuchos::toString (myRows) << endl;
3934  }
3935 
3936  // Add Proc 0's matrix entries to the CrsMatrix.
3937  for (size_type i_ = 0; i_ < myNumRows; ++i_) {
3938  size_type i = myRows[i_] - indexBase;
3939 
3940  const size_type curPos = as<size_type> (rowPtr[i]);
3941  const local_ordinal_type curNumEntries = numEntriesPerRow[i];
3942  ArrayView<global_ordinal_type> curColInd =
3943  colInd.view (curPos, curNumEntries);
3944  ArrayView<scalar_type> curValues =
3945  values.view (curPos, curNumEntries);
3946 
3947  // Modify the column indices in place to have the right index base.
3948  for (size_type k = 0; k < curNumEntries; ++k) {
3949  curColInd[k] += indexBase;
3950  }
3951  if (debug) {
3952  cerr << "------ Columns: " << Teuchos::toString (curColInd) << endl;
3953  cerr << "------ Values: " << Teuchos::toString (curValues) << endl;
3954  }
3955  // Avoid constructing empty views of ArrayRCP objects.
3956  if (curNumEntries > 0) {
3957  A_proc0->insertGlobalValues (myRows[i_], curColInd, curValues);
3958  }
3959  }
3960  // Now we can save space by deallocating numEntriesPerRow,
3961  // rowPtr, colInd, and values, since we've already put those
3962  // data in the matrix.
3963  numEntriesPerRow = null;
3964  rowPtr = null;
3965  colInd = null;
3966  values = null;
3967  } // if myRank == rootRank
3968 
3969  RCP<sparse_matrix_type> A;
3970  if (colMap.is_null ()) {
3971  A = rcp (new sparse_matrix_type (rowMap, 0));
3972  } else {
3973  A = rcp (new sparse_matrix_type (rowMap, colMap, 0));
3974  }
3976  export_type exp (gatherRowMap, rowMap);
3977  A->doExport (*A_proc0, exp, INSERT);
3978 
3979  if (callFillComplete) {
3980  A->fillComplete (domainMap, rangeMap);
3981  }
3982 
3983  // We can't get the dimensions of the matrix until after
3984  // fillComplete() is called. Thus, we can't do the sanity
3985  // check (dimensions read from the Matrix Market data,
3986  // vs. dimensions reported by the CrsMatrix) unless the user
3987  // asked us to call fillComplete().
3988  //
3989  // Note that pMatrix->getGlobalNum{Rows,Cols}() does _not_ do
3990  // what one might think it does, so you have to ask the range
3991  // resp. domain map for the number of rows resp. columns.
3992  if (callFillComplete) {
3993  const int numProcs = pComm->getSize ();
3994 
3995  if (extraDebug && debug) {
3996  const global_size_t globalNumRows = rangeMap->getGlobalNumElements ();
3997  const global_size_t globalNumCols = domainMap->getGlobalNumElements ();
3998  if (myRank == rootRank) {
3999  cerr << "-- Matrix is "
4000  << globalNumRows << " x " << globalNumCols
4001  << " with " << A->getGlobalNumEntries()
4002  << " entries, and index base "
4003  << A->getIndexBase() << "." << endl;
4004  }
4005  pComm->barrier ();
4006  for (int p = 0; p < numProcs; ++p) {
4007  if (myRank == p) {
4008  cerr << "-- Proc " << p << " owns "
4009  << A->getNodeNumCols() << " columns, and "
4010  << A->getNodeNumEntries() << " entries." << endl;
4011  }
4012  pComm->barrier ();
4013  }
4014  } // if (extraDebug && debug)
4015  } // if (callFillComplete)
4016 
4017  if (debug && myRank == rootRank) {
4018  cerr << "-- Done creating the CrsMatrix from the Matrix Market data"
4019  << endl;
4020  }
4021  return A;
4022  }
4023 
4052  static Teuchos::RCP<multivector_type>
4053  readDenseFile (const std::string& filename,
4054  const Teuchos::RCP<const comm_type>& comm,
4055  Teuchos::RCP<const map_type>& map,
4056  const bool tolerant=false,
4057  const bool debug=false)
4058  {
4059  std::ifstream in;
4060  if (comm->getRank () == 0) { // Only open the file on Proc 0.
4061  in.open (filename.c_str ()); // Destructor closes safely
4062  }
4063  return readDense (in, comm, map, tolerant, debug);
4064  }
4065 
4066 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
4067  static Teuchos::RCP<multivector_type> TPETRA_DEPRECATED
4069  readDenseFile (const std::string& filename,
4070  const Teuchos::RCP<const comm_type>& comm,
4071  const Teuchos::RCP<node_type>& /* pNode*/,
4072  Teuchos::RCP<const map_type>& map,
4073  const bool tolerant=false,
4074  const bool debug=false)
4075  {
4076  return readDenseFile(filename, comm, map, tolerant, debug);
4077  }
4078 #endif // TPETRA_ENABLE_DEPRECATED_CODE
4079 
4109  static Teuchos::RCP<vector_type>
4110  readVectorFile (const std::string& filename,
4111  const Teuchos::RCP<const comm_type>& comm,
4112  Teuchos::RCP<const map_type>& map,
4113  const bool tolerant=false,
4114  const bool debug=false)
4115  {
4116  std::ifstream in;
4117  if (comm->getRank () == 0) { // Only open the file on Proc 0.
4118  in.open (filename.c_str ()); // Destructor closes safely
4119  }
4120  return readVector (in, comm, map, tolerant, debug);
4121  }
4122 
4123 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
4124  static Teuchos::RCP<vector_type> TPETRA_DEPRECATED
4127  readVectorFile (const std::string& filename,
4128  const Teuchos::RCP<const comm_type>& comm,
4129  const Teuchos::RCP<node_type>& /* pNode */,
4130  Teuchos::RCP<const map_type>& map,
4131  const bool tolerant=false,
4132  const bool debug=false)
4133  {
4134  return readVectorFile(filename, comm, map, tolerant, debug);
4135  }
4136 #endif // TPETRA_ENABLE_DEPRECATED_CODE
4137 
4204  static Teuchos::RCP<multivector_type>
4205  readDense (std::istream& in,
4206  const Teuchos::RCP<const comm_type>& comm,
4207  Teuchos::RCP<const map_type>& map,
4208  const bool tolerant=false,
4209  const bool debug=false)
4210  {
4211  Teuchos::RCP<Teuchos::FancyOStream> err =
4212  Teuchos::getFancyOStream (Teuchos::rcpFromRef (std::cerr));
4213  return readDenseImpl<scalar_type> (in, comm, map, err, tolerant, debug);
4214  }
4215 
4216 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
4217  static Teuchos::RCP<multivector_type> TPETRA_DEPRECATED
4219  readDense (std::istream& in,
4220  const Teuchos::RCP<const comm_type>& comm,
4221  const Teuchos::RCP<node_type>& /* pNode */,
4222  Teuchos::RCP<const map_type>& map,
4223  const bool tolerant=false,
4224  const bool debug=false)
4225  {
4226  return readDense (in, comm, map, tolerant, debug);
4227  }
4228 #endif // TPETRA_ENABLE_DEPRECATED_CODE
4229 
4231  static Teuchos::RCP<vector_type>
4232  readVector (std::istream& in,
4233  const Teuchos::RCP<const comm_type>& comm,
4234  Teuchos::RCP<const map_type>& map,
4235  const bool tolerant=false,
4236  const bool debug=false)
4237  {
4238  Teuchos::RCP<Teuchos::FancyOStream> err =
4239  Teuchos::getFancyOStream (Teuchos::rcpFromRef (std::cerr));
4240  return readVectorImpl<scalar_type> (in, comm, map, err, tolerant, debug);
4241  }
4242 
4243 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
4244  static Teuchos::RCP<vector_type> TPETRA_DEPRECATED
4246  readVector (std::istream& in,
4247  const Teuchos::RCP<const comm_type>& comm,
4248  const Teuchos::RCP<node_type>& /* pNode */,
4249  Teuchos::RCP<const map_type>& map,
4250  const bool tolerant=false,
4251  const bool debug=false)
4252  {
4253  return readVector(in, comm, map, tolerant, debug);
4254  }
4255 #endif // TPETRA_ENABLE_DEPRECATED_CODE
4256 
4277  static Teuchos::RCP<const map_type>
4278  readMapFile (const std::string& filename,
4279  const Teuchos::RCP<const comm_type>& comm,
4280  const bool tolerant=false,
4281  const bool debug=false)
4282  {
4283  using Teuchos::inOutArg;
4284  using Teuchos::broadcast;
4285  std::ifstream in;
4286 
4287  int success = 1;
4288  if (comm->getRank () == 0) { // Only open the file on Proc 0.
4289  in.open (filename.c_str ()); // Destructor closes safely
4290  if (! in) {
4291  success = 0;
4292  }
4293  }
4294  broadcast<int, int> (*comm, 0, inOutArg (success));
4295  TEUCHOS_TEST_FOR_EXCEPTION(
4296  success == 0, std::runtime_error,
4297  "Tpetra::MatrixMarket::Reader::readMapFile: "
4298  "Failed to read file \"" << filename << "\" on Process 0.");
4299  return readMap (in, comm, tolerant, debug);
4300  }
4301 
4302 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
4303  static Teuchos::RCP<const map_type> TPETRA_DEPRECATED
4306  readMapFile (const std::string& filename,
4307  const Teuchos::RCP<const comm_type>& comm,
4308  const Teuchos::RCP<node_type>& /* pNode */,
4309  const bool tolerant=false,
4310  const bool debug=false)
4311  {
4312  return readMapFile (filename, comm, tolerant, debug);
4313  }
4314 #endif // TPETRA_ENABLE_DEPRECATED_CODE
4315 
4316  private:
4317  template<class MultiVectorScalarType>
4318  static Teuchos::RCP<Tpetra::MultiVector<MultiVectorScalarType,
4321  node_type> >
4322  readDenseImpl (std::istream& in,
4323  const Teuchos::RCP<const comm_type>& comm,
4324  Teuchos::RCP<const map_type>& map,
4325  const Teuchos::RCP<Teuchos::FancyOStream>& err,
4326  const bool tolerant=false,
4327  const bool debug=false)
4328  {
4329  using Teuchos::MatrixMarket::Banner;
4330  using Teuchos::MatrixMarket::checkCommentLine;
4331  using Teuchos::ArrayRCP;
4332  using Teuchos::as;
4333  using Teuchos::broadcast;
4334  using Teuchos::outArg;
4335  using Teuchos::RCP;
4336  using Teuchos::Tuple;
4337  using std::endl;
4338  typedef MultiVectorScalarType ST;
4339  typedef local_ordinal_type LO;
4340  typedef global_ordinal_type GO;
4341  typedef node_type NT;
4342  typedef Teuchos::ScalarTraits<ST> STS;
4343  typedef typename STS::magnitudeType MT;
4344  typedef Teuchos::ScalarTraits<MT> STM;
4346 
4347  // Rank 0 is the only (MPI) process allowed to read from the
4348  // input stream.
4349  const int myRank = comm->getRank ();
4350 
4351  if (! err.is_null ()) {
4352  err->pushTab ();
4353  }
4354  if (debug) {
4355  *err << myRank << ": readDenseImpl" << endl;
4356  }
4357  if (! err.is_null ()) {
4358  err->pushTab ();
4359  }
4360 
4361  // mfh 17 Feb 2013: It's not strictly necessary that the Comm
4362  // instances be identical and that the Node instances be
4363  // identical. The essential condition is more complicated to
4364  // test and isn't the same for all Node types. Thus, we just
4365  // leave it up to the user.
4366 
4367  // // If map is nonnull, check the precondition that its
4368  // // communicator resp. node equal comm resp. node. Checking
4369  // // now avoids doing a lot of file reading before we detect the
4370  // // violated precondition.
4371  // TEUCHOS_TEST_FOR_EXCEPTION(
4372  // ! map.is_null() && (map->getComm() != comm || map->getNode () != node,
4373  // std::invalid_argument, "If you supply a nonnull Map, the Map's "
4374  // "communicator and node must equal the supplied communicator resp. "
4375  // "node.");
4376 
4377  // Process 0 will read in the matrix dimensions from the file,
4378  // and broadcast them to all ranks in the given communicator.
4379  // There are only 2 dimensions in the matrix, but we use the
4380  // third element of the Tuple to encode the banner's reported
4381  // data type: "real" == 0, "complex" == 1, and "integer" == 0
4382  // (same as "real"). We don't allow pattern matrices (i.e.,
4383  // graphs) since they only make sense for sparse data.
4384  Tuple<GO, 3> dims;
4385  dims[0] = 0;
4386  dims[1] = 0;
4387 
4388  // Current line number in the input stream. Only valid on
4389  // Proc 0. Various calls will modify this depending on the
4390  // number of lines that are read from the input stream.
4391  size_t lineNumber = 1;
4392 
4393  // Capture errors and their messages on Proc 0.
4394  std::ostringstream exMsg;
4395  int localBannerReadSuccess = 1;
4396  int localDimsReadSuccess = 1;
4397 
4398  // Only Proc 0 gets to read matrix data from the input stream.
4399  if (myRank == 0) {
4400  if (debug) {
4401  *err << myRank << ": readDenseImpl: Reading banner line (dense)" << endl;
4402  }
4403 
4404  // The "Banner" tells you whether the input stream
4405  // represents a dense matrix, the symmetry type of the
4406  // matrix, and the type of the data it contains.
4407  RCP<const Banner> pBanner;
4408  try {
4409  pBanner = readBanner (in, lineNumber, tolerant, debug);
4410  } catch (std::exception& e) {
4411  exMsg << e.what ();
4412  localBannerReadSuccess = 0;
4413  }
4414  // Make sure the input stream is the right kind of data.
4415  if (localBannerReadSuccess) {
4416  if (pBanner->matrixType () != "array") {
4417  exMsg << "The Matrix Market file does not contain dense matrix "
4418  "data. Its banner (first) line says that its matrix type is \""
4419  << pBanner->matrixType () << "\", rather that the required "
4420  "\"array\".";
4421  localBannerReadSuccess = 0;
4422  } else if (pBanner->dataType() == "pattern") {
4423  exMsg << "The Matrix Market file's banner (first) "
4424  "line claims that the matrix's data type is \"pattern\". This does "
4425  "not make sense for a dense matrix, yet the file reports the matrix "
4426  "as dense. The only valid data types for a dense matrix are "
4427  "\"real\", \"complex\", and \"integer\".";
4428  localBannerReadSuccess = 0;
4429  } else {
4430  // Encode the data type reported by the Banner as the
4431  // third element of the dimensions Tuple.
4432  dims[2] = encodeDataType (pBanner->dataType ());
4433  }
4434  } // if we successfully read the banner line
4435 
4436  // At this point, we've successfully read the banner line.
4437  // Now read the dimensions line.
4438  if (localBannerReadSuccess) {
4439  if (debug) {
4440  *err << myRank << ": readDenseImpl: Reading dimensions line (dense)" << endl;
4441  }
4442  // Keep reading lines from the input stream until we find
4443  // a non-comment line, or until we run out of lines. The
4444  // latter is an error, since every "array" format Matrix
4445  // Market file must have a dimensions line after the
4446  // banner (even if the matrix has zero rows or columns, or
4447  // zero entries).
4448  std::string line;
4449  bool commentLine = true;
4450 
4451  while (commentLine) {
4452  // Test whether it is even valid to read from the input
4453  // stream wrapping the line.
4454  if (in.eof () || in.fail ()) {
4455  exMsg << "Unable to get array dimensions line (at all) from line "
4456  << lineNumber << " of input stream. The input stream "
4457  << "claims that it is "
4458  << (in.eof() ? "at end-of-file." : "in a failed state.");
4459  localDimsReadSuccess = 0;
4460  } else {
4461  // Try to get the next line from the input stream.
4462  if (getline (in, line)) {
4463  ++lineNumber; // We did actually read a line.
4464  }
4465  // Is the current line a comment line? Ignore start
4466  // and size; they are only useful for reading the
4467  // actual matrix entries. (We could use them here as
4468  // an optimization, but we've chosen not to.)
4469  size_t start = 0, size = 0;
4470  commentLine = checkCommentLine (line, start, size, lineNumber, tolerant);
4471  } // whether we failed to read the line at all
4472  } // while the line we just read is a comment line
4473 
4474  //
4475  // Get <numRows> <numCols> from the line we just read.
4476  //
4477  std::istringstream istr (line);
4478 
4479  // Test whether it is even valid to read from the input
4480  // stream wrapping the line.
4481  if (istr.eof () || istr.fail ()) {
4482  exMsg << "Unable to read any data from line " << lineNumber
4483  << " of input; the line should contain the matrix dimensions "
4484  << "\"<numRows> <numCols>\".";
4485  localDimsReadSuccess = 0;
4486  } else { // It's valid to read from the line.
4487  GO theNumRows = 0;
4488  istr >> theNumRows; // Read in the number of rows.
4489  if (istr.fail ()) {
4490  exMsg << "Failed to get number of rows from line "
4491  << lineNumber << " of input; the line should contains the "
4492  << "matrix dimensions \"<numRows> <numCols>\".";
4493  localDimsReadSuccess = 0;
4494  } else { // We successfully read the number of rows
4495  dims[0] = theNumRows; // Save the number of rows
4496  if (istr.eof ()) { // Do we still have data to read?
4497  exMsg << "No more data after number of rows on line "
4498  << lineNumber << " of input; the line should contain the "
4499  << "matrix dimensions \"<numRows> <numCols>\".";
4500  localDimsReadSuccess = 0;
4501  } else { // Still data left to read; read in number of columns.
4502  GO theNumCols = 0;
4503  istr >> theNumCols; // Read in the number of columns
4504  if (istr.fail ()) {
4505  exMsg << "Failed to get number of columns from line "
4506  << lineNumber << " of input; the line should contain "
4507  << "the matrix dimensions \"<numRows> <numCols>\".";
4508  localDimsReadSuccess = 0;
4509  } else { // We successfully read the number of columns
4510  dims[1] = theNumCols; // Save the number of columns
4511  } // if istr.fail ()
4512  } // if istr.eof ()
4513  } // if we read the number of rows
4514  } // if the input stream wrapping the dims line was (in)valid
4515  } // if we successfully read the banner line
4516  } // if (myRank == 0)
4517 
4518  // Broadcast the matrix dimensions, the encoded data type, and
4519  // whether or not Proc 0 succeeded in reading the banner and
4520  // dimensions.
4521  Tuple<GO, 5> bannerDimsReadResult;
4522  if (myRank == 0) {
4523  bannerDimsReadResult[0] = dims[0]; // numRows
4524  bannerDimsReadResult[1] = dims[1]; // numCols
4525  bannerDimsReadResult[2] = dims[2]; // encoded data type
4526  bannerDimsReadResult[3] = localBannerReadSuccess;
4527  bannerDimsReadResult[4] = localDimsReadSuccess;
4528  }
4529  // Broadcast matrix dimensions and the encoded data type from
4530  // Proc 0 to all the MPI processes.
4531  broadcast (*comm, 0, bannerDimsReadResult);
4532 
4533  TEUCHOS_TEST_FOR_EXCEPTION(
4534  bannerDimsReadResult[3] == 0, std::runtime_error,
4535  "Failed to read banner line: " << exMsg.str ());
4536  TEUCHOS_TEST_FOR_EXCEPTION(
4537  bannerDimsReadResult[4] == 0, std::runtime_error,
4538  "Failed to read matrix dimensions line: " << exMsg.str ());
4539  if (myRank != 0) {
4540  dims[0] = bannerDimsReadResult[0];
4541  dims[1] = bannerDimsReadResult[1];
4542  dims[2] = bannerDimsReadResult[2];
4543  }
4544 
4545  // Tpetra objects want the matrix dimensions in these types.
4546  const global_size_t numRows = static_cast<global_size_t> (dims[0]);
4547  const size_t numCols = static_cast<size_t> (dims[1]);
4548 
4549  // Make a "Proc 0 owns everything" Map that we will use to
4550  // read in the multivector entries in the correct order on
4551  // Proc 0. This must be a collective
4552  RCP<const map_type> proc0Map; // "Proc 0 owns everything" Map
4553  if (map.is_null ()) {
4554  // The user didn't supply a Map. Make a contiguous
4555  // distributed Map for them, using the read-in multivector
4556  // dimensions.
4557  map = createUniformContigMapWithNode<LO, GO, NT> (numRows, comm);
4558  const size_t localNumRows = (myRank == 0) ? numRows : 0;
4559  // At this point, map exists and has a nonnull node.
4560  proc0Map = createContigMapWithNode<LO, GO, NT> (numRows, localNumRows,
4561  comm);
4562  }
4563  else { // The user supplied a Map.
4564  proc0Map = Details::computeGatherMap<map_type> (map, err, debug);
4565  }
4566 
4567  // Make a multivector X owned entirely by Proc 0.
4568  RCP<MV> X = createMultiVector<ST, LO, GO, NT> (proc0Map, numCols);
4569 
4570  //
4571  // On Proc 0, read the Matrix Market data from the input
4572  // stream into the multivector X.
4573  //
4574  int localReadDataSuccess = 1;
4575  if (myRank == 0) {
4576  try {
4577  if (debug) {
4578  *err << myRank << ": readDenseImpl: Reading matrix data (dense)"
4579  << endl;
4580  }
4581 
4582  // Make sure that we can get a 1-D view of X.
4583  TEUCHOS_TEST_FOR_EXCEPTION(
4584  ! X->isConstantStride (), std::logic_error,
4585  "Can't get a 1-D view of the entries of the MultiVector X on "
4586  "Process 0, because the stride between the columns of X is not "
4587  "constant. This shouldn't happen because we just created X and "
4588  "haven't filled it in yet. Please report this bug to the Tpetra "
4589  "developers.");
4590 
4591  // Get a writeable 1-D view of the entries of X. Rank 0
4592  // owns all of them. The view will expire at the end of
4593  // scope, so (if necessary) it will be written back to X
4594  // at this time.
4595  ArrayRCP<ST> X_view = X->get1dViewNonConst ();
4596  TEUCHOS_TEST_FOR_EXCEPTION(
4597  as<global_size_t> (X_view.size ()) < numRows * numCols,
4598  std::logic_error,
4599  "The view of X has size " << X_view << " which is not enough to "
4600  "accommodate the expected number of entries numRows*numCols = "
4601  << numRows << "*" << numCols << " = " << numRows*numCols << ". "
4602  "Please report this bug to the Tpetra developers.");
4603  const size_t stride = X->getStride ();
4604 
4605  // The third element of the dimensions Tuple encodes the data
4606  // type reported by the Banner: "real" == 0, "complex" == 1,
4607  // "integer" == 0 (same as "real"), "pattern" == 2. We do not
4608  // allow dense matrices to be pattern matrices, so dims[2] ==
4609  // 0 or 1. We've already checked for this above.
4610  const bool isComplex = (dims[2] == 1);
4611  size_type count = 0, curRow = 0, curCol = 0;
4612 
4613  std::string line;
4614  while (getline (in, line)) {
4615  ++lineNumber;
4616  // Is the current line a comment line? If it's not,
4617  // line.substr(start,size) contains the data.
4618  size_t start = 0, size = 0;
4619  const bool commentLine =
4620  checkCommentLine (line, start, size, lineNumber, tolerant);
4621  if (! commentLine) {
4622  // Make sure we have room in which to put the new matrix
4623  // entry. We check this only after checking for a
4624  // comment line, because there may be one or more
4625  // comment lines at the end of the file. In tolerant
4626  // mode, we simply ignore any extra data.
4627  if (count >= X_view.size()) {
4628  if (tolerant) {
4629  break;
4630  }
4631  else {
4632  TEUCHOS_TEST_FOR_EXCEPTION(
4633  count >= X_view.size(),
4634  std::runtime_error,
4635  "The Matrix Market input stream has more data in it than "
4636  "its metadata reported. Current line number is "
4637  << lineNumber << ".");
4638  }
4639  }
4640 
4641  // mfh 19 Dec 2012: Ignore everything up to the initial
4642  // colon. writeDense() has the option to print out the
4643  // global row index in front of each entry, followed by
4644  // a colon and space.
4645  {
4646  const size_t pos = line.substr (start, size).find (':');
4647  if (pos != std::string::npos) {
4648  start = pos+1;
4649  }
4650  }
4651  std::istringstream istr (line.substr (start, size));
4652  // Does the line contain anything at all? Can we
4653  // safely read from the input stream wrapping the
4654  // line?
4655  if (istr.eof() || istr.fail()) {
4656  // In tolerant mode, simply ignore the line.
4657  if (tolerant) {
4658  break;
4659  }
4660  // We repeat the full test here so the exception
4661  // message is more informative.
4662  TEUCHOS_TEST_FOR_EXCEPTION(
4663  ! tolerant && (istr.eof() || istr.fail()),
4664  std::runtime_error,
4665  "Line " << lineNumber << " of the Matrix Market file is "
4666  "empty, or we cannot read from it for some other reason.");
4667  }
4668  // Current matrix entry to read in.
4669  ST val = STS::zero();
4670  // Real and imaginary parts of the current matrix entry.
4671  // The imaginary part is zero if the matrix is real-valued.
4672  MT real = STM::zero(), imag = STM::zero();
4673 
4674  // isComplex refers to the input stream's data, not to
4675  // the scalar type S. It's OK to read real-valued
4676  // data into a matrix storing complex-valued data; in
4677  // that case, all entries' imaginary parts are zero.
4678  if (isComplex) {
4679  // STS::real() and STS::imag() return a copy of
4680  // their respective components, not a writeable
4681  // reference. Otherwise we could just assign to
4682  // them using the istream extraction operator (>>).
4683  // That's why we have separate magnitude type "real"
4684  // and "imag" variables.
4685 
4686  // Attempt to read the real part of the current entry.
4687  istr >> real;
4688  if (istr.fail()) {
4689  TEUCHOS_TEST_FOR_EXCEPTION(
4690  ! tolerant && istr.eof(), std::runtime_error,
4691  "Failed to get the real part of a complex-valued matrix "
4692  "entry from line " << lineNumber << " of the Matrix Market "
4693  "file.");
4694  // In tolerant mode, just skip bad lines.
4695  if (tolerant) {
4696  break;
4697  }
4698  } else if (istr.eof()) {
4699  TEUCHOS_TEST_FOR_EXCEPTION(
4700  ! tolerant && istr.eof(), std::runtime_error,
4701  "Missing imaginary part of a complex-valued matrix entry "
4702  "on line " << lineNumber << " of the Matrix Market file.");
4703  // In tolerant mode, let any missing imaginary part be 0.
4704  } else {
4705  // Attempt to read the imaginary part of the current
4706  // matrix entry.
4707  istr >> imag;
4708  TEUCHOS_TEST_FOR_EXCEPTION(
4709  ! tolerant && istr.fail(), std::runtime_error,
4710  "Failed to get the imaginary part of a complex-valued "
4711  "matrix entry from line " << lineNumber << " of the "
4712  "Matrix Market file.");
4713  // In tolerant mode, let any missing or corrupted
4714  // imaginary part be 0.
4715  }
4716  } else { // Matrix Market file contains real-valued data.
4717  // Attempt to read the current matrix entry.
4718  istr >> real;
4719  TEUCHOS_TEST_FOR_EXCEPTION(
4720  ! tolerant && istr.fail(), std::runtime_error,
4721  "Failed to get a real-valued matrix entry from line "
4722  << lineNumber << " of the Matrix Market file.");
4723  // In tolerant mode, simply ignore the line if
4724  // we failed to read a matrix entry.
4725  if (istr.fail() && tolerant) {
4726  break;
4727  }
4728  }
4729  // In tolerant mode, we simply let pass through whatever
4730  // data we got.
4731  TEUCHOS_TEST_FOR_EXCEPTION(
4732  ! tolerant && istr.fail(), std::runtime_error,
4733  "Failed to read matrix data from line " << lineNumber
4734  << " of the Matrix Market file.");
4735 
4736  // Assign val = ST(real, imag).
4737  Teuchos::MatrixMarket::details::assignScalar<ST> (val, real, imag);
4738 
4739  curRow = count % numRows;
4740  curCol = count / numRows;
4741  X_view[curRow + curCol*stride] = val;
4742  ++count;
4743  } // if not a comment line
4744  } // while there are still lines in the file, get the next one
4745 
4746  TEUCHOS_TEST_FOR_EXCEPTION(
4747  ! tolerant && static_cast<global_size_t> (count) < numRows * numCols,
4748  std::runtime_error,
4749  "The Matrix Market metadata reports that the dense matrix is "
4750  << numRows << " x " << numCols << ", and thus has "
4751  << numRows*numCols << " total entries, but we only found " << count
4752  << " entr" << (count == 1 ? "y" : "ies") << " in the file.");
4753  } catch (std::exception& e) {
4754  exMsg << e.what ();
4755  localReadDataSuccess = 0;
4756  }
4757  } // if (myRank == 0)
4758 
4759  if (debug) {
4760  *err << myRank << ": readDenseImpl: done reading data" << endl;
4761  }
4762 
4763  // Synchronize on whether Proc 0 successfully read the data.
4764  int globalReadDataSuccess = localReadDataSuccess;
4765  broadcast (*comm, 0, outArg (globalReadDataSuccess));
4766  TEUCHOS_TEST_FOR_EXCEPTION(
4767  globalReadDataSuccess == 0, std::runtime_error,
4768  "Failed to read the multivector's data: " << exMsg.str ());
4769 
4770  // If there's only one MPI process and the user didn't supply
4771  // a Map (i.e., pMap is null), we're done. Set pMap to the
4772  // Map used to distribute X, and return X.
4773  if (comm->getSize () == 1 && map.is_null ()) {
4774  map = proc0Map;
4775  if (! err.is_null ()) {
4776  err->popTab ();
4777  }
4778  if (debug) {
4779  *err << myRank << ": readDenseImpl: done" << endl;
4780  }
4781  if (! err.is_null ()) {
4782  err->popTab ();
4783  }
4784  return X;
4785  }
4786 
4787  if (debug) {
4788  *err << myRank << ": readDenseImpl: Creating target MV" << endl;
4789  }
4790 
4791  // Make a multivector Y with the distributed map pMap.
4792  RCP<MV> Y = createMultiVector<ST, LO, GO, NT> (map, numCols);
4793 
4794  if (debug) {
4795  *err << myRank << ": readDenseImpl: Creating Export" << endl;
4796  }
4797 
4798  // Make an Export object that will export X to Y. First
4799  // argument is the source map, second argument is the target
4800  // map.
4801  Export<LO, GO, NT> exporter (proc0Map, map, err);
4802 
4803  if (debug) {
4804  *err << myRank << ": readDenseImpl: Exporting" << endl;
4805  }
4806  // Export X into Y.
4807  Y->doExport (*X, exporter, INSERT);
4808 
4809  if (! err.is_null ()) {
4810  err->popTab ();
4811  }
4812  if (debug) {
4813  *err << myRank << ": readDenseImpl: done" << endl;
4814  }
4815  if (! err.is_null ()) {
4816  err->popTab ();
4817  }
4818 
4819  // Y is distributed over all process(es) in the communicator.
4820  return Y;
4821  }
4822 
4823 
4824  template<class VectorScalarType>
4825  static Teuchos::RCP<Tpetra::Vector<VectorScalarType,
4828  node_type> >
4829  readVectorImpl (std::istream& in,
4830  const Teuchos::RCP<const comm_type>& comm,
4831  Teuchos::RCP<const map_type>& map,
4832  const Teuchos::RCP<Teuchos::FancyOStream>& err,
4833  const bool tolerant=false,
4834  const bool debug=false)
4835  {
4836  using Teuchos::MatrixMarket::Banner;
4837  using Teuchos::MatrixMarket::checkCommentLine;
4838  using Teuchos::as;
4839  using Teuchos::broadcast;
4840  using Teuchos::outArg;
4841  using Teuchos::RCP;
4842  using Teuchos::Tuple;
4843  using std::endl;
4844  typedef VectorScalarType ST;
4845  typedef local_ordinal_type LO;
4846  typedef global_ordinal_type GO;
4847  typedef node_type NT;
4848  typedef Teuchos::ScalarTraits<ST> STS;
4849  typedef typename STS::magnitudeType MT;
4850  typedef Teuchos::ScalarTraits<MT> STM;
4851  typedef Tpetra::Vector<ST, LO, GO, NT> MV;
4852 
4853  // Rank 0 is the only (MPI) process allowed to read from the
4854  // input stream.
4855  const int myRank = comm->getRank ();
4856 
4857  if (! err.is_null ()) {
4858  err->pushTab ();
4859  }
4860  if (debug) {
4861  *err << myRank << ": readVectorImpl" << endl;
4862  }
4863  if (! err.is_null ()) {
4864  err->pushTab ();
4865  }
4866 
4867  // mfh 17 Feb 2013: It's not strictly necessary that the Comm
4868  // instances be identical and that the Node instances be
4869  // identical. The essential condition is more complicated to
4870  // test and isn't the same for all Node types. Thus, we just
4871  // leave it up to the user.
4872 
4873  // // If map is nonnull, check the precondition that its
4874  // // communicator resp. node equal comm resp. node. Checking
4875  // // now avoids doing a lot of file reading before we detect the
4876  // // violated precondition.
4877  // TEUCHOS_TEST_FOR_EXCEPTION(
4878  // ! map.is_null() && (map->getComm() != comm || map->getNode () != node,
4879  // std::invalid_argument, "If you supply a nonnull Map, the Map's "
4880  // "communicator and node must equal the supplied communicator resp. "
4881  // "node.");
4882 
4883  // Process 0 will read in the matrix dimensions from the file,
4884  // and broadcast them to all ranks in the given communicator.
4885  // There are only 2 dimensions in the matrix, but we use the
4886  // third element of the Tuple to encode the banner's reported
4887  // data type: "real" == 0, "complex" == 1, and "integer" == 0
4888  // (same as "real"). We don't allow pattern matrices (i.e.,
4889  // graphs) since they only make sense for sparse data.
4890  Tuple<GO, 3> dims;
4891  dims[0] = 0;
4892  dims[1] = 0;
4893 
4894  // Current line number in the input stream. Only valid on
4895  // Proc 0. Various calls will modify this depending on the
4896  // number of lines that are read from the input stream.
4897  size_t lineNumber = 1;
4898 
4899  // Capture errors and their messages on Proc 0.
4900  std::ostringstream exMsg;
4901  int localBannerReadSuccess = 1;
4902  int localDimsReadSuccess = 1;
4903 
4904  // Only Proc 0 gets to read matrix data from the input stream.
4905  if (myRank == 0) {
4906  if (debug) {
4907  *err << myRank << ": readVectorImpl: Reading banner line (dense)" << endl;
4908  }
4909 
4910  // The "Banner" tells you whether the input stream
4911  // represents a dense matrix, the symmetry type of the
4912  // matrix, and the type of the data it contains.
4913  RCP<const Banner> pBanner;
4914  try {
4915  pBanner = readBanner (in, lineNumber, tolerant, debug);
4916  } catch (std::exception& e) {
4917  exMsg << e.what ();
4918  localBannerReadSuccess = 0;
4919  }
4920  // Make sure the input stream is the right kind of data.
4921  if (localBannerReadSuccess) {
4922  if (pBanner->matrixType () != "array") {
4923  exMsg << "The Matrix Market file does not contain dense matrix "
4924  "data. Its banner (first) line says that its matrix type is \""
4925  << pBanner->matrixType () << "\", rather that the required "
4926  "\"array\".";
4927  localBannerReadSuccess = 0;
4928  } else if (pBanner->dataType() == "pattern") {
4929  exMsg << "The Matrix Market file's banner (first) "
4930  "line claims that the matrix's data type is \"pattern\". This does "
4931  "not make sense for a dense matrix, yet the file reports the matrix "
4932  "as dense. The only valid data types for a dense matrix are "
4933  "\"real\", \"complex\", and \"integer\".";
4934  localBannerReadSuccess = 0;
4935  } else {
4936  // Encode the data type reported by the Banner as the
4937  // third element of the dimensions Tuple.
4938  dims[2] = encodeDataType (pBanner->dataType ());
4939  }
4940  } // if we successfully read the banner line
4941 
4942  // At this point, we've successfully read the banner line.
4943  // Now read the dimensions line.
4944  if (localBannerReadSuccess) {
4945  if (debug) {
4946  *err << myRank << ": readVectorImpl: Reading dimensions line (dense)" << endl;
4947  }
4948  // Keep reading lines from the input stream until we find
4949  // a non-comment line, or until we run out of lines. The
4950  // latter is an error, since every "array" format Matrix
4951  // Market file must have a dimensions line after the
4952  // banner (even if the matrix has zero rows or columns, or
4953  // zero entries).
4954  std::string line;
4955  bool commentLine = true;
4956 
4957  while (commentLine) {
4958  // Test whether it is even valid to read from the input
4959  // stream wrapping the line.
4960  if (in.eof () || in.fail ()) {
4961  exMsg << "Unable to get array dimensions line (at all) from line "
4962  << lineNumber << " of input stream. The input stream "
4963  << "claims that it is "
4964  << (in.eof() ? "at end-of-file." : "in a failed state.");
4965  localDimsReadSuccess = 0;
4966  } else {
4967  // Try to get the next line from the input stream.
4968  if (getline (in, line)) {
4969  ++lineNumber; // We did actually read a line.
4970  }
4971  // Is the current line a comment line? Ignore start
4972  // and size; they are only useful for reading the
4973  // actual matrix entries. (We could use them here as
4974  // an optimization, but we've chosen not to.)
4975  size_t start = 0, size = 0;
4976  commentLine = checkCommentLine (line, start, size, lineNumber, tolerant);
4977  } // whether we failed to read the line at all
4978  } // while the line we just read is a comment line
4979 
4980  //
4981  // Get <numRows> <numCols> from the line we just read.
4982  //
4983  std::istringstream istr (line);
4984 
4985  // Test whether it is even valid to read from the input
4986  // stream wrapping the line.
4987  if (istr.eof () || istr.fail ()) {
4988  exMsg << "Unable to read any data from line " << lineNumber
4989  << " of input; the line should contain the matrix dimensions "
4990  << "\"<numRows> <numCols>\".";
4991  localDimsReadSuccess = 0;
4992  } else { // It's valid to read from the line.
4993  GO theNumRows = 0;
4994  istr >> theNumRows; // Read in the number of rows.
4995  if (istr.fail ()) {
4996  exMsg << "Failed to get number of rows from line "
4997  << lineNumber << " of input; the line should contains the "
4998  << "matrix dimensions \"<numRows> <numCols>\".";
4999  localDimsReadSuccess = 0;
5000  } else { // We successfully read the number of rows
5001  dims[0] = theNumRows; // Save the number of rows
5002  if (istr.eof ()) { // Do we still have data to read?
5003  exMsg << "No more data after number of rows on line "
5004  << lineNumber << " of input; the line should contain the "
5005  << "matrix dimensions \"<numRows> <numCols>\".";
5006  localDimsReadSuccess = 0;
5007  } else { // Still data left to read; read in number of columns.
5008  GO theNumCols = 0;
5009  istr >> theNumCols; // Read in the number of columns
5010  if (istr.fail ()) {
5011  exMsg << "Failed to get number of columns from line "
5012  << lineNumber << " of input; the line should contain "
5013  << "the matrix dimensions \"<numRows> <numCols>\".";
5014  localDimsReadSuccess = 0;
5015  } else { // We successfully read the number of columns
5016  dims[1] = theNumCols; // Save the number of columns
5017  } // if istr.fail ()
5018  } // if istr.eof ()
5019  } // if we read the number of rows
5020  } // if the input stream wrapping the dims line was (in)valid
5021  } // if we successfully read the banner line
5022  } // if (myRank == 0)
5023 
5024  // Check if file has a Vector
5025  if (dims[1]!=1) {
5026  exMsg << "File does not contain a 1D Vector";
5027  localDimsReadSuccess = 0;
5028  }
5029 
5030  // Broadcast the matrix dimensions, the encoded data type, and
5031  // whether or not Proc 0 succeeded in reading the banner and
5032  // dimensions.
5033  Tuple<GO, 5> bannerDimsReadResult;
5034  if (myRank == 0) {
5035  bannerDimsReadResult[0] = dims[0]; // numRows
5036  bannerDimsReadResult[1] = dims[1]; // numCols
5037  bannerDimsReadResult[2] = dims[2]; // encoded data type
5038  bannerDimsReadResult[3] = localBannerReadSuccess;
5039  bannerDimsReadResult[4] = localDimsReadSuccess;
5040  }
5041 
5042  // Broadcast matrix dimensions and the encoded data type from
5043  // Proc 0 to all the MPI processes.
5044  broadcast (*comm, 0, bannerDimsReadResult);
5045 
5046  TEUCHOS_TEST_FOR_EXCEPTION(
5047  bannerDimsReadResult[3] == 0, std::runtime_error,
5048  "Failed to read banner line: " << exMsg.str ());
5049  TEUCHOS_TEST_FOR_EXCEPTION(
5050  bannerDimsReadResult[4] == 0, std::runtime_error,
5051  "Failed to read matrix dimensions line: " << exMsg.str ());
5052  if (myRank != 0) {
5053  dims[0] = bannerDimsReadResult[0];
5054  dims[1] = bannerDimsReadResult[1];
5055  dims[2] = bannerDimsReadResult[2];
5056  }
5057 
5058  // Tpetra objects want the matrix dimensions in these types.
5059  const global_size_t numRows = static_cast<global_size_t> (dims[0]);
5060  const size_t numCols = static_cast<size_t> (dims[1]);
5061 
5062  // Make a "Proc 0 owns everything" Map that we will use to
5063  // read in the multivector entries in the correct order on
5064  // Proc 0. This must be a collective
5065  RCP<const map_type> proc0Map; // "Proc 0 owns everything" Map
5066  if (map.is_null ()) {
5067  // The user didn't supply a Map. Make a contiguous
5068  // distributed Map for them, using the read-in multivector
5069  // dimensions.
5070  map = createUniformContigMapWithNode<LO, GO, NT> (numRows, comm);
5071  const size_t localNumRows = (myRank == 0) ? numRows : 0;
5072  // At this point, map exists and has a nonnull node.
5073  proc0Map = createContigMapWithNode<LO, GO, NT> (numRows, localNumRows,
5074  comm);
5075  }
5076  else { // The user supplied a Map.
5077  proc0Map = Details::computeGatherMap<map_type> (map, err, debug);
5078  }
5079 
5080  // Make a multivector X owned entirely by Proc 0.
5081  RCP<MV> X = createVector<ST, LO, GO, NT> (proc0Map);
5082 
5083  //
5084  // On Proc 0, read the Matrix Market data from the input
5085  // stream into the multivector X.
5086  //
5087  int localReadDataSuccess = 1;
5088  if (myRank == 0) {
5089  try {
5090  if (debug) {
5091  *err << myRank << ": readVectorImpl: Reading matrix data (dense)"
5092  << endl;
5093  }
5094 
5095  // Make sure that we can get a 1-D view of X.
5096  TEUCHOS_TEST_FOR_EXCEPTION(
5097  ! X->isConstantStride (), std::logic_error,
5098  "Can't get a 1-D view of the entries of the MultiVector X on "
5099  "Process 0, because the stride between the columns of X is not "
5100  "constant. This shouldn't happen because we just created X and "
5101  "haven't filled it in yet. Please report this bug to the Tpetra "
5102  "developers.");
5103 
5104  // Get a writeable 1-D view of the entries of X. Rank 0
5105  // owns all of them. The view will expire at the end of
5106  // scope, so (if necessary) it will be written back to X
5107  // at this time.
5108  Teuchos::ArrayRCP<ST> X_view = X->get1dViewNonConst ();
5109  TEUCHOS_TEST_FOR_EXCEPTION(
5110  as<global_size_t> (X_view.size ()) < numRows * numCols,
5111  std::logic_error,
5112  "The view of X has size " << X_view << " which is not enough to "
5113  "accommodate the expected number of entries numRows*numCols = "
5114  << numRows << "*" << numCols << " = " << numRows*numCols << ". "
5115  "Please report this bug to the Tpetra developers.");
5116  const size_t stride = X->getStride ();
5117 
5118  // The third element of the dimensions Tuple encodes the data
5119  // type reported by the Banner: "real" == 0, "complex" == 1,
5120  // "integer" == 0 (same as "real"), "pattern" == 2. We do not
5121  // allow dense matrices to be pattern matrices, so dims[2] ==
5122  // 0 or 1. We've already checked for this above.
5123  const bool isComplex = (dims[2] == 1);
5124  size_type count = 0, curRow = 0, curCol = 0;
5125 
5126  std::string line;
5127  while (getline (in, line)) {
5128  ++lineNumber;
5129  // Is the current line a comment line? If it's not,
5130  // line.substr(start,size) contains the data.
5131  size_t start = 0, size = 0;
5132  const bool commentLine =
5133  checkCommentLine (line, start, size, lineNumber, tolerant);
5134  if (! commentLine) {
5135  // Make sure we have room in which to put the new matrix
5136  // entry. We check this only after checking for a
5137  // comment line, because there may be one or more
5138  // comment lines at the end of the file. In tolerant
5139  // mode, we simply ignore any extra data.
5140  if (count >= X_view.size()) {
5141  if (tolerant) {
5142  break;
5143  }
5144  else {
5145  TEUCHOS_TEST_FOR_EXCEPTION(
5146  count >= X_view.size(),
5147  std::runtime_error,
5148  "The Matrix Market input stream has more data in it than "
5149  "its metadata reported. Current line number is "
5150  << lineNumber << ".");
5151  }
5152  }
5153 
5154  // mfh 19 Dec 2012: Ignore everything up to the initial
5155  // colon. writeDense() has the option to print out the
5156  // global row index in front of each entry, followed by
5157  // a colon and space.
5158  {
5159  const size_t pos = line.substr (start, size).find (':');
5160  if (pos != std::string::npos) {
5161  start = pos+1;
5162  }
5163  }
5164  std::istringstream istr (line.substr (start, size));
5165  // Does the line contain anything at all? Can we
5166  // safely read from the input stream wrapping the
5167  // line?
5168  if (istr.eof() || istr.fail()) {
5169  // In tolerant mode, simply ignore the line.
5170  if (tolerant) {
5171  break;
5172  }
5173  // We repeat the full test here so the exception
5174  // message is more informative.
5175  TEUCHOS_TEST_FOR_EXCEPTION(
5176  ! tolerant && (istr.eof() || istr.fail()),
5177  std::runtime_error,
5178  "Line " << lineNumber << " of the Matrix Market file is "
5179  "empty, or we cannot read from it for some other reason.");
5180  }
5181  // Current matrix entry to read in.
5182  ST val = STS::zero();
5183  // Real and imaginary parts of the current matrix entry.
5184  // The imaginary part is zero if the matrix is real-valued.
5185  MT real = STM::zero(), imag = STM::zero();
5186 
5187  // isComplex refers to the input stream's data, not to
5188  // the scalar type S. It's OK to read real-valued
5189  // data into a matrix storing complex-valued data; in
5190  // that case, all entries' imaginary parts are zero.
5191  if (isComplex) {
5192  // STS::real() and STS::imag() return a copy of
5193  // their respective components, not a writeable
5194  // reference. Otherwise we could just assign to
5195  // them using the istream extraction operator (>>).
5196  // That's why we have separate magnitude type "real"
5197  // and "imag" variables.
5198 
5199  // Attempt to read the real part of the current entry.
5200  istr >> real;
5201  if (istr.fail()) {
5202  TEUCHOS_TEST_FOR_EXCEPTION(
5203  ! tolerant && istr.eof(), std::runtime_error,
5204  "Failed to get the real part of a complex-valued matrix "
5205  "entry from line " << lineNumber << " of the Matrix Market "
5206  "file.");
5207  // In tolerant mode, just skip bad lines.
5208  if (tolerant) {
5209  break;
5210  }
5211  } else if (istr.eof()) {
5212  TEUCHOS_TEST_FOR_EXCEPTION(
5213  ! tolerant && istr.eof(), std::runtime_error,
5214  "Missing imaginary part of a complex-valued matrix entry "
5215  "on line " << lineNumber << " of the Matrix Market file.");
5216  // In tolerant mode, let any missing imaginary part be 0.
5217  } else {
5218  // Attempt to read the imaginary part of the current
5219  // matrix entry.
5220  istr >> imag;
5221  TEUCHOS_TEST_FOR_EXCEPTION(
5222  ! tolerant && istr.fail(), std::runtime_error,
5223  "Failed to get the imaginary part of a complex-valued "
5224  "matrix entry from line " << lineNumber << " of the "
5225  "Matrix Market file.");
5226  // In tolerant mode, let any missing or corrupted
5227  // imaginary part be 0.
5228  }
5229  } else { // Matrix Market file contains real-valued data.
5230  // Attempt to read the current matrix entry.
5231  istr >> real;
5232  TEUCHOS_TEST_FOR_EXCEPTION(
5233  ! tolerant && istr.fail(), std::runtime_error,
5234  "Failed to get a real-valued matrix entry from line "
5235  << lineNumber << " of the Matrix Market file.");
5236  // In tolerant mode, simply ignore the line if
5237  // we failed to read a matrix entry.
5238  if (istr.fail() && tolerant) {
5239  break;
5240  }
5241  }
5242  // In tolerant mode, we simply let pass through whatever
5243  // data we got.
5244  TEUCHOS_TEST_FOR_EXCEPTION(
5245  ! tolerant && istr.fail(), std::runtime_error,
5246  "Failed to read matrix data from line " << lineNumber
5247  << " of the Matrix Market file.");
5248 
5249  // Assign val = ST(real, imag).
5250  Teuchos::MatrixMarket::details::assignScalar<ST> (val, real, imag);
5251 
5252  curRow = count % numRows;
5253  curCol = count / numRows;
5254  X_view[curRow + curCol*stride] = val;
5255  ++count;
5256  } // if not a comment line
5257  } // while there are still lines in the file, get the next one
5258 
5259  TEUCHOS_TEST_FOR_EXCEPTION(
5260  ! tolerant && static_cast<global_size_t> (count) < numRows * numCols,
5261  std::runtime_error,
5262  "The Matrix Market metadata reports that the dense matrix is "
5263  << numRows << " x " << numCols << ", and thus has "
5264  << numRows*numCols << " total entries, but we only found " << count
5265  << " entr" << (count == 1 ? "y" : "ies") << " in the file.");
5266  } catch (std::exception& e) {
5267  exMsg << e.what ();
5268  localReadDataSuccess = 0;
5269  }
5270  } // if (myRank == 0)
5271 
5272  if (debug) {
5273  *err << myRank << ": readVectorImpl: done reading data" << endl;
5274  }
5275 
5276  // Synchronize on whether Proc 0 successfully read the data.
5277  int globalReadDataSuccess = localReadDataSuccess;
5278  broadcast (*comm, 0, outArg (globalReadDataSuccess));
5279  TEUCHOS_TEST_FOR_EXCEPTION(
5280  globalReadDataSuccess == 0, std::runtime_error,
5281  "Failed to read the multivector's data: " << exMsg.str ());
5282 
5283  // If there's only one MPI process and the user didn't supply
5284  // a Map (i.e., pMap is null), we're done. Set pMap to the
5285  // Map used to distribute X, and return X.
5286  if (comm->getSize () == 1 && map.is_null ()) {
5287  map = proc0Map;
5288  if (! err.is_null ()) {
5289  err->popTab ();
5290  }
5291  if (debug) {
5292  *err << myRank << ": readVectorImpl: done" << endl;
5293  }
5294  if (! err.is_null ()) {
5295  err->popTab ();
5296  }
5297  return X;
5298  }
5299 
5300  if (debug) {
5301  *err << myRank << ": readVectorImpl: Creating target MV" << endl;
5302  }
5303 
5304  // Make a multivector Y with the distributed map pMap.
5305  RCP<MV> Y = createVector<ST, LO, GO, NT> (map);
5306 
5307  if (debug) {
5308  *err << myRank << ": readVectorImpl: Creating Export" << endl;
5309  }
5310 
5311  // Make an Export object that will export X to Y. First
5312  // argument is the source map, second argument is the target
5313  // map.
5314  Export<LO, GO, NT> exporter (proc0Map, map, err);
5315 
5316  if (debug) {
5317  *err << myRank << ": readVectorImpl: Exporting" << endl;
5318  }
5319  // Export X into Y.
5320  Y->doExport (*X, exporter, INSERT);
5321 
5322  if (! err.is_null ()) {
5323  err->popTab ();
5324  }
5325  if (debug) {
5326  *err << myRank << ": readVectorImpl: done" << endl;
5327  }
5328  if (! err.is_null ()) {
5329  err->popTab ();
5330  }
5331 
5332  // Y is distributed over all process(es) in the communicator.
5333  return Y;
5334  }
5335 
5336  public:
5356  static Teuchos::RCP<const map_type>
5357  readMap (std::istream& in,
5358  const Teuchos::RCP<const comm_type>& comm,
5359  const bool tolerant=false,
5360  const bool debug=false)
5361  {
5362  Teuchos::RCP<Teuchos::FancyOStream> err =
5363  Teuchos::getFancyOStream (Teuchos::rcpFromRef (std::cerr));
5364  return readMap (in, comm, err, tolerant, debug);
5365  }
5366 
5367 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
5368  static Teuchos::RCP<const map_type> TPETRA_DEPRECATED
5371  readMap (std::istream& in,
5372  const Teuchos::RCP<const comm_type>& comm,
5373  const Teuchos::RCP<node_type>& /* pNode */,
5374  const bool tolerant=false,
5375  const bool debug=false)
5376  {
5377  return readMap(in, comm, tolerant, debug);
5378  }
5379 #endif // TPETRA_ENABLE_DEPRECATED_CODE
5380 
5406  static Teuchos::RCP<const map_type>
5407  readMap (std::istream& in,
5408  const Teuchos::RCP<const comm_type>& comm,
5409  const Teuchos::RCP<Teuchos::FancyOStream>& err,
5410  const bool tolerant=false,
5411  const bool debug=false)
5412  {
5413  using Teuchos::arcp;
5414  using Teuchos::Array;
5415  using Teuchos::ArrayRCP;
5416  using Teuchos::as;
5417  using Teuchos::broadcast;
5418  using Teuchos::Comm;
5419  using Teuchos::CommRequest;
5420  using Teuchos::inOutArg;
5421  using Teuchos::ireceive;
5422  using Teuchos::outArg;
5423  using Teuchos::RCP;
5424  using Teuchos::receive;
5425  using Teuchos::reduceAll;
5426  using Teuchos::REDUCE_MIN;
5427  using Teuchos::isend;
5428  using Teuchos::SerialComm;
5429  using Teuchos::toString;
5430  using Teuchos::wait;
5431  using std::endl;
5432  typedef Tpetra::global_size_t GST;
5433  typedef ptrdiff_t int_type; // Can hold int and GO
5434  typedef local_ordinal_type LO;
5435  typedef global_ordinal_type GO;
5436  typedef node_type NT;
5438 
5439  const int numProcs = comm->getSize ();
5440  const int myRank = comm->getRank ();
5441 
5442  if (err.is_null ()) {
5443  err->pushTab ();
5444  }
5445  if (debug) {
5446  std::ostringstream os;
5447  os << myRank << ": readMap: " << endl;
5448  *err << os.str ();
5449  }
5450  if (err.is_null ()) {
5451  err->pushTab ();
5452  }
5453 
5454  // Tag for receive-size / send-size messages. writeMap used
5455  // tags 1337 and 1338; we count up from there.
5456  const int sizeTag = 1339;
5457  // Tag for receive-data / send-data messages.
5458  const int dataTag = 1340;
5459 
5460  // These are for sends on Process 0, and for receives on all
5461  // other processes. sizeReq is for the {receive,send}-size
5462  // message, and dataReq is for the message containing the
5463  // actual GIDs to belong to the receiving process.
5464  RCP<CommRequest<int> > sizeReq;
5465  RCP<CommRequest<int> > dataReq;
5466 
5467  // Each process will have to receive the number of GIDs to
5468  // expect. Thus, we can post the receives now, and cancel
5469  // them if something should go wrong in the meantime.
5470  ArrayRCP<int_type> numGidsToRecv (1);
5471  numGidsToRecv[0] = 0;
5472  if (myRank != 0) {
5473  sizeReq = ireceive<int, int_type> (numGidsToRecv, 0, sizeTag, *comm);
5474  }
5475 
5476  int readSuccess = 1;
5477  std::ostringstream exMsg;
5478  RCP<MV> data; // Will only be valid on Proc 0
5479  if (myRank == 0) {
5480  // If we want to reuse readDenseImpl, we have to make a
5481  // communicator that only contains Proc 0. Otherwise,
5482  // readDenseImpl will redistribute the data to all
5483  // processes. While we eventually want that, neither we nor
5484  // readDenseImpl know the correct Map to use at the moment.
5485  // That depends on the second column of the multivector.
5486  RCP<const Comm<int> > proc0Comm (new SerialComm<int> ());
5487  try {
5488  RCP<const map_type> dataMap;
5489  // This is currently the only place where we use the
5490  // 'tolerant' argument. Later, if we want to be clever,
5491  // we could have tolerant mode allow PIDs out of order.
5492  data = readDenseImpl<GO> (in, proc0Comm, dataMap, err, tolerant, debug);
5493  (void) dataMap; // Silence "unused" warnings
5494  if (data.is_null ()) {
5495  readSuccess = 0;
5496  exMsg << "readDenseImpl() returned null." << endl;
5497  }
5498  } catch (std::exception& e) {
5499  readSuccess = 0;
5500  exMsg << e.what () << endl;
5501  }
5502  }
5503 
5504  // Map from PID to all the GIDs for that PID.
5505  // Only populated on Process 0.
5506  std::map<int, Array<GO> > pid2gids;
5507 
5508  // The index base must be the global minimum GID.
5509  // We will compute this on Process 0 and broadcast,
5510  // so that all processes can set up the Map.
5511  int_type globalNumGIDs = 0;
5512 
5513  // The index base must be the global minimum GID.
5514  // We will compute this on Process 0 and broadcast,
5515  // so that all processes can set up the Map.
5516  GO indexBase = 0;
5517 
5518  // Process 0: If the above read of the MultiVector succeeded,
5519  // extract the GIDs and PIDs into pid2gids, and find the
5520  // global min GID.
5521  if (myRank == 0 && readSuccess == 1) {
5522  if (data->getNumVectors () == 2) { // Map format 1.0
5523  ArrayRCP<const GO> GIDs = data->getData (0);
5524  ArrayRCP<const GO> PIDs = data->getData (1); // convert to int
5525  globalNumGIDs = GIDs.size ();
5526 
5527  // Start computing the global min GID, while collecting
5528  // the GIDs for each PID.
5529  if (globalNumGIDs > 0) {
5530  const int pid = static_cast<int> (PIDs[0]);
5531 
5532  if (pid < 0 || pid >= numProcs) {
5533  readSuccess = 0;
5534  exMsg << "Tpetra::MatrixMarket::readMap: "
5535  << "Encountered invalid PID " << pid << "." << endl;
5536  }
5537  else {
5538  const GO gid = GIDs[0];
5539  pid2gids[pid].push_back (gid);
5540  indexBase = gid; // the current min GID
5541  }
5542  }
5543  if (readSuccess == 1) {
5544  // Collect the rest of the GIDs for each PID, and compute
5545  // the global min GID.
5546  for (size_type k = 1; k < globalNumGIDs; ++k) {
5547  const int pid = static_cast<int> (PIDs[k]);
5548  if (pid < 0 || pid >= numProcs) {
5549  readSuccess = 0;
5550  exMsg << "Tpetra::MatrixMarket::readMap: "
5551  << "Encountered invalid PID " << pid << "." << endl;
5552  }
5553  else {
5554  const int_type gid = GIDs[k];
5555  pid2gids[pid].push_back (gid);
5556  if (gid < indexBase) {
5557  indexBase = gid; // the current min GID
5558  }
5559  }
5560  }
5561  }
5562  }
5563  else if (data->getNumVectors () == 1) { // Map format 2.0
5564  if (data->getGlobalLength () % 2 != static_cast<GST> (0)) {
5565  readSuccess = 0;
5566  exMsg << "Tpetra::MatrixMarket::readMap: Input data has the "
5567  "wrong format (for Map format 2.0). The global number of rows "
5568  "in the MultiVector must be even (divisible by 2)." << endl;
5569  }
5570  else {
5571  ArrayRCP<const GO> theData = data->getData (0);
5572  globalNumGIDs = static_cast<GO> (data->getGlobalLength ()) /
5573  static_cast<GO> (2);
5574 
5575  // Start computing the global min GID, while
5576  // collecting the GIDs for each PID.
5577  if (globalNumGIDs > 0) {
5578  const int pid = static_cast<int> (theData[1]);
5579  if (pid < 0 || pid >= numProcs) {
5580  readSuccess = 0;
5581  exMsg << "Tpetra::MatrixMarket::readMap: "
5582  << "Encountered invalid PID " << pid << "." << endl;
5583  }
5584  else {
5585  const GO gid = theData[0];
5586  pid2gids[pid].push_back (gid);
5587  indexBase = gid; // the current min GID
5588  }
5589  }
5590  // Collect the rest of the GIDs for each PID, and
5591  // compute the global min GID.
5592  for (int_type k = 1; k < globalNumGIDs; ++k) {
5593  const int pid = static_cast<int> (theData[2*k + 1]);
5594  if (pid < 0 || pid >= numProcs) {
5595  readSuccess = 0;
5596  exMsg << "Tpetra::MatrixMarket::readMap: "
5597  << "Encountered invalid PID " << pid << "." << endl;
5598  }
5599  else {
5600  const GO gid = theData[2*k];
5601  pid2gids[pid].push_back (gid);
5602  if (gid < indexBase) {
5603  indexBase = gid; // the current min GID
5604  }
5605  }
5606  } // for each GID
5607  } // if the amount of data is correct
5608  }
5609  else {
5610  readSuccess = 0;
5611  exMsg << "Tpetra::MatrixMarket::readMap: Input data must have "
5612  "either 1 column (for the new Map format 2.0) or 2 columns (for "
5613  "the old Map format 1.0).";
5614  }
5615  } // myRank is zero
5616 
5617  // Broadcast the indexBase, the global number of GIDs, and the
5618  // current success status. Use int_type for all of these.
5619  {
5620  int_type readResults[3];
5621  readResults[0] = static_cast<int_type> (indexBase);
5622  readResults[1] = static_cast<int_type> (globalNumGIDs);
5623  readResults[2] = static_cast<int_type> (readSuccess);
5624  broadcast<int, int_type> (*comm, 0, 3, readResults);
5625 
5626  indexBase = static_cast<GO> (readResults[0]);
5627  globalNumGIDs = static_cast<int_type> (readResults[1]);
5628  readSuccess = static_cast<int> (readResults[2]);
5629  }
5630 
5631  // Unwinding the stack will invoke sizeReq's destructor, which
5632  // will cancel the receive-size request on all processes that
5633  // posted it.
5634  TEUCHOS_TEST_FOR_EXCEPTION(
5635  readSuccess != 1, std::runtime_error,
5636  "Tpetra::MatrixMarket::readMap: Reading the Map failed with the "
5637  "following exception message: " << exMsg.str ());
5638 
5639  if (myRank == 0) {
5640  // Proc 0: Send each process' number of GIDs to that process.
5641  for (int p = 1; p < numProcs; ++p) {
5642  ArrayRCP<int_type> numGidsToSend (1);
5643 
5644  typename std::map<int, Array<GO> >::const_iterator it = pid2gids.find (p);
5645  if (it == pid2gids.end ()) {
5646  numGidsToSend[0] = 0;
5647  } else {
5648  numGidsToSend[0] = it->second.size ();
5649  }
5650  sizeReq = isend<int, int_type> (numGidsToSend, p, sizeTag, *comm);
5651  wait<int> (*comm, outArg (sizeReq));
5652  }
5653  }
5654  else {
5655  // Wait on the receive-size message to finish.
5656  wait<int> (*comm, outArg (sizeReq));
5657  }
5658 
5659  // Allocate / get the array for my GIDs.
5660  // Only Process 0 will have its actual GIDs at this point.
5661  ArrayRCP<GO> myGids;
5662  int_type myNumGids = 0;
5663  if (myRank == 0) {
5664  GO* myGidsRaw = NULL;
5665 
5666  typename std::map<int, Array<GO> >::iterator it = pid2gids.find (0);
5667  if (it != pid2gids.end ()) {
5668  myGidsRaw = it->second.getRawPtr ();
5669  myNumGids = it->second.size ();
5670  // Nonowning ArrayRCP just views the Array.
5671  myGids = arcp<GO> (myGidsRaw, 0, myNumGids, false);
5672  }
5673  }
5674  else { // myRank != 0
5675  myNumGids = numGidsToRecv[0];
5676  myGids = arcp<GO> (myNumGids);
5677  }
5678 
5679  if (myRank != 0) {
5680  // Post receive for data, now that we know how much data we
5681  // will receive. Only post receive if my process actually
5682  // has nonzero GIDs.
5683  if (myNumGids > 0) {
5684  dataReq = ireceive<int, GO> (myGids, 0, dataTag, *comm);
5685  }
5686  }
5687 
5688  for (int p = 1; p < numProcs; ++p) {
5689  if (myRank == 0) {
5690  ArrayRCP<GO> sendGids; // to send to Process p
5691  GO* sendGidsRaw = NULL;
5692  int_type numSendGids = 0;
5693 
5694  typename std::map<int, Array<GO> >::iterator it = pid2gids.find (p);
5695  if (it != pid2gids.end ()) {
5696  numSendGids = it->second.size ();
5697  sendGidsRaw = it->second.getRawPtr ();
5698  sendGids = arcp<GO> (sendGidsRaw, 0, numSendGids, false);
5699  }
5700  // Only send if that process actually has nonzero GIDs.
5701  if (numSendGids > 0) {
5702  dataReq = isend<int, GO> (sendGids, p, dataTag, *comm);
5703  }
5704  wait<int> (*comm, outArg (dataReq));
5705  }
5706  else if (myRank == p) {
5707  // Wait on my receive of GIDs to finish.
5708  wait<int> (*comm, outArg (dataReq));
5709  }
5710  } // for each process rank p in 1, 2, ..., numProcs-1
5711 
5712  if (debug) {
5713  std::ostringstream os;
5714  os << myRank << ": readMap: creating Map" << endl;
5715  *err << os.str ();
5716  }
5717  const GST INVALID = Teuchos::OrdinalTraits<GST>::invalid ();
5718  RCP<const map_type> newMap;
5719 
5720  // Create the Map; test whether the constructor threw. This
5721  // avoids deadlock and makes error reporting more readable.
5722 
5723  int lclSuccess = 1;
5724  int gblSuccess = 0; // output argument
5725  std::ostringstream errStrm;
5726  try {
5727  newMap = rcp (new map_type (INVALID, myGids (), indexBase, comm));
5728  }
5729  catch (std::exception& e) {
5730  lclSuccess = 0;
5731  errStrm << "Process " << comm->getRank () << " threw an exception: "
5732  << e.what () << std::endl;
5733  }
5734  catch (...) {
5735  lclSuccess = 0;
5736  errStrm << "Process " << comm->getRank () << " threw an exception "
5737  "not a subclass of std::exception" << std::endl;
5738  }
5739  Teuchos::reduceAll<int, int> (*comm, Teuchos::REDUCE_MIN,
5740  lclSuccess, Teuchos::outArg (gblSuccess));
5741  if (gblSuccess != 1) {
5742  Tpetra::Details::gathervPrint (std::cerr, errStrm.str (), *comm);
5743  }
5744  TEUCHOS_TEST_FOR_EXCEPTION(gblSuccess != 1, std::runtime_error, "Map constructor failed!");
5745 
5746  if (err.is_null ()) {
5747  err->popTab ();
5748  }
5749  if (debug) {
5750  std::ostringstream os;
5751  os << myRank << ": readMap: done" << endl;
5752  *err << os.str ();
5753  }
5754  if (err.is_null ()) {
5755  err->popTab ();
5756  }
5757  return newMap;
5758  }
5759 
5760 #ifdef TPETRA_ENABLE_DEPRECATED_CODE
5761  static Teuchos::RCP<const map_type> TPETRA_DEPRECATED
5764  readMap (std::istream& in,
5765  const Teuchos::RCP<const comm_type>& comm,
5766  const Teuchos::RCP<node_type>& /* pNode */,
5767  const Teuchos::RCP<Teuchos::FancyOStream>& err,
5768  const bool tolerant=false,
5769  const bool debug=false)
5770  {
5771  return readMap (in, comm, err, tolerant, debug);
5772  }
5773 #endif // TPETRA_ENABLE_DEPRECATED_CODE
5774 
5775  private:
5776 
5787  static int
5788  encodeDataType (const std::string& dataType)
5789  {
5790  if (dataType == "real" || dataType == "integer") {
5791  return 0;
5792  } else if (dataType == "complex") {
5793  return 1;
5794  } else if (dataType == "pattern") {
5795  return 2;
5796  } else {
5797  // We should never get here, since Banner validates the
5798  // reported data type and ensures it is one of the accepted
5799  // values.
5800  TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,
5801  "Unrecognized Matrix Market data type \"" << dataType
5802  << "\". We should never get here. "
5803  "Please report this bug to the Tpetra developers.");
5804  }
5805  }
5806  };
5807 
5836  template<class SparseMatrixType>
5837  class Writer {
5838  public:
5840  typedef SparseMatrixType sparse_matrix_type;
5841  typedef Teuchos::RCP<sparse_matrix_type> sparse_matrix_ptr;
5842 
5844  typedef typename SparseMatrixType::scalar_type scalar_type;
5846  typedef typename SparseMatrixType::local_ordinal_type local_ordinal_type;
5852  typedef typename SparseMatrixType::global_ordinal_type global_ordinal_type;
5854  typedef typename SparseMatrixType::node_type node_type;
5855 
5857  typedef MultiVector<scalar_type,
5865 
5868 
5900  static void
5901  writeSparseFile (const std::string& filename,
5902  const Teuchos::RCP<const sparse_matrix_type>& pMatrix,
5903  const std::string& matrixName,
5904  const std::string& matrixDescription,
5905  const bool debug=false)
5906  {
5907  TEUCHOS_TEST_FOR_EXCEPTION(
5908  pMatrix.is_null (), std::invalid_argument,
5909  "The input matrix is null.");
5910  Teuchos::RCP<const Teuchos::Comm<int> > comm = pMatrix->getComm ();
5911  TEUCHOS_TEST_FOR_EXCEPTION(
5912  comm.is_null (), std::invalid_argument,
5913  "The input matrix's communicator (Teuchos::Comm object) is null.");
5914  const int myRank = comm->getRank ();
5915  std::ofstream out;
5916 
5917 
5918  // Only open the file on Rank 0.
5919  if (myRank == 0) {
5920  out.open (filename.c_str ());
5921  }
5922  writeSparse (out, pMatrix, matrixName, matrixDescription, debug);
5923  // We can rely on the destructor of the output stream to close
5924  // the file on scope exit, even if writeSparse() throws an
5925  // exception.
5926  }
5927 
5948  static void
5949  writeSparseFile (const std::string& filename,
5950  const Teuchos::RCP<const sparse_matrix_type>& pMatrix,
5951  const bool debug=false)
5952  {
5953  writeSparseFile (filename, pMatrix, "", "", debug);
5954  }
5955 
5986  static void
5987  writeSparse (std::ostream& out,
5988  const Teuchos::RCP<const sparse_matrix_type>& pMatrix,
5989  const std::string& matrixName,
5990  const std::string& matrixDescription,
5991  const bool debug=false)
5992  {
5993  using Teuchos::ArrayView;
5994  using Teuchos::Comm;
5995  using Teuchos::FancyOStream;
5996  using Teuchos::getFancyOStream;
5997  using Teuchos::null;
5998  using Teuchos::RCP;
5999  using Teuchos::rcpFromRef;
6000  using std::cerr;
6001  using std::endl;
6002  typedef scalar_type ST;
6003  typedef local_ordinal_type LO;
6004  typedef global_ordinal_type GO;
6005  typedef typename Teuchos::ScalarTraits<ST> STS;
6006  typedef typename ArrayView<const LO>::const_iterator lo_iter;
6007  typedef typename ArrayView<const GO>::const_iterator go_iter;
6008  typedef typename ArrayView<const ST>::const_iterator st_iter;
6009 
6010  TEUCHOS_TEST_FOR_EXCEPTION(
6011  pMatrix.is_null (), std::invalid_argument,
6012  "The input matrix is null.");
6013 
6014  // Make the output stream write floating-point numbers in
6015  // scientific notation. It will politely put the output
6016  // stream back to its state on input, when this scope
6017  // terminates.
6018  Teuchos::SetScientific<ST> sci (out);
6019 
6020  // Get the matrix's communicator.
6021  RCP<const Comm<int> > comm = pMatrix->getComm ();
6022  TEUCHOS_TEST_FOR_EXCEPTION(
6023  comm.is_null (), std::invalid_argument,
6024  "The input matrix's communicator (Teuchos::Comm object) is null.");
6025  const int myRank = comm->getRank ();
6026 
6027  // Optionally, make a stream for debugging output.
6028  RCP<FancyOStream> err =
6029  debug ? getFancyOStream (rcpFromRef (std::cerr)) : null;
6030  if (debug) {
6031  std::ostringstream os;
6032  os << myRank << ": writeSparse" << endl;
6033  *err << os.str ();
6034  comm->barrier ();
6035  os << "-- " << myRank << ": past barrier" << endl;
6036  *err << os.str ();
6037  }
6038 
6039  // Whether to print debugging output to stderr.
6040  const bool debugPrint = debug && myRank == 0;
6041 
6042  RCP<const map_type> rowMap = pMatrix->getRowMap ();
6043  RCP<const map_type> colMap = pMatrix->getColMap ();
6044  RCP<const map_type> domainMap = pMatrix->getDomainMap ();
6045  RCP<const map_type> rangeMap = pMatrix->getRangeMap ();
6046 
6047  const global_size_t numRows = rangeMap->getGlobalNumElements ();
6048  const global_size_t numCols = domainMap->getGlobalNumElements ();
6049 
6050  if (debug && myRank == 0) {
6051  std::ostringstream os;
6052  os << "-- Input sparse matrix is:"
6053  << "---- " << numRows << " x " << numCols << endl
6054  << "---- "
6055  << (pMatrix->isGloballyIndexed() ? "Globally" : "Locally")
6056  << " indexed." << endl
6057  << "---- Its row map has " << rowMap->getGlobalNumElements ()
6058  << " elements." << endl
6059  << "---- Its col map has " << colMap->getGlobalNumElements ()
6060  << " elements." << endl;
6061  *err << os.str ();
6062  }
6063  // Make the "gather" row map, where Proc 0 owns all rows and
6064  // the other procs own no rows.
6065  const size_t localNumRows = (myRank == 0) ? numRows : 0;
6066  if (debug) {
6067  std::ostringstream os;
6068  os << "-- " << myRank << ": making gatherRowMap" << endl;
6069  *err << os.str ();
6070  }
6071  RCP<const map_type> gatherRowMap =
6072  Details::computeGatherMap (rowMap, err, debug);
6073 
6074  // Since the matrix may in general be non-square, we need to
6075  // make a column map as well. In this case, the column map
6076  // contains all the columns of the original matrix, because we
6077  // are gathering the whole matrix onto Proc 0. We call
6078  // computeGatherMap to preserve the original order of column
6079  // indices over all the processes.
6080  const size_t localNumCols = (myRank == 0) ? numCols : 0;
6081  RCP<const map_type> gatherColMap =
6082  Details::computeGatherMap (colMap, err, debug);
6083 
6084  // Current map is the source map, gather map is the target map.
6085  typedef Import<LO, GO, node_type> import_type;
6086  import_type importer (rowMap, gatherRowMap);
6087 
6088  // Create a new CrsMatrix to hold the result of the import.
6089  // The constructor needs a column map as well as a row map,
6090  // for the case that the matrix is not square.
6091  RCP<sparse_matrix_type> newMatrix =
6092  rcp (new sparse_matrix_type (gatherRowMap, gatherColMap,
6093  static_cast<size_t> (0)));
6094  // Import the sparse matrix onto Proc 0.
6095  newMatrix->doImport (*pMatrix, importer, INSERT);
6096 
6097  // fillComplete() needs the domain and range maps for the case
6098  // that the matrix is not square.
6099  {
6100  RCP<const map_type> gatherDomainMap =
6101  rcp (new map_type (numCols, localNumCols,
6102  domainMap->getIndexBase (),
6103  comm));
6104  RCP<const map_type> gatherRangeMap =
6105  rcp (new map_type (numRows, localNumRows,
6106  rangeMap->getIndexBase (),
6107  comm));
6108  newMatrix->fillComplete (gatherDomainMap, gatherRangeMap);
6109  }
6110 
6111  if (debugPrint) {
6112  cerr << "-- Output sparse matrix is:"
6113  << "---- " << newMatrix->getRangeMap ()->getGlobalNumElements ()
6114  << " x "
6115  << newMatrix->getDomainMap ()->getGlobalNumElements ()
6116  << " with "
6117  << newMatrix->getGlobalNumEntries () << " entries;" << endl
6118  << "---- "
6119  << (newMatrix->isGloballyIndexed () ? "Globally" : "Locally")
6120  << " indexed." << endl
6121  << "---- Its row map has "
6122  << newMatrix->getRowMap ()->getGlobalNumElements ()
6123  << " elements, with index base "
6124  << newMatrix->getRowMap ()->getIndexBase () << "." << endl
6125  << "---- Its col map has "
6126  << newMatrix->getColMap ()->getGlobalNumElements ()
6127  << " elements, with index base "
6128  << newMatrix->getColMap ()->getIndexBase () << "." << endl
6129  << "---- Element count of output matrix's column Map may differ "
6130  << "from that of the input matrix's column Map, if some columns "
6131  << "of the matrix contain no entries." << endl;
6132  }
6133 
6134  //
6135  // Print the metadata and the matrix entries on Rank 0.
6136  //
6137  if (myRank == 0) {
6138  // Print the Matrix Market banner line. CrsMatrix stores
6139  // data nonsymmetrically ("general"). This implies that
6140  // readSparse() on a symmetrically stored input file,
6141  // followed by writeSparse() on the resulting sparse matrix,
6142  // will result in an output file with a different banner
6143  // line than the original input file.
6144  out << "%%MatrixMarket matrix coordinate "
6145  << (STS::isComplex ? "complex" : "real")
6146  << " general" << endl;
6147 
6148  // Print comments (the matrix name and / or description).
6149  if (matrixName != "") {
6150  printAsComment (out, matrixName);
6151  }
6152  if (matrixDescription != "") {
6153  printAsComment (out, matrixDescription);
6154  }
6155 
6156  // Print the Matrix Market header (# rows, # columns, #
6157  // nonzeros). Use the range resp. domain map for the number
6158  // of rows resp. columns, since Tpetra::CrsMatrix uses the
6159  // column map for the number of columns. That only
6160  // corresponds to the "linear-algebraic" number of columns
6161  // when the column map is uniquely owned (a.k.a. one-to-one),
6162  // which only happens if the matrix is (block) diagonal.
6163  out << newMatrix->getRangeMap ()->getGlobalNumElements () << " "
6164  << newMatrix->getDomainMap ()->getGlobalNumElements () << " "
6165  << newMatrix->getGlobalNumEntries () << endl;
6166 
6167  // The Matrix Market format expects one-based row and column
6168  // indices. We'll convert the indices on output from
6169  // whatever index base they use to one-based indices.
6170  const GO rowIndexBase = gatherRowMap->getIndexBase ();
6171  const GO colIndexBase = newMatrix->getColMap()->getIndexBase ();
6172  //
6173  // Print the entries of the matrix.
6174  //
6175  // newMatrix can never be globally indexed, since we called
6176  // fillComplete() on it. We include code for both cases
6177  // (globally or locally indexed) just in case that ever
6178  // changes.
6179  if (newMatrix->isGloballyIndexed()) {
6180  // We know that the "gather" row Map is contiguous, so we
6181  // don't need to get the list of GIDs.
6182  const GO minAllGlobalIndex = gatherRowMap->getMinAllGlobalIndex ();
6183  const GO maxAllGlobalIndex = gatherRowMap->getMaxAllGlobalIndex ();
6184  for (GO globalRowIndex = minAllGlobalIndex;
6185  globalRowIndex <= maxAllGlobalIndex; // inclusive range
6186  ++globalRowIndex) {
6187  ArrayView<const GO> ind;
6188  ArrayView<const ST> val;
6189  newMatrix->getGlobalRowView (globalRowIndex, ind, val);
6190  go_iter indIter = ind.begin ();
6191  st_iter valIter = val.begin ();
6192  for (; indIter != ind.end() && valIter != val.end();
6193  ++indIter, ++valIter) {
6194  const GO globalColIndex = *indIter;
6195  // Convert row and column indices to 1-based.
6196  // This works because the global index type is signed.
6197  out << (globalRowIndex + 1 - rowIndexBase) << " "
6198  << (globalColIndex + 1 - colIndexBase) << " ";
6199  if (STS::isComplex) {
6200  out << STS::real (*valIter) << " " << STS::imag (*valIter);
6201  } else {
6202  out << *valIter;
6203  }
6204  out << endl;
6205  } // For each entry in the current row
6206  } // For each row of the "gather" matrix
6207  } else { // newMatrix is locally indexed
6208  typedef Teuchos::OrdinalTraits<GO> OTG;
6209  for (LO localRowIndex = gatherRowMap->getMinLocalIndex();
6210  localRowIndex <= gatherRowMap->getMaxLocalIndex();
6211  ++localRowIndex) {
6212  // Convert from local to global row index.
6213  const GO globalRowIndex =
6214  gatherRowMap->getGlobalElement (localRowIndex);
6215  TEUCHOS_TEST_FOR_EXCEPTION(
6216  globalRowIndex == OTG::invalid(), std::logic_error,
6217  "Failed to convert the supposed local row index "
6218  << localRowIndex << " into a global row index. "
6219  "Please report this bug to the Tpetra developers.");
6220  ArrayView<const LO> ind;
6221  ArrayView<const ST> val;
6222  newMatrix->getLocalRowView (localRowIndex, ind, val);
6223  lo_iter indIter = ind.begin ();
6224  st_iter valIter = val.begin ();
6225  for (; indIter != ind.end() && valIter != val.end();
6226  ++indIter, ++valIter) {
6227  // Convert the column index from local to global.
6228  const GO globalColIndex =
6229  newMatrix->getColMap()->getGlobalElement (*indIter);
6230  TEUCHOS_TEST_FOR_EXCEPTION(
6231  globalColIndex == OTG::invalid(), std::logic_error,
6232  "On local row " << localRowIndex << " of the sparse matrix: "
6233  "Failed to convert the supposed local column index "
6234  << *indIter << " into a global column index. Please report "
6235  "this bug to the Tpetra developers.");
6236  // Convert row and column indices to 1-based.
6237  // This works because the global index type is signed.
6238  out << (globalRowIndex + 1 - rowIndexBase) << " "
6239  << (globalColIndex + 1 - colIndexBase) << " ";
6240  if (STS::isComplex) {
6241  out << STS::real (*valIter) << " " << STS::imag (*valIter);
6242  } else {
6243  out << *valIter;
6244  }
6245  out << endl;
6246  } // For each entry in the current row
6247  } // For each row of the "gather" matrix
6248  } // Whether the "gather" matrix is locally or globally indexed
6249  } // If my process' rank is 0
6250  }
6251 
6252 
6283  static void
6284  writeSparseGraph (std::ostream& out,
6285  const crs_graph_type& graph,
6286  const std::string& graphName,
6287  const std::string& graphDescription,
6288  const bool debug=false)
6289  {
6290  using Teuchos::ArrayView;
6291  using Teuchos::Comm;
6292  using Teuchos::FancyOStream;
6293  using Teuchos::getFancyOStream;
6294  using Teuchos::null;
6295  using Teuchos::RCP;
6296  using Teuchos::rcpFromRef;
6297  using std::cerr;
6298  using std::endl;
6299  typedef local_ordinal_type LO;
6300  typedef global_ordinal_type GO;
6301 
6302  // Get the graph's communicator. Processes on which the
6303  // graph's Map or communicator is null don't participate in
6304  // this operation. This function shouldn't even be called on
6305  // those processes.
6306  auto rowMap = graph.getRowMap ();
6307  if (rowMap.is_null ()) {
6308  return;
6309  }
6310  auto comm = rowMap->getComm ();
6311  if (comm.is_null ()) {
6312  return;
6313  }
6314  const int myRank = comm->getRank ();
6315 
6316  // Optionally, make a stream for debugging output.
6317  RCP<FancyOStream> err =
6318  debug ? getFancyOStream (rcpFromRef (std::cerr)) : null;
6319  if (debug) {
6320  std::ostringstream os;
6321  os << myRank << ": writeSparseGraph" << endl;
6322  *err << os.str ();
6323  comm->barrier ();
6324  os << "-- " << myRank << ": past barrier" << endl;
6325  *err << os.str ();
6326  }
6327 
6328  // Whether to print debugging output to stderr.
6329  const bool debugPrint = debug && myRank == 0;
6330 
6331  // We've already gotten the rowMap above.
6332  auto colMap = graph.getColMap ();
6333  auto domainMap = graph.getDomainMap ();
6334  auto rangeMap = graph.getRangeMap ();
6335 
6336  const global_size_t numRows = rangeMap->getGlobalNumElements ();
6337  const global_size_t numCols = domainMap->getGlobalNumElements ();
6338 
6339  if (debug && myRank == 0) {
6340  std::ostringstream os;
6341  os << "-- Input sparse graph is:"
6342  << "---- " << numRows << " x " << numCols << " with "
6343  << graph.getGlobalNumEntries () << " entries;" << endl
6344  << "---- "
6345  << (graph.isGloballyIndexed () ? "Globally" : "Locally")
6346  << " indexed." << endl
6347  << "---- Its row Map has " << rowMap->getGlobalNumElements ()
6348  << " elements." << endl
6349  << "---- Its col Map has " << colMap->getGlobalNumElements ()
6350  << " elements." << endl;
6351  *err << os.str ();
6352  }
6353  // Make the "gather" row map, where Proc 0 owns all rows and
6354  // the other procs own no rows.
6355  const size_t localNumRows = (myRank == 0) ? numRows : 0;
6356  if (debug) {
6357  std::ostringstream os;
6358  os << "-- " << myRank << ": making gatherRowMap" << endl;
6359  *err << os.str ();
6360  }
6361  auto gatherRowMap = Details::computeGatherMap (rowMap, err, debug);
6362 
6363  // Since the graph may in general be non-square, we need to
6364  // make a column map as well. In this case, the column map
6365  // contains all the columns of the original graph, because we
6366  // are gathering the whole graph onto Proc 0. We call
6367  // computeGatherMap to preserve the original order of column
6368  // indices over all the processes.
6369  const size_t localNumCols = (myRank == 0) ? numCols : 0;
6370  auto gatherColMap = Details::computeGatherMap (colMap, err, debug);
6371 
6372  // Current map is the source map, gather map is the target map.
6373  Import<LO, GO, node_type> importer (rowMap, gatherRowMap);
6374 
6375  // Create a new CrsGraph to hold the result of the import.
6376  // The constructor needs a column map as well as a row map,
6377  // for the case that the graph is not square.
6378  crs_graph_type newGraph (gatherRowMap, gatherColMap,
6379  static_cast<size_t> (0));
6380  // Import the sparse graph onto Proc 0.
6381  newGraph.doImport (graph, importer, INSERT);
6382 
6383  // fillComplete() needs the domain and range maps for the case
6384  // that the graph is not square.
6385  {
6386  RCP<const map_type> gatherDomainMap =
6387  rcp (new map_type (numCols, localNumCols,
6388  domainMap->getIndexBase (),
6389  comm));
6390  RCP<const map_type> gatherRangeMap =
6391  rcp (new map_type (numRows, localNumRows,
6392  rangeMap->getIndexBase (),
6393  comm));
6394  newGraph.fillComplete (gatherDomainMap, gatherRangeMap);
6395  }
6396 
6397  if (debugPrint) {
6398  cerr << "-- Output sparse graph is:"
6399  << "---- " << newGraph.getRangeMap ()->getGlobalNumElements ()
6400  << " x "
6401  << newGraph.getDomainMap ()->getGlobalNumElements ()
6402  << " with "
6403  << newGraph.getGlobalNumEntries () << " entries;" << endl
6404  << "---- "
6405  << (newGraph.isGloballyIndexed () ? "Globally" : "Locally")
6406  << " indexed." << endl
6407  << "---- Its row map has "
6408  << newGraph.getRowMap ()->getGlobalNumElements ()
6409  << " elements, with index base "
6410  << newGraph.getRowMap ()->getIndexBase () << "." << endl
6411  << "---- Its col map has "
6412  << newGraph.getColMap ()->getGlobalNumElements ()
6413  << " elements, with index base "
6414  << newGraph.getColMap ()->getIndexBase () << "." << endl
6415  << "---- Element count of output graph's column Map may differ "
6416  << "from that of the input matrix's column Map, if some columns "
6417  << "of the matrix contain no entries." << endl;
6418  }
6419 
6420  //
6421  // Print the metadata and the graph entries on Process 0 of
6422  // the graph's communicator.
6423  //
6424  if (myRank == 0) {
6425  // Print the Matrix Market banner line. CrsGraph stores
6426  // data nonsymmetrically ("general"). This implies that
6427  // readSparseGraph() on a symmetrically stored input file,
6428  // followed by writeSparseGraph() on the resulting sparse
6429  // graph, will result in an output file with a different
6430  // banner line than the original input file.
6431  out << "%%MatrixMarket matrix coordinate pattern general" << endl;
6432 
6433  // Print comments (the graph name and / or description).
6434  if (graphName != "") {
6435  printAsComment (out, graphName);
6436  }
6437  if (graphDescription != "") {
6438  printAsComment (out, graphDescription);
6439  }
6440 
6441  // Print the Matrix Market header (# rows, # columns, #
6442  // stored entries). Use the range resp. domain map for the
6443  // number of rows resp. columns, since Tpetra::CrsGraph uses
6444  // the column map for the number of columns. That only
6445  // corresponds to the "linear-algebraic" number of columns
6446  // when the column map is uniquely owned
6447  // (a.k.a. one-to-one), which only happens if the graph is
6448  // block diagonal (one block per process).
6449  out << newGraph.getRangeMap ()->getGlobalNumElements () << " "
6450  << newGraph.getDomainMap ()->getGlobalNumElements () << " "
6451  << newGraph.getGlobalNumEntries () << endl;
6452 
6453  // The Matrix Market format expects one-based row and column
6454  // indices. We'll convert the indices on output from
6455  // whatever index base they use to one-based indices.
6456  const GO rowIndexBase = gatherRowMap->getIndexBase ();
6457  const GO colIndexBase = newGraph.getColMap()->getIndexBase ();
6458  //
6459  // Print the entries of the graph.
6460  //
6461  // newGraph can never be globally indexed, since we called
6462  // fillComplete() on it. We include code for both cases
6463  // (globally or locally indexed) just in case that ever
6464  // changes.
6465  if (newGraph.isGloballyIndexed ()) {
6466  // We know that the "gather" row Map is contiguous, so we
6467  // don't need to get the list of GIDs.
6468  const GO minAllGlobalIndex = gatherRowMap->getMinAllGlobalIndex ();
6469  const GO maxAllGlobalIndex = gatherRowMap->getMaxAllGlobalIndex ();
6470  for (GO globalRowIndex = minAllGlobalIndex;
6471  globalRowIndex <= maxAllGlobalIndex; // inclusive range
6472  ++globalRowIndex) {
6473  ArrayView<const GO> ind;
6474  newGraph.getGlobalRowView (globalRowIndex, ind);
6475  for (auto indIter = ind.begin (); indIter != ind.end (); ++indIter) {
6476  const GO globalColIndex = *indIter;
6477  // Convert row and column indices to 1-based.
6478  // This works because the global index type is signed.
6479  out << (globalRowIndex + 1 - rowIndexBase) << " "
6480  << (globalColIndex + 1 - colIndexBase) << " ";
6481  out << endl;
6482  } // For each entry in the current row
6483  } // For each row of the "gather" graph
6484  }
6485  else { // newGraph is locally indexed
6486  typedef Teuchos::OrdinalTraits<GO> OTG;
6487  for (LO localRowIndex = gatherRowMap->getMinLocalIndex ();
6488  localRowIndex <= gatherRowMap->getMaxLocalIndex ();
6489  ++localRowIndex) {
6490  // Convert from local to global row index.
6491  const GO globalRowIndex =
6492  gatherRowMap->getGlobalElement (localRowIndex);
6493  TEUCHOS_TEST_FOR_EXCEPTION
6494  (globalRowIndex == OTG::invalid (), std::logic_error, "Failed "
6495  "to convert the supposed local row index " << localRowIndex <<
6496  " into a global row index. Please report this bug to the "
6497  "Tpetra developers.");
6498  ArrayView<const LO> ind;
6499  newGraph.getLocalRowView (localRowIndex, ind);
6500  for (auto indIter = ind.begin (); indIter != ind.end (); ++indIter) {
6501  // Convert the column index from local to global.
6502  const GO globalColIndex =
6503  newGraph.getColMap ()->getGlobalElement (*indIter);
6504  TEUCHOS_TEST_FOR_EXCEPTION(
6505  globalColIndex == OTG::invalid(), std::logic_error,
6506  "On local row " << localRowIndex << " of the sparse graph: "
6507  "Failed to convert the supposed local column index "
6508  << *indIter << " into a global column index. Please report "
6509  "this bug to the Tpetra developers.");
6510  // Convert row and column indices to 1-based.
6511  // This works because the global index type is signed.
6512  out << (globalRowIndex + 1 - rowIndexBase) << " "
6513  << (globalColIndex + 1 - colIndexBase) << " ";
6514  out << endl;
6515  } // For each entry in the current row
6516  } // For each row of the "gather" graph
6517  } // Whether the "gather" graph is locally or globally indexed
6518  } // If my process' rank is 0
6519  }
6520 
6526  static void
6527  writeSparseGraph (std::ostream& out,
6528  const crs_graph_type& graph,
6529  const bool debug=false)
6530  {
6531  writeSparseGraph (out, graph, "", "", debug);
6532  }
6533 
6568  static void
6569  writeSparseGraphFile (const std::string& filename,
6570  const crs_graph_type& graph,
6571  const std::string& graphName,
6572  const std::string& graphDescription,
6573  const bool debug=false)
6574  {
6575  auto comm = graph.getComm ();
6576  if (comm.is_null ()) {
6577  // Processes on which the communicator is null shouldn't
6578  // even call this function. The convention is that
6579  // processes on which the object's communicator is null do
6580  // not participate in collective operations involving the
6581  // object.
6582  return;
6583  }
6584  const int myRank = comm->getRank ();
6585  std::ofstream out;
6586 
6587  // Only open the file on Process 0.
6588  if (myRank == 0) {
6589  out.open (filename.c_str ());
6590  }
6591  writeSparseGraph (out, graph, graphName, graphDescription, debug);
6592  // We can rely on the destructor of the output stream to close
6593  // the file on scope exit, even if writeSparseGraph() throws
6594  // an exception.
6595  }
6596 
6601  static void
6602  writeSparseGraphFile (const std::string& filename,
6603  const crs_graph_type& graph,
6604  const bool debug=false)
6605  {
6606  writeSparseGraphFile (filename, graph, "", "", debug);
6607  }
6608 
6617  static void
6618  writeSparseGraphFile (const std::string& filename,
6619  const Teuchos::RCP<const crs_graph_type>& pGraph,
6620  const std::string& graphName,
6621  const std::string& graphDescription,
6622  const bool debug=false)
6623  {
6624  writeSparseGraphFile (filename, *pGraph, graphName, graphDescription, debug);
6625  }
6626 
6636  static void
6637  writeSparseGraphFile (const std::string& filename,
6638  const Teuchos::RCP<const crs_graph_type>& pGraph,
6639  const bool debug=false)
6640  {
6641  writeSparseGraphFile (filename, *pGraph, "", "", debug);
6642  }
6643 
6666  static void
6667  writeSparse (std::ostream& out,
6668  const Teuchos::RCP<const sparse_matrix_type>& pMatrix,
6669  const bool debug=false)
6670  {
6671  writeSparse (out, pMatrix, "", "", debug);
6672  }
6673 
6702  static void
6703  writeDenseFile (const std::string& filename,
6704  const multivector_type& X,
6705  const std::string& matrixName,
6706  const std::string& matrixDescription,
6707  const Teuchos::RCP<Teuchos::FancyOStream>& err = Teuchos::null,
6708  const Teuchos::RCP<Teuchos::FancyOStream>& dbg = Teuchos::null)
6709  {
6710  const int myRank = X.getMap ().is_null () ? 0 :
6711  (X.getMap ()->getComm ().is_null () ? 0 :
6712  X.getMap ()->getComm ()->getRank ());
6713  std::ofstream out;
6714 
6715  if (myRank == 0) { // Only open the file on Process 0.
6716  out.open (filename.c_str());
6717  }
6718 
6719  writeDense (out, X, matrixName, matrixDescription, err, dbg);
6720  // We can rely on the destructor of the output stream to close
6721  // the file on scope exit, even if writeDense() throws an
6722  // exception.
6723  }
6724 
6730  static void
6731  writeDenseFile (const std::string& filename,
6732  const Teuchos::RCP<const multivector_type>& X,
6733  const std::string& matrixName,
6734  const std::string& matrixDescription,
6735  const Teuchos::RCP<Teuchos::FancyOStream>& err = Teuchos::null,
6736  const Teuchos::RCP<Teuchos::FancyOStream>& dbg = Teuchos::null)
6737  {
6738  TEUCHOS_TEST_FOR_EXCEPTION(
6739  X.is_null (), std::invalid_argument, "Tpetra::MatrixMarket::"
6740  "writeDenseFile: The input MultiVector X is null.");
6741  writeDenseFile (filename, *X, matrixName, matrixDescription, err, dbg);
6742  }
6743 
6749  static void
6750  writeDenseFile (const std::string& filename,
6751  const multivector_type& X,
6752  const Teuchos::RCP<Teuchos::FancyOStream>& err = Teuchos::null,
6753  const Teuchos::RCP<Teuchos::FancyOStream>& dbg = Teuchos::null)
6754  {
6755  writeDenseFile (filename, X, "", "", err, dbg);
6756  }
6757 
6763  static void
6764  writeDenseFile (const std::string& filename,
6765  const Teuchos::RCP<const multivector_type>& X,
6766  const Teuchos::RCP<Teuchos::FancyOStream>& err = Teuchos::null,
6767  const Teuchos::RCP<Teuchos::FancyOStream>& dbg = Teuchos::null)
6768  {
6769  TEUCHOS_TEST_FOR_EXCEPTION(
6770  X.is_null (), std::invalid_argument, "Tpetra::MatrixMarket::"
6771  "writeDenseFile: The input MultiVector X is null.");
6772  writeDenseFile (filename, *X, err, dbg);
6773  }
6774 
6775 
6806  static void
6807  writeDense (std::ostream& out,
6808  const multivector_type& X,
6809  const std::string& matrixName,
6810  const std::string& matrixDescription,
6811  const Teuchos::RCP<Teuchos::FancyOStream>& err = Teuchos::null,
6812  const Teuchos::RCP<Teuchos::FancyOStream>& dbg = Teuchos::null)
6813  {
6814  using Teuchos::Comm;
6815  using Teuchos::outArg;
6816  using Teuchos::REDUCE_MAX;
6817  using Teuchos::reduceAll;
6818  using Teuchos::RCP;
6819  using std::endl;
6820 
6821  RCP<const Comm<int> > comm = X.getMap ().is_null () ?
6822  Teuchos::null : X.getMap ()->getComm ();
6823  const int myRank = comm.is_null () ? 0 : comm->getRank ();
6824 
6825  // If the caller provides a nonnull debug output stream, we
6826  // print debugging output to it. This is a local thing; we
6827  // don't have to check across processes.
6828  const bool debug = ! dbg.is_null ();
6829  if (debug) {
6830  dbg->pushTab ();
6831  std::ostringstream os;
6832  os << myRank << ": writeDense" << endl;
6833  *dbg << os.str ();
6834  dbg->pushTab ();
6835  }
6836  // Print the Matrix Market header.
6837  writeDenseHeader (out, X, matrixName, matrixDescription, err, dbg);
6838 
6839  // Print each column one at a time. This is a (perhaps)
6840  // temporary fix for Bug 6288.
6841  const size_t numVecs = X.getNumVectors ();
6842  for (size_t j = 0; j < numVecs; ++j) {
6843  writeDenseColumn (out, * (X.getVector (j)), err, dbg);
6844  }
6845 
6846  if (debug) {
6847  dbg->popTab ();
6848  std::ostringstream os;
6849  os << myRank << ": writeDense: Done" << endl;
6850  *dbg << os.str ();
6851  dbg->popTab ();
6852  }
6853  }
6854 
6855  private:
6856 
6882  static void
6883  writeDenseHeader (std::ostream& out,
6884  const multivector_type& X,
6885  const std::string& matrixName,
6886  const std::string& matrixDescription,
6887  const Teuchos::RCP<Teuchos::FancyOStream>& err = Teuchos::null,
6888  const Teuchos::RCP<Teuchos::FancyOStream>& dbg = Teuchos::null)
6889  {
6890  using Teuchos::Comm;
6891  using Teuchos::outArg;
6892  using Teuchos::RCP;
6893  using Teuchos::REDUCE_MAX;
6894  using Teuchos::reduceAll;
6895  using std::endl;
6896  typedef Teuchos::ScalarTraits<scalar_type> STS;
6897  const char prefix[] = "Tpetra::MatrixMarket::writeDenseHeader: ";
6898 
6899  RCP<const Comm<int> > comm = X.getMap ().is_null () ?
6900  Teuchos::null : X.getMap ()->getComm ();
6901  const int myRank = comm.is_null () ? 0 : comm->getRank ();
6902  int lclErr = 0; // whether this MPI process has seen an error
6903  int gblErr = 0; // whether we know if some MPI process has seen an error
6904 
6905  // If the caller provides a nonnull debug output stream, we
6906  // print debugging output to it. This is a local thing; we
6907  // don't have to check across processes.
6908  const bool debug = ! dbg.is_null ();
6909 
6910  if (debug) {
6911  dbg->pushTab ();
6912  std::ostringstream os;
6913  os << myRank << ": writeDenseHeader" << endl;
6914  *dbg << os.str ();
6915  dbg->pushTab ();
6916  }
6917 
6918  //
6919  // Process 0: Write the MatrixMarket header.
6920  //
6921  if (myRank == 0) {
6922  try {
6923  // Print the Matrix Market header. MultiVector stores data
6924  // nonsymmetrically, hence "general" in the banner line.
6925  // Print first to a temporary string output stream, and then
6926  // write it to the main output stream, so that at least the
6927  // header output has transactional semantics. We can't
6928  // guarantee transactional semantics for the whole output,
6929  // since that would not be memory scalable. (This could be
6930  // done in the file system by using a temporary file; we
6931  // don't do this, but users could.)
6932  std::ostringstream hdr;
6933  {
6934  std::string dataType;
6935  if (STS::isComplex) {
6936  dataType = "complex";
6937  } else if (STS::isOrdinal) {
6938  dataType = "integer";
6939  } else {
6940  dataType = "real";
6941  }
6942  hdr << "%%MatrixMarket matrix array " << dataType << " general"
6943  << endl;
6944  }
6945 
6946  // Print comments (the matrix name and / or description).
6947  if (matrixName != "") {
6948  printAsComment (hdr, matrixName);
6949  }
6950  if (matrixDescription != "") {
6951  printAsComment (hdr, matrixDescription);
6952  }
6953  // Print the Matrix Market dimensions header for dense matrices.
6954  hdr << X.getGlobalLength () << " " << X.getNumVectors () << endl;
6955 
6956  // Write the MatrixMarket header to the output stream.
6957  out << hdr.str ();
6958  } catch (std::exception& e) {
6959  if (! err.is_null ()) {
6960  *err << prefix << "While writing the Matrix Market header, "
6961  "Process 0 threw an exception: " << e.what () << endl;
6962  }
6963  lclErr = 1;
6964  }
6965  } // if I am Process 0
6966 
6967  // Establish global agreement on the error state. It wouldn't
6968  // be good for other processes to keep going, if Process 0
6969  // finds out that it can't write to the given output stream.
6970  reduceAll<int, int> (*comm, REDUCE_MAX, lclErr, outArg (gblErr));
6971  TEUCHOS_TEST_FOR_EXCEPTION(
6972  gblErr == 1, std::runtime_error, prefix << "Some error occurred "
6973  "which prevented this method from completing.");
6974 
6975  if (debug) {
6976  dbg->popTab ();
6977  *dbg << myRank << ": writeDenseHeader: Done" << endl;
6978  dbg->popTab ();
6979  }
6980  }
6981 
6999  static void
7000  writeDenseColumn (std::ostream& out,
7001  const multivector_type& X,
7002  const Teuchos::RCP<Teuchos::FancyOStream>& err = Teuchos::null,
7003  const Teuchos::RCP<Teuchos::FancyOStream>& dbg = Teuchos::null)
7004  {
7005  using Teuchos::arcp;
7006  using Teuchos::Array;
7007  using Teuchos::ArrayRCP;
7008  using Teuchos::ArrayView;
7009  using Teuchos::Comm;
7010  using Teuchos::CommRequest;
7011  using Teuchos::ireceive;
7012  using Teuchos::isend;
7013  using Teuchos::outArg;
7014  using Teuchos::REDUCE_MAX;
7015  using Teuchos::reduceAll;
7016  using Teuchos::RCP;
7017  using Teuchos::TypeNameTraits;
7018  using Teuchos::wait;
7019  using std::endl;
7020  typedef Teuchos::ScalarTraits<scalar_type> STS;
7021 
7022  const Comm<int>& comm = * (X.getMap ()->getComm ());
7023  const int myRank = comm.getRank ();
7024  const int numProcs = comm.getSize ();
7025  int lclErr = 0; // whether this MPI process has seen an error
7026  int gblErr = 0; // whether we know if some MPI process has seen an error
7027 
7028  // If the caller provides a nonnull debug output stream, we
7029  // print debugging output to it. This is a local thing; we
7030  // don't have to check across processes.
7031  const bool debug = ! dbg.is_null ();
7032 
7033  if (debug) {
7034  dbg->pushTab ();
7035  std::ostringstream os;
7036  os << myRank << ": writeDenseColumn" << endl;
7037  *dbg << os.str ();
7038  dbg->pushTab ();
7039  }
7040 
7041  // Make the output stream write floating-point numbers in
7042  // scientific notation. It will politely put the output
7043  // stream back to its state on input, when this scope
7044  // terminates.
7045  Teuchos::SetScientific<scalar_type> sci (out);
7046 
7047  const size_t myNumRows = X.getLocalLength ();
7048  const size_t numCols = X.getNumVectors ();
7049  // Use a different tag for the "size" messages than for the
7050  // "data" messages, in order to help us debug any mix-ups.
7051  const int sizeTag = 1337;
7052  const int dataTag = 1338;
7053 
7054  // Process 0 pipelines nonblocking receives with file output.
7055  //
7056  // Constraints:
7057  // - Process 0 can't post a receive for another process'
7058  // actual data, until it posts and waits on the receive
7059  // from that process with the amount of data to receive.
7060  // (We could just post receives with a max data size, but
7061  // I feel uncomfortable about that.)
7062  // - The C++ standard library doesn't allow nonblocking
7063  // output to an std::ostream. (Thus, we have to start a
7064  // receive or send before starting the write, and hope
7065  // that MPI completes it in the background.)
7066  //
7067  // Process 0: Post receive-size receives from Processes 1 and 2.
7068  // Process 1: Post send-size send to Process 0.
7069  // Process 2: Post send-size send to Process 0.
7070  //
7071  // All processes: Pack my entries.
7072  //
7073  // Process 1:
7074  // - Post send-data send to Process 0.
7075  // - Wait on my send-size send to Process 0.
7076  //
7077  // Process 0:
7078  // - Print MatrixMarket header.
7079  // - Print my entries.
7080  // - Wait on receive-size receive from Process 1.
7081  // - Post receive-data receive from Process 1.
7082  //
7083  // For each process p = 1, 2, ... numProcs-1:
7084  // If I am Process 0:
7085  // - Post receive-size receive from Process p + 2
7086  // - Wait on receive-size receive from Process p + 1
7087  // - Post receive-data receive from Process p + 1
7088  // - Wait on receive-data receive from Process p
7089  // - Write data from Process p.
7090  // Else if I am Process p:
7091  // - Wait on my send-data send.
7092  // Else if I am Process p+1:
7093  // - Post send-data send to Process 0.
7094  // - Wait on my send-size send.
7095  // Else if I am Process p+2:
7096  // - Post send-size send to Process 0.
7097  //
7098  // Pipelining has three goals here:
7099  // 1. Overlap communication (the receives) with file I/O
7100  // 2. Give Process 0 a chance to prepost some receives,
7101  // before sends show up, by packing local data before
7102  // posting sends
7103  // 3. Don't post _all_ receives or _all_ sends, because that
7104  // wouldn't be memory scalable. (Just because we can't
7105  // see how much memory MPI consumes, doesn't mean that it
7106  // doesn't consume any!)
7107 
7108  // These are used on every process. sendReqSize[0] holds the
7109  // number of rows on this process, and sendReqBuf holds this
7110  // process' data. Process 0 packs into sendReqBuf, but
7111  // doesn't send; it only uses that for printing. All other
7112  // processes send both of these to Process 0.
7113  RCP<CommRequest<int> > sendReqSize, sendReqData;
7114 
7115  // These are used only on Process 0, for received data. Keep
7116  // 3 of each, and treat the arrays as circular buffers. When
7117  // receiving from Process p, the corresponding array index
7118  // here is p % 3.
7119  Array<ArrayRCP<size_t> > recvSizeBufs (3);
7120  Array<ArrayRCP<scalar_type> > recvDataBufs (3);
7121  Array<RCP<CommRequest<int> > > recvSizeReqs (3);
7122  Array<RCP<CommRequest<int> > > recvDataReqs (3);
7123 
7124  // Buffer for nonblocking send of the "send size."
7125  ArrayRCP<size_t> sendDataSize (1);
7126  sendDataSize[0] = myNumRows;
7127 
7128  if (myRank == 0) {
7129  if (debug) {
7130  std::ostringstream os;
7131  os << myRank << ": Post receive-size receives from "
7132  "Procs 1 and 2: tag = " << sizeTag << endl;
7133  *dbg << os.str ();
7134  }
7135  // Process 0: Post receive-size receives from Processes 1 and 2.
7136  recvSizeBufs[0].resize (1);
7137  // Set these three to an invalid value as a flag. If we
7138  // don't get these messages, then the invalid value will
7139  // remain, so we can test for it.
7140  (recvSizeBufs[0])[0] = Teuchos::OrdinalTraits<size_t>::invalid ();
7141  recvSizeBufs[1].resize (1);
7142  (recvSizeBufs[1])[0] = Teuchos::OrdinalTraits<size_t>::invalid ();
7143  recvSizeBufs[2].resize (1);
7144  (recvSizeBufs[2])[0] = Teuchos::OrdinalTraits<size_t>::invalid ();
7145  if (numProcs > 1) {
7146  recvSizeReqs[1] =
7147  ireceive<int, size_t> (recvSizeBufs[1], 1, sizeTag, comm);
7148  }
7149  if (numProcs > 2) {
7150  recvSizeReqs[2] =
7151  ireceive<int, size_t> (recvSizeBufs[2], 2, sizeTag, comm);
7152  }
7153  }
7154  else if (myRank == 1 || myRank == 2) {
7155  if (debug) {
7156  std::ostringstream os;
7157  os << myRank << ": Post send-size send: size = "
7158  << sendDataSize[0] << ", tag = " << sizeTag << endl;
7159  *dbg << os.str ();
7160  }
7161  // Prime the pipeline by having Processes 1 and 2 start
7162  // their send-size sends. We don't want _all_ the processes
7163  // to start their send-size sends, because that wouldn't be
7164  // memory scalable.
7165  sendReqSize = isend<int, size_t> (sendDataSize, 0, sizeTag, comm);
7166  }
7167  else {
7168  if (debug) {
7169  std::ostringstream os;
7170  os << myRank << ": Not posting my send-size send yet" << endl;
7171  *dbg << os.str ();
7172  }
7173  }
7174 
7175  //
7176  // Pack my entries, in column-major order.
7177  //
7178  if (debug) {
7179  std::ostringstream os;
7180  os << myRank << ": Pack my entries" << endl;
7181  *dbg << os.str ();
7182  }
7183  ArrayRCP<scalar_type> sendDataBuf;
7184  try {
7185  sendDataBuf = arcp<scalar_type> (myNumRows * numCols);
7186  X.get1dCopy (sendDataBuf (), myNumRows);
7187  }
7188  catch (std::exception& e) {
7189  lclErr = 1;
7190  if (! err.is_null ()) {
7191  std::ostringstream os;
7192  os << "Process " << myRank << ": Attempt to pack my MultiVector "
7193  "entries threw an exception: " << e.what () << endl;
7194  *err << os.str ();
7195  }
7196  }
7197  if (debug) {
7198  std::ostringstream os;
7199  os << myRank << ": Done packing my entries" << endl;
7200  *dbg << os.str ();
7201  }
7202 
7203  //
7204  // Process 1: post send-data send to Process 0.
7205  //
7206  if (myRank == 1) {
7207  if (debug) {
7208  *dbg << myRank << ": Post send-data send: tag = " << dataTag
7209  << endl;
7210  }
7211  sendReqData = isend<int, scalar_type> (sendDataBuf, 0, dataTag, comm);
7212  }
7213 
7214  //
7215  // Process 0: Write my entries.
7216  //
7217  if (myRank == 0) {
7218  if (debug) {
7219  std::ostringstream os;
7220  os << myRank << ": Write my entries" << endl;
7221  *dbg << os.str ();
7222  }
7223 
7224  // Write Process 0's data to the output stream.
7225  // Matrix Market prints dense matrices in column-major order.
7226  const size_t printNumRows = myNumRows;
7227  ArrayView<const scalar_type> printData = sendDataBuf ();
7228  const size_t printStride = printNumRows;
7229  if (static_cast<size_t> (printData.size ()) < printStride * numCols) {
7230  lclErr = 1;
7231  if (! err.is_null ()) {
7232  std::ostringstream os;
7233  os << "Process " << myRank << ": My MultiVector data's size "
7234  << printData.size () << " does not match my local dimensions "
7235  << printStride << " x " << numCols << "." << endl;
7236  *err << os.str ();
7237  }
7238  }
7239  else {
7240  // Matrix Market dense format wants one number per line.
7241  // It wants each complex number as two real numbers (real
7242  // resp. imaginary parts) with a space between.
7243  for (size_t col = 0; col < numCols; ++col) {
7244  for (size_t row = 0; row < printNumRows; ++row) {
7245  if (STS::isComplex) {
7246  out << STS::real (printData[row + col * printStride]) << " "
7247  << STS::imag (printData[row + col * printStride]) << endl;
7248  } else {
7249  out << printData[row + col * printStride] << endl;
7250  }
7251  }
7252  }
7253  }
7254  }
7255 
7256  if (myRank == 0) {
7257  // Wait on receive-size receive from Process 1.
7258  const int recvRank = 1;
7259  const int circBufInd = recvRank % 3;
7260  if (debug) {
7261  std::ostringstream os;
7262  os << myRank << ": Wait on receive-size receive from Process "
7263  << recvRank << endl;
7264  *dbg << os.str ();
7265  }
7266  if (numProcs > 1) {
7267  wait<int> (comm, outArg (recvSizeReqs[circBufInd]));
7268 
7269  // We received the number of rows of data. (The data
7270  // come in two columns.)
7271  size_t recvNumRows = (recvSizeBufs[circBufInd])[0];
7272  if (recvNumRows == Teuchos::OrdinalTraits<size_t>::invalid ()) {
7273  lclErr = 1;
7274  if (! err.is_null ()) {
7275  std::ostringstream os;
7276  os << myRank << ": Result of receive-size receive from Process "
7277  << recvRank << " is Teuchos::OrdinalTraits<size_t>::invalid() "
7278  << "= " << Teuchos::OrdinalTraits<size_t>::invalid () << ". "
7279  "This should never happen, and suggests that the receive never "
7280  "got posted. Please report this bug to the Tpetra developers."
7281  << endl;
7282  *err << os.str ();
7283  }
7284 
7285  // If we're going to continue after error, set the
7286  // number of rows to receive to a reasonable size. This
7287  // may cause MPI_ERR_TRUNCATE if the sending process is
7288  // sending more than 0 rows, but that's better than MPI
7289  // overflowing due to the huge positive value that is
7290  // Teuchos::OrdinalTraits<size_t>::invalid().
7291  recvNumRows = 0;
7292  }
7293 
7294  // Post receive-data receive from Process 1.
7295  recvDataBufs[circBufInd].resize (recvNumRows * numCols);
7296  if (debug) {
7297  std::ostringstream os;
7298  os << myRank << ": Post receive-data receive from Process "
7299  << recvRank << ": tag = " << dataTag << ", buffer size = "
7300  << recvDataBufs[circBufInd].size () << endl;
7301  *dbg << os.str ();
7302  }
7303  if (! recvSizeReqs[circBufInd].is_null ()) {
7304  lclErr = 1;
7305  if (! err.is_null ()) {
7306  std::ostringstream os;
7307  os << myRank << ": recvSizeReqs[" << circBufInd << "] is not "
7308  "null, before posting the receive-data receive from Process "
7309  << recvRank << ". This should never happen. Please report "
7310  "this bug to the Tpetra developers." << endl;
7311  *err << os.str ();
7312  }
7313  }
7314  recvDataReqs[circBufInd] =
7315  ireceive<int, scalar_type> (recvDataBufs[circBufInd],
7316  recvRank, dataTag, comm);
7317  } // numProcs > 1
7318  }
7319  else if (myRank == 1) {
7320  // Wait on my send-size send.
7321  if (debug) {
7322  std::ostringstream os;
7323  os << myRank << ": Wait on my send-size send" << endl;
7324  *dbg << os.str ();
7325  }
7326  wait<int> (comm, outArg (sendReqSize));
7327  }
7328 
7329  //
7330  // Pipeline loop
7331  //
7332  for (int p = 1; p < numProcs; ++p) {
7333  if (myRank == 0) {
7334  if (p + 2 < numProcs) {
7335  // Post receive-size receive from Process p + 2.
7336  const int recvRank = p + 2;
7337  const int circBufInd = recvRank % 3;
7338  if (debug) {
7339  std::ostringstream os;
7340  os << myRank << ": Post receive-size receive from Process "
7341  << recvRank << ": tag = " << sizeTag << endl;
7342  *dbg << os.str ();
7343  }
7344  if (! recvSizeReqs[circBufInd].is_null ()) {
7345  lclErr = 1;
7346  if (! err.is_null ()) {
7347  std::ostringstream os;
7348  os << myRank << ": recvSizeReqs[" << circBufInd << "] is not "
7349  << "null, for the receive-size receive from Process "
7350  << recvRank << "! This may mean that this process never "
7351  << "finished waiting for the receive from Process "
7352  << (recvRank - 3) << "." << endl;
7353  *err << os.str ();
7354  }
7355  }
7356  recvSizeReqs[circBufInd] =
7357  ireceive<int, size_t> (recvSizeBufs[circBufInd],
7358  recvRank, sizeTag, comm);
7359  }
7360 
7361  if (p + 1 < numProcs) {
7362  const int recvRank = p + 1;
7363  const int circBufInd = recvRank % 3;
7364 
7365  // Wait on receive-size receive from Process p + 1.
7366  if (debug) {
7367  std::ostringstream os;
7368  os << myRank << ": Wait on receive-size receive from Process "
7369  << recvRank << endl;
7370  *dbg << os.str ();
7371  }
7372  wait<int> (comm, outArg (recvSizeReqs[circBufInd]));
7373 
7374  // We received the number of rows of data. (The data
7375  // come in two columns.)
7376  size_t recvNumRows = (recvSizeBufs[circBufInd])[0];
7377  if (recvNumRows == Teuchos::OrdinalTraits<size_t>::invalid ()) {
7378  lclErr = 1;
7379  if (! err.is_null ()) {
7380  std::ostringstream os;
7381  os << myRank << ": Result of receive-size receive from Process "
7382  << recvRank << " is Teuchos::OrdinalTraits<size_t>::invalid() "
7383  << "= " << Teuchos::OrdinalTraits<size_t>::invalid () << ". "
7384  "This should never happen, and suggests that the receive never "
7385  "got posted. Please report this bug to the Tpetra developers."
7386  << endl;
7387  *err << os.str ();
7388  }
7389  // If we're going to continue after error, set the
7390  // number of rows to receive to a reasonable size.
7391  // This may cause MPI_ERR_TRUNCATE if the sending
7392  // process sends more than 0 rows, but that's better
7393  // than MPI overflowing due to the huge positive value
7394  // Teuchos::OrdinalTraits<size_t>::invalid().
7395  recvNumRows = 0;
7396  }
7397 
7398  // Post receive-data receive from Process p + 1.
7399  recvDataBufs[circBufInd].resize (recvNumRows * numCols);
7400  if (debug) {
7401  std::ostringstream os;
7402  os << myRank << ": Post receive-data receive from Process "
7403  << recvRank << ": tag = " << dataTag << ", buffer size = "
7404  << recvDataBufs[circBufInd].size () << endl;
7405  *dbg << os.str ();
7406  }
7407  if (! recvDataReqs[circBufInd].is_null ()) {
7408  lclErr = 1;
7409  if (! err.is_null ()) {
7410  std::ostringstream os;
7411  os << myRank << ": recvDataReqs[" << circBufInd << "] is not "
7412  << "null, for the receive-data receive from Process "
7413  << recvRank << "! This may mean that this process never "
7414  << "finished waiting for the receive from Process "
7415  << (recvRank - 3) << "." << endl;
7416  *err << os.str ();
7417  }
7418  }
7419  recvDataReqs[circBufInd] =
7420  ireceive<int, scalar_type> (recvDataBufs[circBufInd],
7421  recvRank, dataTag, comm);
7422  }
7423 
7424  // Wait on receive-data receive from Process p.
7425  const int recvRank = p;
7426  const int circBufInd = recvRank % 3;
7427  if (debug) {
7428  std::ostringstream os;
7429  os << myRank << ": Wait on receive-data receive from Process "
7430  << recvRank << endl;
7431  *dbg << os.str ();
7432  }
7433  wait<int> (comm, outArg (recvDataReqs[circBufInd]));
7434 
7435  // Write Process p's data. Number of rows lives in
7436  // recvSizeBufs[circBufInd], and the actual data live in
7437  // recvDataBufs[circBufInd]. Do this after posting receives,
7438  // in order to expose overlap of comm. with file I/O.
7439  if (debug) {
7440  std::ostringstream os;
7441  os << myRank << ": Write entries from Process " << recvRank
7442  << endl;
7443  *dbg << os.str () << endl;
7444  }
7445  size_t printNumRows = (recvSizeBufs[circBufInd])[0];
7446  if (printNumRows == Teuchos::OrdinalTraits<size_t>::invalid ()) {
7447  lclErr = 1;
7448  if (! err.is_null ()) {
7449  std::ostringstream os;
7450  os << myRank << ": Result of receive-size receive from Process "
7451  << recvRank << " was Teuchos::OrdinalTraits<size_t>::"
7452  "invalid() = " << Teuchos::OrdinalTraits<size_t>::invalid ()
7453  << ". This should never happen, and suggests that its "
7454  "receive-size receive was never posted. "
7455  "Please report this bug to the Tpetra developers." << endl;
7456  *err << os.str ();
7457  }
7458  // If we're going to continue after error, set the
7459  // number of rows to print to a reasonable size.
7460  printNumRows = 0;
7461  }
7462  if (printNumRows > 0 && recvDataBufs[circBufInd].is_null ()) {
7463  lclErr = 1;
7464  if (! err.is_null ()) {
7465  std::ostringstream os;
7466  os << myRank << ": Result of receive-size receive from Proc "
7467  << recvRank << " was " << printNumRows << " > 0, but "
7468  "recvDataBufs[" << circBufInd << "] is null. This should "
7469  "never happen. Please report this bug to the Tpetra "
7470  "developers." << endl;
7471  *err << os.str ();
7472  }
7473  // If we're going to continue after error, set the
7474  // number of rows to print to a reasonable size.
7475  printNumRows = 0;
7476  }
7477 
7478  // Write the received data to the output stream.
7479  // Matrix Market prints dense matrices in column-major order.
7480  ArrayView<const scalar_type> printData = (recvDataBufs[circBufInd]) ();
7481  const size_t printStride = printNumRows;
7482  // Matrix Market dense format wants one number per line.
7483  // It wants each complex number as two real numbers (real
7484  // resp. imaginary parts) with a space between.
7485  for (size_t col = 0; col < numCols; ++col) {
7486  for (size_t row = 0; row < printNumRows; ++row) {
7487  if (STS::isComplex) {
7488  out << STS::real (printData[row + col * printStride]) << " "
7489  << STS::imag (printData[row + col * printStride]) << endl;
7490  } else {
7491  out << printData[row + col * printStride] << endl;
7492  }
7493  }
7494  }
7495  }
7496  else if (myRank == p) { // Process p
7497  // Wait on my send-data send.
7498  if (debug) {
7499  std::ostringstream os;
7500  os << myRank << ": Wait on my send-data send" << endl;
7501  *dbg << os.str ();
7502  }
7503  wait<int> (comm, outArg (sendReqData));
7504  }
7505  else if (myRank == p + 1) { // Process p + 1
7506  // Post send-data send to Process 0.
7507  if (debug) {
7508  std::ostringstream os;
7509  os << myRank << ": Post send-data send: tag = " << dataTag
7510  << endl;
7511  *dbg << os.str ();
7512  }
7513  sendReqData = isend<int, scalar_type> (sendDataBuf, 0, dataTag, comm);
7514  // Wait on my send-size send.
7515  if (debug) {
7516  std::ostringstream os;
7517  os << myRank << ": Wait on my send-size send" << endl;
7518  *dbg << os.str ();
7519  }
7520  wait<int> (comm, outArg (sendReqSize));
7521  }
7522  else if (myRank == p + 2) { // Process p + 2
7523  // Post send-size send to Process 0.
7524  if (debug) {
7525  std::ostringstream os;
7526  os << myRank << ": Post send-size send: size = "
7527  << sendDataSize[0] << ", tag = " << sizeTag << endl;
7528  *dbg << os.str ();
7529  }
7530  sendReqSize = isend<int, size_t> (sendDataSize, 0, sizeTag, comm);
7531  }
7532  }
7533 
7534  // Establish global agreement on the error state.
7535  reduceAll<int, int> (comm, REDUCE_MAX, lclErr, outArg (gblErr));
7536  TEUCHOS_TEST_FOR_EXCEPTION(
7537  gblErr == 1, std::runtime_error, "Tpetra::MatrixMarket::writeDense "
7538  "experienced some kind of error and was unable to complete.");
7539 
7540  if (debug) {
7541  dbg->popTab ();
7542  *dbg << myRank << ": writeDenseColumn: Done" << endl;
7543  dbg->popTab ();
7544  }
7545  }
7546 
7547  public:
7548 
7554  static void
7555  writeDense (std::ostream& out,
7556  const Teuchos::RCP<const multivector_type>& X,
7557  const std::string& matrixName,
7558  const std::string& matrixDescription,
7559  const Teuchos::RCP<Teuchos::FancyOStream>& err = Teuchos::null,
7560  const Teuchos::RCP<Teuchos::FancyOStream>& dbg = Teuchos::null)
7561  {
7562  TEUCHOS_TEST_FOR_EXCEPTION(
7563  X.is_null (), std::invalid_argument, "Tpetra::MatrixMarket::"
7564  "writeDense: The input MultiVector X is null.");
7565  writeDense (out, *X, matrixName, matrixDescription, err, dbg);
7566  }
7567 
7573  static void
7574  writeDense (std::ostream& out,
7575  const multivector_type& X,
7576  const Teuchos::RCP<Teuchos::FancyOStream>& err = Teuchos::null,
7577  const Teuchos::RCP<Teuchos::FancyOStream>& dbg = Teuchos::null)
7578  {
7579  writeDense (out, X, "", "", err, dbg);
7580  }
7581 
7587  static void
7588  writeDense (std::ostream& out,
7589  const Teuchos::RCP<const multivector_type>& X,
7590  const Teuchos::RCP<Teuchos::FancyOStream>& err = Teuchos::null,
7591  const Teuchos::RCP<Teuchos::FancyOStream>& dbg = Teuchos::null)
7592  {
7593  TEUCHOS_TEST_FOR_EXCEPTION(
7594  X.is_null (), std::invalid_argument, "Tpetra::MatrixMarket::"
7595  "writeDense: The input MultiVector X is null.");
7596  writeDense (out, *X, "", "", err, dbg);
7597  }
7598 
7618  static void
7619  writeMap (std::ostream& out, const map_type& map, const bool debug=false)
7620  {
7621  Teuchos::RCP<Teuchos::FancyOStream> err =
7622  Teuchos::getFancyOStream (Teuchos::rcpFromRef (std::cerr));
7623  writeMap (out, map, err, debug);
7624  }
7625 
7634  static void
7635  writeMap (std::ostream& out,
7636  const map_type& map,
7637  const Teuchos::RCP<Teuchos::FancyOStream>& err,
7638  const bool debug=false)
7639  {
7640  using Teuchos::Array;
7641  using Teuchos::ArrayRCP;
7642  using Teuchos::ArrayView;
7643  using Teuchos::Comm;
7644  using Teuchos::CommRequest;
7645  using Teuchos::ireceive;
7646  using Teuchos::isend;
7647  using Teuchos::RCP;
7648  using Teuchos::TypeNameTraits;
7649  using Teuchos::wait;
7650  using std::endl;
7651  typedef global_ordinal_type GO;
7652  typedef int pid_type;
7653 
7654  // Treat the Map as a 1-column "multivector." This differs
7655  // from the previous two-column format, in which column 0 held
7656  // the GIDs, and column 1 held the corresponding PIDs. It
7657  // differs because printing that format requires knowing the
7658  // entire first column -- that is, all the GIDs -- in advance.
7659  // Sending messages from each process one at a time saves
7660  // memory, but it means that Process 0 doesn't ever have all
7661  // the GIDs at once.
7662  //
7663  // We pack the entries as ptrdiff_t, since this should be the
7664  // biggest signed built-in integer type that can hold any GO
7665  // or pid_type (= int) quantity without overflow. Test this
7666  // assumption at run time.
7667  typedef ptrdiff_t int_type;
7668  TEUCHOS_TEST_FOR_EXCEPTION(
7669  sizeof (GO) > sizeof (int_type), std::logic_error,
7670  "The global ordinal type GO=" << TypeNameTraits<GO>::name ()
7671  << " is too big for ptrdiff_t. sizeof(GO) = " << sizeof (GO)
7672  << " > sizeof(ptrdiff_t) = " << sizeof (ptrdiff_t) << ".");
7673  TEUCHOS_TEST_FOR_EXCEPTION(
7674  sizeof (pid_type) > sizeof (int_type), std::logic_error,
7675  "The (MPI) process rank type pid_type=" <<
7676  TypeNameTraits<pid_type>::name () << " is too big for ptrdiff_t. "
7677  "sizeof(pid_type) = " << sizeof (pid_type) << " > sizeof(ptrdiff_t)"
7678  " = " << sizeof (ptrdiff_t) << ".");
7679 
7680  const Comm<int>& comm = * (map.getComm ());
7681  const int myRank = comm.getRank ();
7682  const int numProcs = comm.getSize ();
7683 
7684  if (! err.is_null ()) {
7685  err->pushTab ();
7686  }
7687  if (debug) {
7688  std::ostringstream os;
7689  os << myRank << ": writeMap" << endl;
7690  *err << os.str ();
7691  }
7692  if (! err.is_null ()) {
7693  err->pushTab ();
7694  }
7695 
7696  const size_t myNumRows = map.getNodeNumElements ();
7697  // Use a different tag for the "size" messages than for the
7698  // "data" messages, in order to help us debug any mix-ups.
7699  const int sizeTag = 1337;
7700  const int dataTag = 1338;
7701 
7702  // Process 0 pipelines nonblocking receives with file output.
7703  //
7704  // Constraints:
7705  // - Process 0 can't post a receive for another process'
7706  // actual data, until it posts and waits on the receive
7707  // from that process with the amount of data to receive.
7708  // (We could just post receives with a max data size, but
7709  // I feel uncomfortable about that.)
7710  // - The C++ standard library doesn't allow nonblocking
7711  // output to an std::ostream.
7712  //
7713  // Process 0: Post receive-size receives from Processes 1 and 2.
7714  // Process 1: Post send-size send to Process 0.
7715  // Process 2: Post send-size send to Process 0.
7716  //
7717  // All processes: Pack my GIDs and PIDs.
7718  //
7719  // Process 1:
7720  // - Post send-data send to Process 0.
7721  // - Wait on my send-size send to Process 0.
7722  //
7723  // Process 0:
7724  // - Print MatrixMarket header.
7725  // - Print my GIDs and PIDs.
7726  // - Wait on receive-size receive from Process 1.
7727  // - Post receive-data receive from Process 1.
7728  //
7729  // For each process p = 1, 2, ... numProcs-1:
7730  // If I am Process 0:
7731  // - Post receive-size receive from Process p + 2
7732  // - Wait on receive-size receive from Process p + 1
7733  // - Post receive-data receive from Process p + 1
7734  // - Wait on receive-data receive from Process p
7735  // - Write data from Process p.
7736  // Else if I am Process p:
7737  // - Wait on my send-data send.
7738  // Else if I am Process p+1:
7739  // - Post send-data send to Process 0.
7740  // - Wait on my send-size send.
7741  // Else if I am Process p+2:
7742  // - Post send-size send to Process 0.
7743  //
7744  // Pipelining has three goals here:
7745  // 1. Overlap communication (the receives) with file I/O
7746  // 2. Give Process 0 a chance to prepost some receives,
7747  // before sends show up, by packing local data before
7748  // posting sends
7749  // 3. Don't post _all_ receives or _all_ sends, because that
7750  // wouldn't be memory scalable. (Just because we can't
7751  // see how much memory MPI consumes, doesn't mean that it
7752  // doesn't consume any!)
7753 
7754  // These are used on every process. sendReqSize[0] holds the
7755  // number of rows on this process, and sendReqBuf holds this
7756  // process' data. Process 0 packs into sendReqBuf, but
7757  // doesn't send; it only uses that for printing. All other
7758  // processes send both of these to Process 0.
7759  RCP<CommRequest<int> > sendReqSize, sendReqData;
7760 
7761  // These are used only on Process 0, for received data. Keep
7762  // 3 of each, and treat the arrays as circular buffers. When
7763  // receiving from Process p, the corresponding array index
7764  // here is p % 3.
7765  Array<ArrayRCP<int_type> > recvSizeBufs (3);
7766  Array<ArrayRCP<int_type> > recvDataBufs (3);
7767  Array<RCP<CommRequest<int> > > recvSizeReqs (3);
7768  Array<RCP<CommRequest<int> > > recvDataReqs (3);
7769 
7770  // Buffer for nonblocking send of the "send size."
7771  ArrayRCP<int_type> sendDataSize (1);
7772  sendDataSize[0] = myNumRows;
7773 
7774  if (myRank == 0) {
7775  if (debug) {
7776  std::ostringstream os;
7777  os << myRank << ": Post receive-size receives from "
7778  "Procs 1 and 2: tag = " << sizeTag << endl;
7779  *err << os.str ();
7780  }
7781  // Process 0: Post receive-size receives from Processes 1 and 2.
7782  recvSizeBufs[0].resize (1);
7783  (recvSizeBufs[0])[0] = -1; // error flag
7784  recvSizeBufs[1].resize (1);
7785  (recvSizeBufs[1])[0] = -1; // error flag
7786  recvSizeBufs[2].resize (1);
7787  (recvSizeBufs[2])[0] = -1; // error flag
7788  if (numProcs > 1) {
7789  recvSizeReqs[1] =
7790  ireceive<int, int_type> (recvSizeBufs[1], 1, sizeTag, comm);
7791  }
7792  if (numProcs > 2) {
7793  recvSizeReqs[2] =
7794  ireceive<int, int_type> (recvSizeBufs[2], 2, sizeTag, comm);
7795  }
7796  }
7797  else if (myRank == 1 || myRank == 2) {
7798  if (debug) {
7799  std::ostringstream os;
7800  os << myRank << ": Post send-size send: size = "
7801  << sendDataSize[0] << ", tag = " << sizeTag << endl;
7802  *err << os.str ();
7803  }
7804  // Prime the pipeline by having Processes 1 and 2 start
7805  // their send-size sends. We don't want _all_ the processes
7806  // to start their send-size sends, because that wouldn't be
7807  // memory scalable.
7808  sendReqSize = isend<int, int_type> (sendDataSize, 0, sizeTag, comm);
7809  }
7810  else {
7811  if (debug) {
7812  std::ostringstream os;
7813  os << myRank << ": Not posting my send-size send yet" << endl;
7814  *err << os.str ();
7815  }
7816  }
7817 
7818  //
7819  // Pack my GIDs and PIDs. Each (GID,PID) pair gets packed
7820  // consecutively, for better locality.
7821  //
7822 
7823  if (debug) {
7824  std::ostringstream os;
7825  os << myRank << ": Pack my GIDs and PIDs" << endl;
7826  *err << os.str ();
7827  }
7828 
7829  ArrayRCP<int_type> sendDataBuf (myNumRows * 2);
7830 
7831  if (map.isContiguous ()) {
7832  const int_type myMinGblIdx =
7833  static_cast<int_type> (map.getMinGlobalIndex ());
7834  for (size_t k = 0; k < myNumRows; ++k) {
7835  const int_type gid = myMinGblIdx + static_cast<int_type> (k);
7836  const int_type pid = static_cast<int_type> (myRank);
7837  sendDataBuf[2*k] = gid;
7838  sendDataBuf[2*k+1] = pid;
7839  }
7840  }
7841  else {
7842  ArrayView<const GO> myGblInds = map.getNodeElementList ();
7843  for (size_t k = 0; k < myNumRows; ++k) {
7844  const int_type gid = static_cast<int_type> (myGblInds[k]);
7845  const int_type pid = static_cast<int_type> (myRank);
7846  sendDataBuf[2*k] = gid;
7847  sendDataBuf[2*k+1] = pid;
7848  }
7849  }
7850 
7851  if (debug) {
7852  std::ostringstream os;
7853  os << myRank << ": Done packing my GIDs and PIDs" << endl;
7854  *err << os.str ();
7855  }
7856 
7857  if (myRank == 1) {
7858  // Process 1: post send-data send to Process 0.
7859  if (debug) {
7860  *err << myRank << ": Post send-data send: tag = " << dataTag
7861  << endl;
7862  }
7863  sendReqData = isend<int, int_type> (sendDataBuf, 0, dataTag, comm);
7864  }
7865 
7866  if (myRank == 0) {
7867  if (debug) {
7868  *err << myRank << ": Write MatrixMarket header" << endl;
7869  }
7870 
7871  // Process 0: Write the MatrixMarket header.
7872  // Description section explains each column.
7873  std::ostringstream hdr;
7874 
7875  // Print the Matrix Market header. MultiVector stores data
7876  // nonsymmetrically, hence "general" in the banner line.
7877  hdr << "%%MatrixMarket matrix array integer general" << endl
7878  << "% Format: Version 2.0" << endl
7879  << "%" << endl
7880  << "% This file encodes a Tpetra::Map." << endl
7881  << "% It is stored as a dense vector, with twice as many " << endl
7882  << "% entries as the global number of GIDs (global indices)." << endl
7883  << "% (GID, PID) pairs are stored contiguously, where the PID " << endl
7884  << "% is the rank of the process owning that GID." << endl
7885  << (2 * map.getGlobalNumElements ()) << " " << 1 << endl;
7886  out << hdr.str ();
7887 
7888  if (debug) {
7889  std::ostringstream os;
7890  os << myRank << ": Write my GIDs and PIDs" << endl;
7891  *err << os.str ();
7892  }
7893 
7894  // Write Process 0's data to the output stream.
7895  // Matrix Market prints dense matrices in column-major order.
7896  const int_type printNumRows = myNumRows;
7897  ArrayView<const int_type> printData = sendDataBuf ();
7898  for (int_type k = 0; k < printNumRows; ++k) {
7899  const int_type gid = printData[2*k];
7900  const int_type pid = printData[2*k+1];
7901  out << gid << endl << pid << endl;
7902  }
7903  }
7904 
7905  if (myRank == 0) {
7906  // Wait on receive-size receive from Process 1.
7907  const int recvRank = 1;
7908  const int circBufInd = recvRank % 3;
7909  if (debug) {
7910  std::ostringstream os;
7911  os << myRank << ": Wait on receive-size receive from Process "
7912  << recvRank << endl;
7913  *err << os.str ();
7914  }
7915  if (numProcs > 1) {
7916  wait<int> (comm, outArg (recvSizeReqs[circBufInd]));
7917 
7918  // We received the number of rows of data. (The data
7919  // come in two columns.)
7920  const int_type recvNumRows = (recvSizeBufs[circBufInd])[0];
7921  if (debug && recvNumRows == -1) {
7922  std::ostringstream os;
7923  os << myRank << ": Result of receive-size receive from Process "
7924  << recvRank << " is -1. This should never happen, and "
7925  "suggests that the receive never got posted. Please report "
7926  "this bug to the Tpetra developers." << endl;
7927  *err << os.str ();
7928  }
7929 
7930  // Post receive-data receive from Process 1.
7931  recvDataBufs[circBufInd].resize (recvNumRows * 2);
7932  if (debug) {
7933  std::ostringstream os;
7934  os << myRank << ": Post receive-data receive from Process "
7935  << recvRank << ": tag = " << dataTag << ", buffer size = "
7936  << recvDataBufs[circBufInd].size () << endl;
7937  *err << os.str ();
7938  }
7939  if (! recvSizeReqs[circBufInd].is_null ()) {
7940  std::ostringstream os;
7941  os << myRank << ": recvSizeReqs[" << circBufInd << "] is not "
7942  "null, before posting the receive-data receive from Process "
7943  << recvRank << ". This should never happen. Please report "
7944  "this bug to the Tpetra developers." << endl;
7945  *err << os.str ();
7946  }
7947  recvDataReqs[circBufInd] =
7948  ireceive<int, int_type> (recvDataBufs[circBufInd],
7949  recvRank, dataTag, comm);
7950  } // numProcs > 1
7951  }
7952  else if (myRank == 1) {
7953  // Wait on my send-size send.
7954  if (debug) {
7955  std::ostringstream os;
7956  os << myRank << ": Wait on my send-size send" << endl;
7957  *err << os.str ();
7958  }
7959  wait<int> (comm, outArg (sendReqSize));
7960  }
7961 
7962  //
7963  // Pipeline loop
7964  //
7965  for (int p = 1; p < numProcs; ++p) {
7966  if (myRank == 0) {
7967  if (p + 2 < numProcs) {
7968  // Post receive-size receive from Process p + 2.
7969  const int recvRank = p + 2;
7970  const int circBufInd = recvRank % 3;
7971  if (debug) {
7972  std::ostringstream os;
7973  os << myRank << ": Post receive-size receive from Process "
7974  << recvRank << ": tag = " << sizeTag << endl;
7975  *err << os.str ();
7976  }
7977  if (! recvSizeReqs[circBufInd].is_null ()) {
7978  std::ostringstream os;
7979  os << myRank << ": recvSizeReqs[" << circBufInd << "] is not "
7980  << "null, for the receive-size receive from Process "
7981  << recvRank << "! This may mean that this process never "
7982  << "finished waiting for the receive from Process "
7983  << (recvRank - 3) << "." << endl;
7984  *err << os.str ();
7985  }
7986  recvSizeReqs[circBufInd] =
7987  ireceive<int, int_type> (recvSizeBufs[circBufInd],
7988  recvRank, sizeTag, comm);
7989  }
7990 
7991  if (p + 1 < numProcs) {
7992  const int recvRank = p + 1;
7993  const int circBufInd = recvRank % 3;
7994 
7995  // Wait on receive-size receive from Process p + 1.
7996  if (debug) {
7997  std::ostringstream os;
7998  os << myRank << ": Wait on receive-size receive from Process "
7999  << recvRank << endl;
8000  *err << os.str ();
8001  }
8002  wait<int> (comm, outArg (recvSizeReqs[circBufInd]));
8003 
8004  // We received the number of rows of data. (The data
8005  // come in two columns.)
8006  const int_type recvNumRows = (recvSizeBufs[circBufInd])[0];
8007  if (debug && recvNumRows == -1) {
8008  std::ostringstream os;
8009  os << myRank << ": Result of receive-size receive from Process "
8010  << recvRank << " is -1. This should never happen, and "
8011  "suggests that the receive never got posted. Please report "
8012  "this bug to the Tpetra developers." << endl;
8013  *err << os.str ();
8014  }
8015 
8016  // Post receive-data receive from Process p + 1.
8017  recvDataBufs[circBufInd].resize (recvNumRows * 2);
8018  if (debug) {
8019  std::ostringstream os;
8020  os << myRank << ": Post receive-data receive from Process "
8021  << recvRank << ": tag = " << dataTag << ", buffer size = "
8022  << recvDataBufs[circBufInd].size () << endl;
8023  *err << os.str ();
8024  }
8025  if (! recvDataReqs[circBufInd].is_null ()) {
8026  std::ostringstream os;
8027  os << myRank << ": recvDataReqs[" << circBufInd << "] is not "
8028  << "null, for the receive-data receive from Process "
8029  << recvRank << "! This may mean that this process never "
8030  << "finished waiting for the receive from Process "
8031  << (recvRank - 3) << "." << endl;
8032  *err << os.str ();
8033  }
8034  recvDataReqs[circBufInd] =
8035  ireceive<int, int_type> (recvDataBufs[circBufInd],
8036  recvRank, dataTag, comm);
8037  }
8038 
8039  // Wait on receive-data receive from Process p.
8040  const int recvRank = p;
8041  const int circBufInd = recvRank % 3;
8042  if (debug) {
8043  std::ostringstream os;
8044  os << myRank << ": Wait on receive-data receive from Process "
8045  << recvRank << endl;
8046  *err << os.str ();
8047  }
8048  wait<int> (comm, outArg (recvDataReqs[circBufInd]));
8049 
8050  // Write Process p's data. Number of rows lives in
8051  // recvSizeBufs[circBufInd], and the actual data live in
8052  // recvDataBufs[circBufInd]. Do this after posting receives,
8053  // in order to expose overlap of comm. with file I/O.
8054  if (debug) {
8055  std::ostringstream os;
8056  os << myRank << ": Write GIDs and PIDs from Process "
8057  << recvRank << endl;
8058  *err << os.str () << endl;
8059  }
8060  const int_type printNumRows = (recvSizeBufs[circBufInd])[0];
8061  if (debug && printNumRows == -1) {
8062  std::ostringstream os;
8063  os << myRank << ": Result of receive-size receive from Process "
8064  << recvRank << " was -1. This should never happen, and "
8065  "suggests that its receive-size receive was never posted. "
8066  "Please report this bug to the Tpetra developers." << endl;
8067  *err << os.str ();
8068  }
8069  if (debug && printNumRows > 0 && recvDataBufs[circBufInd].is_null ()) {
8070  std::ostringstream os;
8071  os << myRank << ": Result of receive-size receive from Proc "
8072  << recvRank << " was " << printNumRows << " > 0, but "
8073  "recvDataBufs[" << circBufInd << "] is null. This should "
8074  "never happen. Please report this bug to the Tpetra "
8075  "developers." << endl;
8076  *err << os.str ();
8077  }
8078  ArrayView<const int_type> printData = (recvDataBufs[circBufInd]) ();
8079  for (int_type k = 0; k < printNumRows; ++k) {
8080  const int_type gid = printData[2*k];
8081  const int_type pid = printData[2*k+1];
8082  out << gid << endl << pid << endl;
8083  }
8084  }
8085  else if (myRank == p) { // Process p
8086  // Wait on my send-data send.
8087  if (debug) {
8088  std::ostringstream os;
8089  os << myRank << ": Wait on my send-data send" << endl;
8090  *err << os.str ();
8091  }
8092  wait<int> (comm, outArg (sendReqData));
8093  }
8094  else if (myRank == p + 1) { // Process p + 1
8095  // Post send-data send to Process 0.
8096  if (debug) {
8097  std::ostringstream os;
8098  os << myRank << ": Post send-data send: tag = " << dataTag
8099  << endl;
8100  *err << os.str ();
8101  }
8102  sendReqData = isend<int, int_type> (sendDataBuf, 0, dataTag, comm);
8103  // Wait on my send-size send.
8104  if (debug) {
8105  std::ostringstream os;
8106  os << myRank << ": Wait on my send-size send" << endl;
8107  *err << os.str ();
8108  }
8109  wait<int> (comm, outArg (sendReqSize));
8110  }
8111  else if (myRank == p + 2) { // Process p + 2
8112  // Post send-size send to Process 0.
8113  if (debug) {
8114  std::ostringstream os;
8115  os << myRank << ": Post send-size send: size = "
8116  << sendDataSize[0] << ", tag = " << sizeTag << endl;
8117  *err << os.str ();
8118  }
8119  sendReqSize = isend<int, int_type> (sendDataSize, 0, sizeTag, comm);
8120  }
8121  }
8122 
8123  if (! err.is_null ()) {
8124  err->popTab ();
8125  }
8126  if (debug) {
8127  *err << myRank << ": writeMap: Done" << endl;
8128  }
8129  if (! err.is_null ()) {
8130  err->popTab ();
8131  }
8132  }
8133 
8135  static void
8136  writeMapFile (const std::string& filename,
8137  const map_type& map)
8138  {
8139  const int myRank = map.getComm ()->getRank ();
8140  std::ofstream out;
8141  if (myRank == 0) { // Only open the file on Proc 0.
8142  out.open (filename.c_str());
8143  }
8144  writeMap (out, map);
8145  // We can rely on the destructor of the output stream to close
8146  // the file on scope exit, even if writeDense() throws an
8147  // exception.
8148  }
8149 
8150  private:
8174  static void
8175  printAsComment (std::ostream& out, const std::string& str)
8176  {
8177  using std::endl;
8178  std::istringstream inpstream (str);
8179  std::string line;
8180 
8181  while (getline (inpstream, line)) {
8182  if (! line.empty()) {
8183  // Note that getline() doesn't store '\n', so we have to
8184  // append the endline ourselves.
8185  if (line[0] == '%') { // Line starts with a comment character.
8186  out << line << endl;
8187  }
8188  else { // Line doesn't start with a comment character.
8189  out << "%% " << line << endl;
8190  }
8191  }
8192  }
8193  }
8194 
8195  public:
8196 
8215  static void
8216  writeOperator(const std::string& fileName, operator_type const &A) {
8217  Teuchos::ParameterList pl;
8218  writeOperator(fileName, A, pl);
8219  }
8220 
8241  static void
8242  writeOperator (std::ostream& out, const operator_type& A) {
8243  Teuchos::ParameterList pl;
8244  writeOperator (out, A, pl);
8245  }
8246 
8283  static void
8284  writeOperator (const std::string& fileName,
8285  const operator_type& A,
8286  const Teuchos::ParameterList& params)
8287  {
8288  std::ofstream out;
8289  std::string tmpFile = "__TMP__" + fileName;
8290  const int myRank = A.getDomainMap()->getComm()->getRank();
8291  bool precisionChanged=false;
8292  int oldPrecision;
8293  // The number of nonzero entries in a Tpetra::Operator is
8294  // unknown until probing is completed. In order to write a
8295  // MatrixMarket header, we write the matrix to a temporary
8296  // file.
8297  //
8298  // FIXME (mfh 23 May 2015) IT WASN'T MY IDEA TO WRITE TO A
8299  // TEMPORARY FILE.
8300  if (myRank==0) {
8301  if (std::ifstream(tmpFile))
8302  TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
8303  "writeOperator: temporary file " << tmpFile << " already exists");
8304  out.open(tmpFile.c_str());
8305  if (params.isParameter("precision")) {
8306  oldPrecision = out.precision(params.get<int>("precision"));
8307  precisionChanged=true;
8308  }
8309  }
8310 
8311  const std::string header = writeOperatorImpl(out, A, params);
8312 
8313  if (myRank==0) {
8314  if (precisionChanged)
8315  out.precision(oldPrecision);
8316  out.close();
8317  out.open(fileName.c_str(), std::ios::binary);
8318  bool printMatrixMarketHeader = true;
8319  if (params.isParameter("print MatrixMarket header"))
8320  printMatrixMarketHeader = params.get<bool>("print MatrixMarket header");
8321  if (printMatrixMarketHeader && myRank == 0) {
8322  // Write header to final file.
8323  out << header;
8324  }
8325  // Append matrix from temporary to final file.
8326  std::ifstream src(tmpFile, std::ios_base::binary);
8327  out << src.rdbuf();
8328  src.close();
8329  // Delete the temporary file.
8330  remove(tmpFile.c_str());
8331  }
8332  }
8333 
8372  static void
8373  writeOperator (std::ostream& out,
8374  const operator_type& A,
8375  const Teuchos::ParameterList& params)
8376  {
8377  const int myRank = A.getDomainMap ()->getComm ()->getRank ();
8378 
8379  // The number of nonzero entries in a Tpetra::Operator is
8380  // unknown until probing is completed. In order to write a
8381  // MatrixMarket header, we write the matrix to a temporary
8382  // output stream.
8383  //
8384  // NOTE (mfh 23 May 2015): Writing to a temporary output
8385  // stream may double the memory usage, depending on whether
8386  // 'out' is a file stream or an in-memory output stream (e.g.,
8387  // std::ostringstream). It might be wise to use a temporary
8388  // file instead. However, please look carefully at POSIX
8389  // functions for safe creation of temporary files. Don't just
8390  // prepend "__TMP__" to the filename and hope for the best.
8391  // Furthermore, it should be valid to call the std::ostream
8392  // overload of this method even when Process 0 does not have
8393  // access to a file system.
8394  std::ostringstream tmpOut;
8395  if (myRank == 0) {
8396  if (params.isParameter ("precision") && params.isType<int> ("precision")) {
8397  (void) tmpOut.precision (params.get<int> ("precision"));
8398  }
8399  }
8400 
8401  const std::string header = writeOperatorImpl (tmpOut, A, params);
8402 
8403  if (myRank == 0) {
8404  bool printMatrixMarketHeader = true;
8405  if (params.isParameter ("print MatrixMarket header") &&
8406  params.isType<bool> ("print MatrixMarket header")) {
8407  printMatrixMarketHeader = params.get<bool> ("print MatrixMarket header");
8408  }
8409  if (printMatrixMarketHeader && myRank == 0) {
8410  out << header; // write header to final output stream
8411  }
8412  // Append matrix from temporary output stream to final output stream.
8413  //
8414  // NOTE (mfh 23 May 2015) This might use a lot of memory.
8415  // However, we should not use temporary files in this
8416  // method. Since it does not access the file system (unlike
8417  // the overload that takes a file name), it should not
8418  // require the file system at all.
8419  //
8420  // If memory usage becomes a problem, one thing we could do
8421  // is write the entries of the Operator one column (or a few
8422  // columns) at a time. The Matrix Market sparse format does
8423  // not impose an order on its entries, so it would be OK to
8424  // write them in that order.
8425  out << tmpOut.str ();
8426  }
8427  }
8428 
8429  private:
8430 
8438  static std::string
8439  writeOperatorImpl (std::ostream& os,
8440  const operator_type& A,
8441  const Teuchos::ParameterList& params)
8442  {
8443  using Teuchos::RCP;
8444  using Teuchos::rcp;
8445  using Teuchos::ArrayRCP;
8446  using Teuchos::Array;
8447 
8448  typedef local_ordinal_type LO;
8449  typedef global_ordinal_type GO;
8450  typedef scalar_type Scalar;
8451  typedef Teuchos::OrdinalTraits<LO> TLOT;
8452  typedef Teuchos::OrdinalTraits<GO> TGOT;
8453  typedef Tpetra::Import<LO, GO, node_type> import_type;
8454  typedef Tpetra::MultiVector<GO, LO, GO, node_type> mv_type_go;
8455 
8456  const map_type& domainMap = *(A.getDomainMap());
8457  RCP<const map_type> rangeMap = A.getRangeMap();
8458  RCP<const Teuchos::Comm<int> > comm = rangeMap->getComm();
8459  const int myRank = comm->getRank();
8460  const size_t numProcs = comm->getSize();
8461 
8462  size_t numMVs = 10;
8463  if (params.isParameter("probing size"))
8464  numMVs = params.get<int>("probing size");
8465 
8466  GO globalNnz = 0;
8467  GO minColGid = domainMap.getMinAllGlobalIndex();
8468  GO maxColGid = domainMap.getMaxAllGlobalIndex();
8469  // Rather than replicating the domainMap on all processors, we instead
8470  // iterate from the min GID to the max GID. If the map is gappy,
8471  // there will be invalid GIDs, i.e., GIDs no one has. This will require
8472  // unnecessary matvecs against potentially zero vectors.
8473  GO numGlobElts = maxColGid - minColGid + TGOT::one();
8474  GO numChunks = numGlobElts / numMVs;
8475  GO rem = numGlobElts % numMVs;
8476  GO indexBase = rangeMap->getIndexBase();
8477 
8478  int offsetToUseInPrinting = 1 - indexBase; // default is 1-based indexing
8479  if (params.isParameter("zero-based indexing")) {
8480  if (params.get<bool>("zero-based indexing") == true)
8481  offsetToUseInPrinting = -indexBase; // If 0-based, use as-is. If 1-based, subtract 1.
8482  }
8483 
8484  // Create map that replicates the range map on pid 0 and is empty for all other pids
8485  size_t numLocalRangeEntries = rangeMap->getNodeNumElements();
8486 
8487  // Create contiguous source map
8488  RCP<const map_type> allGidsMap = rcp(new map_type(TGOT::invalid(), numLocalRangeEntries,
8489  indexBase, comm));
8490  // Create vector based on above map. Populate it with GIDs corresponding to this pid's GIDs in rangeMap.
8491  mv_type_go allGids(allGidsMap,1);
8492  Teuchos::ArrayRCP<GO> allGidsData = allGids.getDataNonConst(0);
8493 
8494  for (size_t i=0; i<numLocalRangeEntries; i++)
8495  allGidsData[i] = rangeMap->getGlobalElement(i);
8496  allGidsData = Teuchos::null;
8497 
8498  // Create target map that is nontrivial only on pid 0
8499  GO numTargetMapEntries=TGOT::zero();
8500  Teuchos::Array<GO> importGidList;
8501  if (myRank==0) {
8502  numTargetMapEntries = rangeMap->getGlobalNumElements();
8503  importGidList.reserve(numTargetMapEntries);
8504  for (GO j=0; j<numTargetMapEntries; ++j) importGidList.push_back(j + indexBase);
8505  } else {
8506  importGidList.reserve(numTargetMapEntries);
8507  }
8508  RCP<map_type> importGidMap = rcp(new map_type(TGOT::invalid(), importGidList(), indexBase, comm));
8509 
8510  // Import all rangeMap GIDs to pid 0
8511  import_type gidImporter(allGidsMap, importGidMap);
8512  mv_type_go importedGids(importGidMap, 1);
8513  importedGids.doImport(allGids, gidImporter, INSERT);
8514 
8515  // The following import map will be non-trivial only on pid 0.
8516  ArrayRCP<const GO> importedGidsData = importedGids.getData(0);
8517  RCP<const map_type> importMap = rcp(new map_type(TGOT::invalid(), importedGidsData(), indexBase, comm) );
8518 
8519  // Importer from original range map to pid 0
8520  import_type importer(rangeMap, importMap);
8521  // Target vector on pid 0
8522  RCP<mv_type> colsOnPid0 = rcp(new mv_type(importMap,numMVs));
8523 
8524  RCP<mv_type> ei = rcp(new mv_type(A.getDomainMap(),numMVs)); //probing vector
8525  RCP<mv_type> colsA = rcp(new mv_type(A.getRangeMap(),numMVs)); //columns of A revealed by probing
8526 
8527  Array<GO> globalColsArray, localColsArray;
8528  globalColsArray.reserve(numMVs);
8529  localColsArray.reserve(numMVs);
8530 
8531  ArrayRCP<ArrayRCP<Scalar> > eiData(numMVs);
8532  for (size_t i=0; i<numMVs; ++i)
8533  eiData[i] = ei->getDataNonConst(i);
8534 
8535  // //////////////////////////////////////
8536  // Discover A by chunks
8537  // //////////////////////////////////////
8538  for (GO k=0; k<numChunks; ++k) {
8539  for (size_t j=0; j<numMVs; ++j ) {
8540  //GO curGlobalCol = maxColGid - numMVs + j + TGOT::one();
8541  GO curGlobalCol = minColGid + k*numMVs + j;
8542  globalColsArray.push_back(curGlobalCol);
8543  //TODO extract the g2l map outside of this loop loop
8544  LO curLocalCol = domainMap.getLocalElement(curGlobalCol);
8545  if (curLocalCol != TLOT::invalid()) {
8546  eiData[j][curLocalCol] = TGOT::one();
8547  localColsArray.push_back(curLocalCol);
8548  }
8549  }
8550  //TODO Do the views eiData need to be released prior to the matvec?
8551 
8552  // probe
8553  A.apply(*ei,*colsA);
8554 
8555  colsOnPid0->doImport(*colsA,importer,INSERT);
8556 
8557  if (myRank==0)
8558  globalNnz += writeColumns(os,*colsOnPid0, numMVs, importedGidsData(),
8559  globalColsArray, offsetToUseInPrinting);
8560 
8561  //zero out the ei's
8562  for (size_t j=0; j<numMVs; ++j ) {
8563  for (int i=0; i<localColsArray.size(); ++i)
8564  eiData[j][localColsArray[i]] = TGOT::zero();
8565  }
8566  globalColsArray.clear();
8567  localColsArray.clear();
8568 
8569  }
8570 
8571  // //////////////////////////////////////
8572  // Handle leftover part of A
8573  // //////////////////////////////////////
8574  if (rem > 0) {
8575  for (int j=0; j<rem; ++j ) {
8576  GO curGlobalCol = maxColGid - rem + j + TGOT::one();
8577  globalColsArray.push_back(curGlobalCol);
8578  //TODO extract the g2l map outside of this loop loop
8579  LO curLocalCol = domainMap.getLocalElement(curGlobalCol);
8580  if (curLocalCol != TLOT::invalid()) {
8581  eiData[j][curLocalCol] = TGOT::one();
8582  localColsArray.push_back(curLocalCol);
8583  }
8584  }
8585  //TODO Do the views eiData need to be released prior to the matvec?
8586 
8587  // probe
8588  A.apply(*ei,*colsA);
8589 
8590  colsOnPid0->doImport(*colsA,importer,INSERT);
8591  if (myRank==0)
8592  globalNnz += writeColumns(os,*colsOnPid0, rem, importedGidsData(),
8593  globalColsArray, offsetToUseInPrinting);
8594 
8595  //zero out the ei's
8596  for (int j=0; j<rem; ++j ) {
8597  for (int i=0; i<localColsArray.size(); ++i)
8598  eiData[j][localColsArray[i]] = TGOT::zero();
8599  }
8600  globalColsArray.clear();
8601  localColsArray.clear();
8602 
8603  }
8604 
8605  // Return the Matrix Market header. It includes the header
8606  // line (that starts with "%%"), some comments, and the triple
8607  // of matrix dimensions and number of nonzero entries. We
8608  // don't actually print this here, because we don't know the
8609  // number of nonzero entries until after probing.
8610  std::ostringstream oss;
8611  if (myRank == 0) {
8612  oss << "%%MatrixMarket matrix coordinate ";
8613  if (Teuchos::ScalarTraits<typename operator_type::scalar_type>::isComplex) {
8614  oss << "complex";
8615  } else {
8616  oss << "real";
8617  }
8618  oss << " general" << std::endl;
8619  oss << "% Tpetra::Operator" << std::endl;
8620  std::time_t now = std::time(NULL);
8621  oss << "% time stamp: " << ctime(&now);
8622  oss << "% written from " << numProcs << " processes" << std::endl;
8623  size_t numRows = rangeMap->getGlobalNumElements();
8624  size_t numCols = domainMap.getGlobalNumElements();
8625  oss << numRows << " " << numCols << " " << globalNnz << std::endl;
8626  }
8627 
8628  return oss.str ();
8629  }
8630 
8631  static global_ordinal_type
8632  writeColumns(std::ostream& os, mv_type const &colsA, size_t const &numCols,
8633  Teuchos::ArrayView<const global_ordinal_type> const &rowGids,
8634  Teuchos::Array<global_ordinal_type> const &colsArray,
8635  global_ordinal_type const & indexBase) {
8636 
8637  typedef global_ordinal_type GO;
8638  typedef scalar_type Scalar;
8639  typedef Teuchos::ScalarTraits<Scalar> STS;
8640 
8641  GO nnz=0;
8642  const Scalar zero = STS::zero();
8643  const size_t numRows = colsA.getGlobalLength();
8644  for (size_t j=0; j<numCols; ++j) {
8645  Teuchos::ArrayRCP<const Scalar> const curCol = colsA.getData(j);
8646  const GO J = colsArray[j];
8647  for (size_t i=0; i<numRows; ++i) {
8648  const Scalar val = curCol[i];
8649  if (val!=zero) {
8650  os << rowGids[i]+indexBase << " " << J+indexBase << " " << val << std::endl;
8651  ++nnz;
8652  }
8653  }
8654  }
8655 
8656  return nnz;
8657 
8658  }
8659 
8660  public:
8661 
8662  }; // class Writer
8663 
8664  } // namespace MatrixMarket
8665 } // namespace Tpetra
8666 
8667 #endif // __MatrixMarket_Tpetra_hpp
static Teuchos::RCP< vector_type > readVectorFile(const std::string &filename, const Teuchos::RCP< const comm_type > &comm, Teuchos::RCP< const map_type > &map, const bool tolerant=false, const bool debug=false)
Read a Vector from the given Matrix Market file.
Communication plan for data redistribution from a uniquely-owned to a (possibly) multiply-owned distr...
Teuchos::RCP< const map_type > getRowMap() const override
Returns the Map that describes the row distribution in this graph.
static Teuchos::RCP< sparse_graph_type > readSparseGraph(std::istream &in, const Teuchos::RCP< const Teuchos::Comm< int > > &pComm, const Teuchos::RCP< Teuchos::ParameterList > &constructorParams, const Teuchos::RCP< Teuchos::ParameterList > &fillCompleteParams, const bool tolerant=false, const bool debug=false)
Read sparse graph from the given Matrix Market input stream.
static void writeSparseGraphFile(const std::string &filename, const Teuchos::RCP< const crs_graph_type > &pGraph, const bool debug=false)
Print the sparse graph in Matrix Market format to the given file (by filename), with no comments...
static void writeOperator(std::ostream &out, const operator_type &A)
Write a Tpetra::Operator to an output stream.
GlobalOrdinal getMinGlobalIndex() const
The minimum global index owned by the calling process.
Teuchos::RCP< const map_type > getColMap() const override
Returns the Map that describes the column distribution in this graph.
static void writeSparseFile(const std::string &filename, const Teuchos::RCP< const sparse_matrix_type > &pMatrix, const std::string &matrixName, const std::string &matrixDescription, const bool debug=false)
Print the sparse matrix in Matrix Market format, with comments.
SparseMatrixType::node_type node_type
The fourth template parameter of CrsMatrix and MultiVector.
static void writeDense(std::ostream &out, const multivector_type &X, const std::string &matrixName, const std::string &matrixDescription, const Teuchos::RCP< Teuchos::FancyOStream > &err=Teuchos::null, const Teuchos::RCP< Teuchos::FancyOStream > &dbg=Teuchos::null)
Print the multivector in Matrix Market format, with matrix name and description.
static Teuchos::RCP< sparse_matrix_type > readSparseFile(const std::string &filename, const Teuchos::RCP< const Teuchos::Comm< int > > &pComm, const bool callFillComplete=true, const bool tolerant=false, const bool debug=false)
Read sparse matrix from the given Matrix Market file.
bool isContiguous() const
True if this Map is distributed contiguously, else false.
void doImport(const SrcDistObject &source, const Import< LocalOrdinal, GlobalOrdinal, Node > &importer, const CombineMode CM, const bool restrictedMode=false)
Import data into this object using an Import object (&quot;forward mode&quot;).
size_t getNumVectors() const
Number of columns in the multivector.
SparseMatrixType::local_ordinal_type local_ordinal_type
SparseMatrixType::global_ordinal_type global_ordinal_type
One or more distributed dense vectors.
static Teuchos::RCP< sparse_matrix_type > readSparse(std::istream &in, const Teuchos::RCP< const Teuchos::Comm< int > > &pComm, const bool callFillComplete=true, const bool tolerant=false, const bool debug=false)
Read sparse matrix from the given Matrix Market input stream.
Teuchos::RCP< const map_type > getDomainMap() const override
Returns the Map associated with the domain of this graph.
SparseMatrixType::local_ordinal_type local_ordinal_type
Type of the local indices of the sparse matrix.
void getLocalRowView(const local_ordinal_type lclRow, Teuchos::ArrayView< const local_ordinal_type > &lclColInds) const override
Get a const, non-persisting view of the given local row&#39;s local column indices, as a Teuchos::ArrayVi...
static Teuchos::RCP< sparse_matrix_type > readSparseFile(const std::string &filename, const Teuchos::RCP< const Teuchos::Comm< int > > &pComm, const Teuchos::RCP< Teuchos::ParameterList > &constructorParams, const Teuchos::RCP< Teuchos::ParameterList > &fillCompleteParams, const bool tolerant=false, const bool debug=false)
Read sparse matrix from the given Matrix Market file.
static void writeDenseFile(const std::string &filename, const Teuchos::RCP< const multivector_type > &X, const std::string &matrixName, const std::string &matrixDescription, const Teuchos::RCP< Teuchos::FancyOStream > &err=Teuchos::null, const Teuchos::RCP< Teuchos::FancyOStream > &dbg=Teuchos::null)
Print the multivector in Matrix Market format, with matrix name and description.
SparseMatrixType::scalar_type scalar_type
Teuchos::RCP< const Teuchos::Comm< int > > getComm() const
Accessors for the Teuchos::Comm and Kokkos Node objects.
static Teuchos::RCP< sparse_graph_type > readSparseGraphFile(const std::string &filename, const Teuchos::RCP< const map_type > &rowMap, Teuchos::RCP< const map_type > &colMap, const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap, const bool callFillComplete=true, const bool tolerant=false, const bool debug=false)
Read sparse graph from the given Matrix Market file, with provided Maps.
MultiVector< scalar_type, local_ordinal_type, global_ordinal_type, node_type > multivector_type
Specialization of Tpetra::MultiVector that matches SparseMatrixType.
static Teuchos::RCP< const map_type > readMapFile(const std::string &filename, const Teuchos::RCP< const comm_type > &comm, const bool tolerant=false, const bool debug=false)
Read Map (as a MultiVector) from the given Matrix Market file.
bool isGloballyIndexed() const override
Whether the graph&#39;s column indices are stored as global indices.
static void writeMap(std::ostream &out, const map_type &map, const bool debug=false)
Print the Map to the given output stream.
static void writeDenseFile(const std::string &filename, const multivector_type &X, const Teuchos::RCP< Teuchos::FancyOStream > &err=Teuchos::null, const Teuchos::RCP< Teuchos::FancyOStream > &dbg=Teuchos::null)
Print the multivector in Matrix Market format, with no matrix name or description.
static void writeDenseFile(const std::string &filename, const Teuchos::RCP< const multivector_type > &X, const Teuchos::RCP< Teuchos::FancyOStream > &err=Teuchos::null, const Teuchos::RCP< Teuchos::FancyOStream > &dbg=Teuchos::null)
Print the multivector in Matrix Market format, with no matrix name or description.
static Teuchos::RCP< sparse_graph_type > readSparseGraphFile(const std::string &filename, const Teuchos::RCP< const Teuchos::Comm< int > > &comm, const bool callFillComplete=true, const bool tolerant=false, const bool debug=false)
Read sparse graph from the given Matrix Market file.
SparseMatrixType::global_ordinal_type global_ordinal_type
Type of indices as read from the Matrix Market file.
static Teuchos::RCP< sparse_matrix_type > readSparse(std::istream &in, const Teuchos::RCP< const map_type > &rowMap, Teuchos::RCP< const map_type > &colMap, const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap, const bool callFillComplete=true, const bool tolerant=false, const bool debug=false)
Read sparse matrix from the given Matrix Market input stream, with provided Maps. ...
static Teuchos::RCP< multivector_type > readDense(std::istream &in, const Teuchos::RCP< const comm_type > &comm, Teuchos::RCP< const map_type > &map, const bool tolerant=false, const bool debug=false)
Read dense matrix (as a MultiVector) from the given Matrix Market input stream.
void gathervPrint(std::ostream &out, const std::string &s, const Teuchos::Comm< int > &comm)
On Process 0 in the given communicator, print strings from each process in that communicator, in rank order.
static void writeSparseGraph(std::ostream &out, const crs_graph_type &graph, const bool debug=false)
Print the sparse graph in Matrix Market format to the given output stream, with no comments...
size_t global_size_t
Global size_t object.
static Teuchos::RCP< vector_type > readVector(std::istream &in, const Teuchos::RCP< const comm_type > &comm, Teuchos::RCP< const map_type > &map, const bool tolerant=false, const bool debug=false)
Read Vector from the given Matrix Market input stream.
SparseMatrixType::node_type node_type
The Kokkos Node type; fourth template parameter of Tpetra::CrsMatrix.
static void writeOperator(const std::string &fileName, const operator_type &A, const Teuchos::ParameterList &params)
Write a Tpetra::Operator to a file, with options.
static Teuchos::RCP< const map_type > readMap(std::istream &in, const Teuchos::RCP< const comm_type > &comm, const bool tolerant=false, const bool debug=false)
Read Map (as a MultiVector) from the given input stream.
Map< local_ordinal_type, global_ordinal_type, node_type > map_type
Specialization of Tpetra::Map that matches SparseMatrixType.
Insert new values that don&#39;t currently exist.
SparseMatrixType::scalar_type scalar_type
Type of the entries of the sparse matrix.
static void writeMapFile(const std::string &filename, const map_type &map)
Write the Map to the given file.
Abstract interface for operators (e.g., matrices and preconditioners).
static void writeSparseGraphFile(const std::string &filename, const Teuchos::RCP< const crs_graph_type > &pGraph, const std::string &graphName, const std::string &graphDescription, const bool debug=false)
Print the sparse graph in Matrix Market format to the given file (by filename), taking the graph by T...
CrsGraph< local_ordinal_type, global_ordinal_type, node_type > crs_graph_type
Specialization of Tpetra::CrsGraph that matches SparseMatrixType.
static void writeDense(std::ostream &out, const Teuchos::RCP< const multivector_type > &X, const Teuchos::RCP< Teuchos::FancyOStream > &err=Teuchos::null, const Teuchos::RCP< Teuchos::FancyOStream > &dbg=Teuchos::null)
Print the multivector in Matrix Market format, with no matrix name or description.
static Teuchos::RCP< sparse_matrix_type > readSparseFile(const std::string &filename, const Teuchos::RCP< const map_type > &rowMap, Teuchos::RCP< const map_type > &colMap, const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap, const bool callFillComplete=true, const bool tolerant=false, const bool debug=false)
Read sparse matrix from the given Matrix Market file, with provided Maps.
Communication plan for data redistribution from a (possibly) multiply-owned to a uniquely-owned distr...
From a distributed map build a map with all GIDs on the root node.
virtual Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getDomainMap() const =0
The Map associated with the domain of this operator, which must be compatible with X...
static Teuchos::RCP< sparse_graph_type > readSparseGraphFile(const std::string &filename, const Teuchos::RCP< const Teuchos::Comm< int > > &pComm, const Teuchos::RCP< Teuchos::ParameterList > &constructorParams, const Teuchos::RCP< Teuchos::ParameterList > &fillCompleteParams, const bool tolerant=false, const bool debug=false)
Read sparse graph from the given Matrix Market file.
static void writeMap(std::ostream &out, const map_type &map, const Teuchos::RCP< Teuchos::FancyOStream > &err, const bool debug=false)
Print the Map to the given output stream out.
static Teuchos::RCP< const map_type > readMap(std::istream &in, const Teuchos::RCP< const comm_type > &comm, const Teuchos::RCP< Teuchos::FancyOStream > &err, const bool tolerant=false, const bool debug=false)
Read Map (as a MultiVector) from the given input stream, with optional debugging output stream...
static Teuchos::RCP< multivector_type > readDenseFile(const std::string &filename, const Teuchos::RCP< const comm_type > &comm, Teuchos::RCP< const map_type > &map, const bool tolerant=false, const bool debug=false)
Read dense matrix (as a MultiVector) from the given Matrix Market file.
static void writeSparseGraphFile(const std::string &filename, const crs_graph_type &graph, const bool debug=false)
Print the sparse graph in Matrix Market format to the given file (by filename), with no comments...
Teuchos::RCP< const map_type > getRangeMap() const override
Returns the Map associated with the domain of this graph.
static void writeDense(std::ostream &out, const multivector_type &X, const Teuchos::RCP< Teuchos::FancyOStream > &err=Teuchos::null, const Teuchos::RCP< Teuchos::FancyOStream > &dbg=Teuchos::null)
Print the multivector in Matrix Market format, with no matrix name or description.
static void writeSparse(std::ostream &out, const Teuchos::RCP< const sparse_matrix_type > &pMatrix, const std::string &matrixName, const std::string &matrixDescription, const bool debug=false)
Print the sparse matrix in Matrix Market format, with comments.
void getGlobalRowView(const global_ordinal_type gblRow, Teuchos::ArrayView< const global_ordinal_type > &gblColInds) const override
Get a const, non-persisting view of the given global row&#39;s global column indices, as a Teuchos::Array...
static void writeSparseFile(const std::string &filename, const Teuchos::RCP< const sparse_matrix_type > &pMatrix, const bool debug=false)
Print the sparse matrix in Matrix Market format.
Teuchos::RCP< const Vector< Scalar, LocalOrdinal, GlobalOrdinal, Node > > getVector(const size_t j) const
Return a Vector which is a const view of column j.
MultiVector< scalar_type, local_ordinal_type, global_ordinal_type, node_type > multivector_type
The MultiVector specialization associated with SparseMatrixType.
static void writeSparseGraphFile(const std::string &filename, const crs_graph_type &graph, const std::string &graphName, const std::string &graphDescription, const bool debug=false)
Print the sparse graph in Matrix Market format to the given file (by filename).
Teuchos::ArrayView< const GlobalOrdinal > getNodeElementList() const
Return a NONOWNING view of the global indices owned by this process.
A distributed graph accessed by rows (adjacency lists) and stored sparsely.
Vector< scalar_type, local_ordinal_type, global_ordinal_type, node_type > vector_type
The Vector specialization associated with SparseMatrixType.
size_t getNodeNumElements() const
The number of elements belonging to the calling process.
A parallel distribution of indices over processes.
static Teuchos::RCP< sparse_graph_type > readSparseGraph(std::istream &in, const Teuchos::RCP< const Teuchos::Comm< int > > &pComm, const bool callFillComplete=true, const bool tolerant=false, const bool debug=false)
Read sparse graph from the given Matrix Market input stream.
void fillComplete(const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null)
Tell the graph that you are done changing its structure.
static Teuchos::RCP< sparse_matrix_type > readSparse(std::istream &in, const Teuchos::RCP< const Teuchos::Comm< int > > &pComm, const Teuchos::RCP< Teuchos::ParameterList > &constructorParams, const Teuchos::RCP< Teuchos::ParameterList > &fillCompleteParams, const bool tolerant=false, const bool debug=false)
Read sparse matrix from the given Matrix Market input stream.
A distributed dense vector.
static void writeSparseGraph(std::ostream &out, const crs_graph_type &graph, const std::string &graphName, const std::string &graphDescription, const bool debug=false)
Print the sparse graph in Matrix Market format to the given output stream.
CrsGraph< local_ordinal_type, global_ordinal_type, node_type > sparse_graph_type
The CrsGraph specialization associated with SparseMatrixType.
Teuchos::RCP< const Teuchos::Comm< int > > getComm() const override
Returns the communicator.
SparseMatrixType sparse_matrix_type
This class&#39; template parameter; a specialization of CrsMatrix.
virtual Teuchos::RCP< const map_type > getMap() const
The Map describing the parallel distribution of this object.
SparseMatrixType sparse_matrix_type
Template parameter of this class; specialization of CrsMatrix.
static Teuchos::RCP< sparse_graph_type > readSparseGraph(std::istream &in, const Teuchos::RCP< const map_type > &rowMap, Teuchos::RCP< const map_type > &colMap, const Teuchos::RCP< const map_type > &domainMap, const Teuchos::RCP< const map_type > &rangeMap, const bool callFillComplete=true, const bool tolerant=false, const bool debug=false)
Read sparse matrix from the given Matrix Market input stream, with provided Maps. ...
static void writeDenseFile(const std::string &filename, const multivector_type &X, const std::string &matrixName, const std::string &matrixDescription, const Teuchos::RCP< Teuchos::FancyOStream > &err=Teuchos::null, const Teuchos::RCP< Teuchos::FancyOStream > &dbg=Teuchos::null)
Print the multivector in Matrix Market format, with matrix name and description.
static void writeOperator(std::ostream &out, const operator_type &A, const Teuchos::ParameterList &params)
Write a Tpetra::Operator to an output stream, with options.
global_size_t getGlobalNumEntries() const override
Returns the global number of entries in the graph.
Matrix Market file reader for CrsMatrix and MultiVector.
Matrix Market file writer for CrsMatrix and MultiVector.
static void writeSparse(std::ostream &out, const Teuchos::RCP< const sparse_matrix_type > &pMatrix, const bool debug=false)
Print the sparse matrix in Matrix Market format.
static void writeDense(std::ostream &out, const Teuchos::RCP< const multivector_type > &X, const std::string &matrixName, const std::string &matrixDescription, const Teuchos::RCP< Teuchos::FancyOStream > &err=Teuchos::null, const Teuchos::RCP< Teuchos::FancyOStream > &dbg=Teuchos::null)
Print the multivector in Matrix Market format, with matrix name and or description.
global_size_t getGlobalNumElements() const
The number of elements in this Map.
static void writeOperator(const std::string &fileName, operator_type const &A)
Write a Tpetra::Operator to a file.