45 #ifndef AMESOS2_STRUMPACK_DEF_HPP
46 #define AMESOS2_STRUMPACK_DEF_HPP
48 #include <Teuchos_Tuple.hpp>
49 #include <Teuchos_StandardParameterEntryValidators.hpp>
56 #include <Teuchos_DefaultMpiComm.hpp>
57 #include "StrumpackSparseSolverMPIDist.hpp"
59 #include "StrumpackSparseSolver.hpp"
65 template <
class Matrix,
class Vector>
67 Teuchos::RCP<Vector> X,
68 Teuchos::RCP<const Vector> B)
74 using Teuchos::MpiComm;
76 using Teuchos::ParameterList;
77 using Teuchos::parameterList;
80 using Teuchos::rcp_dynamic_cast;
81 typedef global_ordinal_type GO;
83 typedef Tpetra::Map<local_ordinal_type, GO, node_type> map_type;
85 RCP<const Comm<int> > comm = this->
getComm ();
88 RCP<const MpiComm<int> > mpiComm =
89 rcp_dynamic_cast<
const MpiComm<int> > (comm);
90 TEUCHOS_TEST_FOR_EXCEPTION
91 (mpiComm.is_null (), std::logic_error,
"Amesos2::STRUMPACK "
92 "constructor: The matrix's communicator is not an MpiComm!");
93 MPI_Comm rawMpiComm = (* (mpiComm->getRawMpiComm ())) ();
95 sp_ = Teuchos::RCP<strumpack::StrumpackSparseSolverMPIDist<scalar_type,GO>>
97 (
new strumpack::StrumpackSparseSolverMPIDist<scalar_type,GO>(rawMpiComm,
true));
99 sp_ = Teuchos::RCP<strumpack::StrumpackSparseSolver<scalar_type,GO>>
100 (
new strumpack::StrumpackSparseSolver<scalar_type,GO>(this->
control_.verbose_, this->
root_));
108 RCP<ParameterList> default_params =
113 const size_t myNumRows = this->
matrixA_->getLocalNumRows();
114 const GO indexBase = this->
matrixA_->getRowMap ()->getIndexBase ();
116 rcp (
new map_type (this->
globalNumRows_, myNumRows, indexBase, comm));
125 template <
class Matrix,
class Vector>
134 template<
class Matrix,
class Vector>
138 #ifdef HAVE_AMESOS2_TIMERS
139 Teuchos::TimeMonitor preOrderTime( this->timers_.preOrderTime_ );
154 template <
class Matrix,
class Vector>
158 #ifdef HAVE_AMESOS2_TIMERS
159 Teuchos::TimeMonitor symFactTime( this->timers_.symFactTime_ );
162 strumpack::ReturnCode ret = sp_->reorder();
163 TEUCHOS_TEST_FOR_EXCEPTION( ret == strumpack::ReturnCode::MATRIX_NOT_SET,
165 "STRUMPACK matrix reordering failed: "
166 "matrix was not set." );
167 TEUCHOS_TEST_FOR_EXCEPTION( ret == strumpack::ReturnCode::REORDERING_ERROR,
169 "STRUMPACK matrix reordering failed." );
179 template <
class Matrix,
class Vector>
183 #ifdef HAVE_AMESOS2_TIMERS
184 Teuchos::TimeMonitor numFactTimer(this->timers_.numFactTime_);
187 strumpack::ReturnCode ret = sp_->factor();
189 TEUCHOS_TEST_FOR_EXCEPTION( ret != strumpack::ReturnCode::SUCCESS,
191 "Error in STRUMPACK factorization." );
201 template <
class Matrix,
class Vector>
210 const size_t local_len_rhs = strumpack_rowmap_->getLocalNumElements();
211 const global_size_type nrhs = X->getGlobalNumVectors();
214 bvals_.resize(nrhs * local_len_rhs);
215 xvals_.resize(nrhs * local_len_rhs);
219 #ifdef HAVE_AMESOS2_TIMERS
220 Teuchos::TimeMonitor convTimer(this->timers_.vecConvTime_);
224 #ifdef HAVE_AMESOS2_TIMERS
225 Teuchos::TimeMonitor redistTimer(this->timers_.vecRedistTime_);
229 copy_helper::do_get(B,
232 Teuchos::ptrInArg(*strumpack_rowmap_));
237 #ifdef HAVE_AMESOS2_TIMERS
238 Teuchos::TimeMonitor solveTimer(this->timers_.solveTime_);
240 strumpack::DenseMatrixWrapper<scalar_type>
241 Bsp(local_len_rhs, nrhs, bvals_().getRawPtr(), local_len_rhs),
242 Xsp(local_len_rhs, nrhs, xvals_().getRawPtr(), local_len_rhs);
243 strumpack::ReturnCode ret =sp_->solve(Bsp, Xsp);
245 TEUCHOS_TEST_FOR_EXCEPTION( ret != strumpack::ReturnCode::SUCCESS,
247 "Error in STRUMPACK solve" );
252 #ifdef HAVE_AMESOS2_TIMERS
253 Teuchos::TimeMonitor redistTimer(this->timers_.vecRedistTime_);
258 put_helper::do_put(X,
261 Teuchos::ptrInArg(*strumpack_rowmap_));
265 const global_size_type ld_rhs = this->root_ ? X->getGlobalLength() : 0;
266 const size_t nrhs = X->getGlobalNumVectors();
267 bvals_.resize(nrhs * ld_rhs);
268 xvals_.resize(nrhs * ld_rhs);
271 #ifdef HAVE_AMESOS2_TIMERS
272 Teuchos::TimeMonitor solveTimer(this->timers_.solveTime_);
275 strumpack::DenseMatrixWrapper<scalar_type>
276 Bsp(ld_rhs, nrhs, bvals_().getRawPtr(), ld_rhs),
277 Xsp(ld_rhs, nrhs, xvals_().getRawPtr(), ld_rhs);
278 strumpack::ReturnCode ret =sp_->solve(Bsp, Xsp);
280 TEUCHOS_TEST_FOR_EXCEPTION( ret != strumpack::ReturnCode::SUCCESS,
282 "Error in STRUMPACK solve" );
286 #ifdef HAVE_AMESOS2_TIMERS
287 Teuchos::TimeMonitor redistTimer(this->timers_.vecRedistTime_);
293 ROOTED, this->rowIndexBase_);
300 template <
class Matrix,
class Vector>
306 return( this->globalNumRows_ == this->globalNumCols_ );
308 return( this->matrixA_->getGlobalNumRows() == this->matrixA_->getGlobalNumCols() );
318 template <
class Matrix,
class Vector>
324 using Teuchos::getIntegralValue;
325 using Teuchos::ParameterEntryValidator;
327 RCP<const Teuchos::ParameterList> valid_params = getValidParameters_impl();
329 if( parameterList->isParameter(
"Matching") ){
330 RCP<const ParameterEntryValidator> matching_validator = valid_params->getEntry(
"Matching").validator();
331 parameterList->getEntry(
"Matching").setValidator(matching_validator);
333 sp_->options().set_matching(getIntegralValue<strumpack::MatchingJob>(*parameterList,
"Matching"));
336 if( parameterList->isParameter(
"Ordering") ){
337 RCP<const ParameterEntryValidator> reordering_validator = valid_params->getEntry(
"Ordering").validator();
338 parameterList->getEntry(
"Ordering").setValidator(reordering_validator);
340 sp_->options().set_reordering_method(getIntegralValue<strumpack::ReorderingStrategy>(*parameterList,
"Ordering"));
343 if( parameterList->isParameter(
"ReplaceTinyPivot") ){
344 RCP<const ParameterEntryValidator> replacepivot_validator = valid_params->getEntry(
"ReplaceTinyPivot").validator();
345 parameterList->getEntry(
"ReplaceTinyPivot").setValidator(replacepivot_validator);
347 if( replacepivot_validator) {
348 sp_->options().enable_replace_tiny_pivots();
351 sp_->options().disable_replace_tiny_pivots();
355 if( parameterList->isParameter(
"IterRefine") ){
356 RCP<const ParameterEntryValidator> iter_refine_validator = valid_params->getEntry(
"IterRefine").validator();
357 parameterList->getEntry(
"IterRefine").setValidator(iter_refine_validator);
359 sp_->options().set_Krylov_solver(getIntegralValue<strumpack::KrylovSolver>(*parameterList,
"IterRefine"));
362 if( parameterList->isParameter(
"Compression") ){
363 RCP<const ParameterEntryValidator> compression_validator = valid_params->getEntry(
"Compression").validator();
364 parameterList->getEntry(
"Compression").setValidator(compression_validator);
366 sp_->options().set_compression(getIntegralValue<strumpack::CompressionType>(*parameterList,
"Compression"));
369 TEUCHOS_TEST_FOR_EXCEPTION( this->control_.useTranspose_,
370 std::invalid_argument,
371 "STRUMPACK does not support solving the tranpose system" );
375 template <
class Matrix,
class Vector>
376 Teuchos::RCP<const Teuchos::ParameterList>
380 using Teuchos::tuple;
381 using Teuchos::ParameterList;
382 using Teuchos::EnhancedNumberValidator;
383 using Teuchos::setStringToIntegralParameter;
384 using Teuchos::stringToIntegralParameterEntryValidator;
386 static Teuchos::RCP<const Teuchos::ParameterList> valid_params;
388 if( is_null(valid_params) ){
389 Teuchos::RCP<Teuchos::ParameterList> pl = Teuchos::parameterList();
391 setStringToIntegralParameter<strumpack::MatchingJob>(
"Matching",
"NONE",
392 "Specifies how to permute the "
393 "matrix for numerical stability",
394 tuple<string>(
"NONE",
"MAX_CARDINALITY",
"MAX_SMALLEST_DIAGONAL",
"MAX_SMALLEST_DIAGONAL_2",
"MAX_DIAGONAL_SUM",
"MAX_DIAGONAL_PRODUCT_SCALING",
"COMBBLAS"),
395 tuple<string>(
"NONE",
"MAX_CARDINALITY",
"MAX_SMALLEST_DIAGONAL",
"MAX_SMALLEST_DIAGONAL_2",
"MAX_DIAGONAL_SUM",
"MAX_DIAGONAL_PRODUCT_SCALING",
"COMBBLAS"),
396 tuple<strumpack::MatchingJob>(strumpack::MatchingJob::NONE,
397 strumpack::MatchingJob::MAX_CARDINALITY,
398 strumpack::MatchingJob::MAX_SMALLEST_DIAGONAL,
399 strumpack::MatchingJob::MAX_SMALLEST_DIAGONAL_2,
400 strumpack::MatchingJob::MAX_DIAGONAL_SUM,
401 strumpack::MatchingJob::MAX_DIAGONAL_PRODUCT_SCALING,
402 strumpack::MatchingJob::COMBBLAS),
405 #if defined(STRUMPACK_USE_PARMETIS)
406 std::string default_ordering(
"PARMETIS");
408 std::string default_ordering(
"METIS");
410 setStringToIntegralParameter<strumpack::ReorderingStrategy>(
"Ordering", default_ordering,
411 "Specifies how to permute the "
412 "matrix for sparsity preservation",
413 tuple<string>(
"NATURAL",
"PARMETIS",
"METIS",
"SCOTCH",
"GEOMETRIC",
"PTSCOTCH",
"RCM"),
414 tuple<string>(
"Natural ordering",
418 "Geometric ordering",
421 tuple<strumpack::ReorderingStrategy>(strumpack::ReorderingStrategy::NATURAL,
422 strumpack::ReorderingStrategy::PARMETIS,
423 strumpack::ReorderingStrategy::METIS,
424 strumpack::ReorderingStrategy::SCOTCH,
425 strumpack::ReorderingStrategy::GEOMETRIC,
426 strumpack::ReorderingStrategy::PTSCOTCH,
427 strumpack::ReorderingStrategy::RCM),
430 pl->set(
"ReplaceTinyPivot",
true,
"Specifies whether to replace tiny diagonals during LU factorization");
435 setStringToIntegralParameter<strumpack::KrylovSolver>(
"IterRefine",
"DIRECT",
436 "Type of iterative refinement to use",
437 tuple<string>(
"AUTO",
"DIRECT",
"REFINE",
"PREC_GMRES",
"GMRES",
"PREC_BICGSTAB",
"BICGSTAB"),
438 tuple<string>(
"Use iterative refinement if no compression is used, otherwise use GMRes.",
439 "Single application of the multifrontal solver.",
440 "Iterative refinement.",
441 "Preconditioned GMRes.",
442 "UN-preconditioned GMRes.",
443 "Preconditioned BiCGStab.",
444 "UN-preconditioned BiCGStab."),
445 tuple<strumpack::KrylovSolver>(strumpack::KrylovSolver::AUTO,
446 strumpack::KrylovSolver::DIRECT,
447 strumpack::KrylovSolver::REFINE,
448 strumpack::KrylovSolver::PREC_GMRES,
449 strumpack::KrylovSolver::GMRES,
450 strumpack::KrylovSolver::PREC_BICGSTAB,
451 strumpack::KrylovSolver::BICGSTAB),
456 setStringToIntegralParameter<strumpack::CompressionType>(
"Compression",
"NONE",
457 "Type of compression to use",
458 tuple<string>(
"NONE",
"HSS",
"BLR",
"HODLR",
"LOSSLESS",
"LOSSY"),
459 tuple<string>(
"No compression, purely direct solver.",
460 "HSS compression of frontal matrices.",
461 "Block low-rank compression of fronts.",
462 "Hierarchically Off-diagonal Low-Rank compression of frontal matrices.",
463 "Lossless compresssion.",
464 "Lossy compresssion."),
465 tuple<strumpack::CompressionType>(strumpack::CompressionType::NONE,
466 strumpack::CompressionType::HSS,
467 strumpack::CompressionType::BLR,
468 strumpack::CompressionType::HODLR,
469 strumpack::CompressionType::LOSSLESS,
470 strumpack::CompressionType::LOSSY),
488 template <
class Matrix,
class Vector>
491 using Teuchos::Array;
492 using Teuchos::ArrayView;
493 using Teuchos::ptrInArg;
495 using Teuchos::rcp_dynamic_cast;
499 using Teuchos::MpiComm;
501 #ifdef HAVE_AMESOS2_TIMERS
502 Teuchos::TimeMonitor convTimer(this->timers_.mtxConvTime_);
505 Teuchos::RCP<const MatrixAdapter<Matrix> > redist_mat
506 = this->matrixA_->get(ptrInArg(*strumpack_rowmap_));
508 typedef global_ordinal_type GO;
510 l_nnz = as<GO>(redist_mat->getLocalNNZ());
511 l_rows = as<GO>(redist_mat->getLocalNumRows());
513 RCP<const Comm<int> > comm = this->getComm ();
514 RCP<const MpiComm<int> > mpiComm =
515 rcp_dynamic_cast<
const MpiComm<int> > (comm);
517 const int numProcs = comm->getSize ();
518 const int myRank = comm->getRank ();
519 Array<GO> dist(numProcs+1);
521 dist[myRank+1] = as<GO>(strumpack_rowmap_->getMaxGlobalIndex()) + 1;
523 MPI_Allgather(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, dist.data()+1,
sizeof(GO), MPI_BYTE,
524 mpiComm->getRawMpiComm()->operator()());
525 Kokkos::resize(nzvals_view_, l_nnz);
526 Kokkos::resize(colind_view_, l_nnz);
527 Kokkos::resize(rowptr_view_, l_rows + 1);
532 #ifdef HAVE_AMESOS2_TIMERS
533 Teuchos::TimeMonitor mtxRedistTimer( this->timers_.mtxRedistTime_ );
537 host_value_type_array, host_ordinal_type_array, host_ordinal_type_array >::do_get(
539 nzvals_view_, colind_view_, rowptr_view_,
541 ptrInArg(*strumpack_rowmap_),
547 TEUCHOS_TEST_FOR_EXCEPTION( nnz_ret != l_nnz,
549 "Did not get the expected number of non-zero vals");
552 sp_->set_distributed_csr_matrix
553 (l_rows, rowptr_view_.data(), colind_view_.data(),
554 nzvals_view_.data(), dist.getRawPtr(),
false);
557 #ifdef HAVE_AMESOS2_TIMERS
558 Teuchos::TimeMonitor convTimer(this->timers_.mtxConvTime_);
561 typedef global_ordinal_type GO;
565 Kokkos::resize(nzvals_view_, this->globalNumNonZeros_);
566 Kokkos::resize(colind_view_, this->globalNumNonZeros_);
567 Kokkos::resize(rowptr_view_, this->globalNumRows_ + 1);
570 #ifdef HAVE_AMESOS2_TIMERS
571 Teuchos::TimeMonitor mtxRedistTimer( this->timers_.mtxRedistTime_ );
575 host_value_type_array, host_ordinal_type_array, host_ordinal_type_array >::do_get(
576 this->matrixA_.ptr(),
577 nzvals_view_, colind_view_, rowptr_view_,
580 ARBITRARY, this->rowIndexBase_);
583 TEUCHOS_TEST_FOR_EXCEPTION( nnz_ret != this->globalNumNonZeros_,
585 "Did not get the expected number of non-zero vals");
588 sp_->set_csr_matrix(this->globalNumRows_, rowptr_view_.data(), colind_view_.data(),
589 nzvals_view_.data(),
false);
596 template<
class Matrix,
class Vector>
601 #endif // AMESOS2_STRUMPACK_DEF_HPP
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const override
Return a const parameter list of all of the valid parameters that this->setParameterList(...) will accept.
Definition: Amesos2_SolverCore_def.hpp:558
Amesos2::SolverCore: A templated interface for interaction with third-party direct sparse solvers...
Definition: Amesos2_SolverCore_decl.hpp:105
STRUMPACK(Teuchos::RCP< const Matrix > A, Teuchos::RCP< Vector > X, Teuchos::RCP< const Vector > B)
Initialize from Teuchos::RCP.
Definition: Amesos2_STRUMPACK_def.hpp:66
int preOrdering_impl()
Performs pre-ordering on the matrix to increase efficiency.
Definition: Amesos2_STRUMPACK_def.hpp:136
EPhase
Used to indicate a phase in the direct solution.
Definition: Amesos2_TypeDecl.hpp:65
super_type & setParameters(const Teuchos::RCP< Teuchos::ParameterList > ¶meterList) override
Set/update internal variables and solver options.
Definition: Amesos2_SolverCore_def.hpp:526
void setParameters_impl(const Teuchos::RCP< Teuchos::ParameterList > ¶meterList)
Definition: Amesos2_STRUMPACK_def.hpp:320
bool root_
If true, then this is the root processor.
Definition: Amesos2_SolverCore_decl.hpp:506
global_size_type globalNumRows_
Number of global rows in matrixA_.
Definition: Amesos2_SolverCore_decl.hpp:476
Helper class for getting 1-D copies of multivectors.
Definition: Amesos2_MultiVecAdapter_decl.hpp:267
int symbolicFactorization_impl()
Perform symbolic factorization of the matrix using STRUMPACK.
Definition: Amesos2_STRUMPACK_def.hpp:156
Utility functions for Amesos2.
Control control_
Parameters for solving.
Definition: Amesos2_SolverCore_decl.hpp:494
Teuchos::RCP< const Teuchos::Comm< int > > getComm() const override
Returns a pointer to the Teuchos::Comm communicator with this operator.
Definition: Amesos2_SolverCore_decl.hpp:363
bool matrixShapeOK_impl() const
Determines whether the shape of the matrix is OK for this solver.
Definition: Amesos2_STRUMPACK_def.hpp:302
bool loadA_impl(EPhase current_phase)
Reads matrix data into internal solver structures.
Definition: Amesos2_STRUMPACK_def.hpp:490
Similar to get_ccs_helper , but used to get a CRS representation of the given matrix.
Definition: Amesos2_Util.hpp:659
Amesos2 interface to STRUMPACK direct solver and preconditioner.
Definition: Amesos2_STRUMPACK_decl.hpp:71
~STRUMPACK()
Destructor.
Definition: Amesos2_STRUMPACK_def.hpp:126
Helper class for putting 1-D data arrays into multivectors.
Definition: Amesos2_MultiVecAdapter_decl.hpp:373
A templated MultiVector class adapter for Amesos2.
Definition: Amesos2_MultiVecAdapter_decl.hpp:176
Teuchos::RCP< const MatrixAdapter< Matrix > > matrixA_
The LHS operator.
Definition: Amesos2_SolverCore_decl.hpp:455
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters_impl() const
Definition: Amesos2_STRUMPACK_def.hpp:377
int solve_impl(const Teuchos::Ptr< MultiVecAdapter< Vector > > X, const Teuchos::Ptr< const MultiVecAdapter< Vector > > B) const
STRUMPACK specific solve.
Definition: Amesos2_STRUMPACK_def.hpp:203
int numericFactorization_impl()
STRUMPACK specific numeric factorization.
Definition: Amesos2_STRUMPACK_def.hpp:181