11 #ifndef AMESOS2_STRUMPACK_DEF_HPP
12 #define AMESOS2_STRUMPACK_DEF_HPP
14 #include <Teuchos_Tuple.hpp>
15 #include <Teuchos_StandardParameterEntryValidators.hpp>
22 #include <Teuchos_DefaultMpiComm.hpp>
23 #include "StrumpackSparseSolverMPIDist.hpp"
25 #include "StrumpackSparseSolver.hpp"
31 template <
class Matrix,
class Vector>
33 Teuchos::RCP<Vector> X,
34 Teuchos::RCP<const Vector> B)
40 using Teuchos::MpiComm;
42 using Teuchos::ParameterList;
43 using Teuchos::parameterList;
46 using Teuchos::rcp_dynamic_cast;
47 typedef global_ordinal_type GO;
49 typedef Tpetra::Map<local_ordinal_type, GO, node_type> map_type;
51 RCP<const Comm<int> > comm = this->
getComm ();
54 RCP<const MpiComm<int> > mpiComm =
55 rcp_dynamic_cast<
const MpiComm<int> > (comm);
56 TEUCHOS_TEST_FOR_EXCEPTION
57 (mpiComm.is_null (), std::logic_error,
"Amesos2::STRUMPACK "
58 "constructor: The matrix's communicator is not an MpiComm!");
59 MPI_Comm rawMpiComm = (* (mpiComm->getRawMpiComm ())) ();
61 sp_ = Teuchos::RCP<strumpack::StrumpackSparseSolverMPIDist<scalar_type,GO>>
63 (
new strumpack::StrumpackSparseSolverMPIDist<scalar_type,GO>(rawMpiComm,
true));
65 sp_ = Teuchos::RCP<strumpack::StrumpackSparseSolver<scalar_type,GO>>
66 (
new strumpack::StrumpackSparseSolver<scalar_type,GO>(this->
control_.verbose_, this->
root_));
74 RCP<ParameterList> default_params =
79 const size_t myNumRows = this->
matrixA_->getLocalNumRows();
80 const GO indexBase = this->
matrixA_->getRowMap ()->getIndexBase ();
82 rcp (
new map_type (this->
globalNumRows_, myNumRows, indexBase, comm));
91 template <
class Matrix,
class Vector>
100 template<
class Matrix,
class Vector>
104 #ifdef HAVE_AMESOS2_TIMERS
105 Teuchos::TimeMonitor preOrderTime( this->timers_.preOrderTime_ );
120 template <
class Matrix,
class Vector>
124 #ifdef HAVE_AMESOS2_TIMERS
125 Teuchos::TimeMonitor symFactTime( this->timers_.symFactTime_ );
128 strumpack::ReturnCode ret = sp_->reorder();
129 TEUCHOS_TEST_FOR_EXCEPTION( ret == strumpack::ReturnCode::MATRIX_NOT_SET,
131 "STRUMPACK matrix reordering failed: "
132 "matrix was not set." );
133 TEUCHOS_TEST_FOR_EXCEPTION( ret == strumpack::ReturnCode::REORDERING_ERROR,
135 "STRUMPACK matrix reordering failed." );
145 template <
class Matrix,
class Vector>
149 #ifdef HAVE_AMESOS2_TIMERS
150 Teuchos::TimeMonitor numFactTimer(this->timers_.numFactTime_);
153 strumpack::ReturnCode ret = sp_->factor();
155 TEUCHOS_TEST_FOR_EXCEPTION( ret != strumpack::ReturnCode::SUCCESS,
157 "Error in STRUMPACK factorization." );
167 template <
class Matrix,
class Vector>
176 const size_t local_len_rhs = strumpack_rowmap_->getLocalNumElements();
177 const global_size_type nrhs = X->getGlobalNumVectors();
180 bvals_.resize(nrhs * local_len_rhs);
181 xvals_.resize(nrhs * local_len_rhs);
185 #ifdef HAVE_AMESOS2_TIMERS
186 Teuchos::TimeMonitor convTimer(this->timers_.vecConvTime_);
190 #ifdef HAVE_AMESOS2_TIMERS
191 Teuchos::TimeMonitor redistTimer(this->timers_.vecRedistTime_);
195 copy_helper::do_get(B,
198 Teuchos::ptrInArg(*strumpack_rowmap_));
203 #ifdef HAVE_AMESOS2_TIMERS
204 Teuchos::TimeMonitor solveTimer(this->timers_.solveTime_);
206 strumpack::DenseMatrixWrapper<scalar_type>
207 Bsp(local_len_rhs, nrhs, bvals_().getRawPtr(), local_len_rhs),
208 Xsp(local_len_rhs, nrhs, xvals_().getRawPtr(), local_len_rhs);
209 strumpack::ReturnCode ret =sp_->solve(Bsp, Xsp);
211 TEUCHOS_TEST_FOR_EXCEPTION( ret != strumpack::ReturnCode::SUCCESS,
213 "Error in STRUMPACK solve" );
218 #ifdef HAVE_AMESOS2_TIMERS
219 Teuchos::TimeMonitor redistTimer(this->timers_.vecRedistTime_);
224 put_helper::do_put(X,
227 Teuchos::ptrInArg(*strumpack_rowmap_));
231 const global_size_type ld_rhs = this->root_ ? X->getGlobalLength() : 0;
232 const size_t nrhs = X->getGlobalNumVectors();
233 bvals_.resize(nrhs * ld_rhs);
234 xvals_.resize(nrhs * ld_rhs);
237 #ifdef HAVE_AMESOS2_TIMERS
238 Teuchos::TimeMonitor solveTimer(this->timers_.solveTime_);
241 strumpack::DenseMatrixWrapper<scalar_type>
242 Bsp(ld_rhs, nrhs, bvals_().getRawPtr(), ld_rhs),
243 Xsp(ld_rhs, nrhs, xvals_().getRawPtr(), ld_rhs);
244 strumpack::ReturnCode ret =sp_->solve(Bsp, Xsp);
246 TEUCHOS_TEST_FOR_EXCEPTION( ret != strumpack::ReturnCode::SUCCESS,
248 "Error in STRUMPACK solve" );
252 #ifdef HAVE_AMESOS2_TIMERS
253 Teuchos::TimeMonitor redistTimer(this->timers_.vecRedistTime_);
259 ROOTED, this->rowIndexBase_);
266 template <
class Matrix,
class Vector>
272 return( this->globalNumRows_ == this->globalNumCols_ );
274 return( this->matrixA_->getGlobalNumRows() == this->matrixA_->getGlobalNumCols() );
284 template <
class Matrix,
class Vector>
290 using Teuchos::getIntegralValue;
291 using Teuchos::ParameterEntryValidator;
293 RCP<const Teuchos::ParameterList> valid_params = getValidParameters_impl();
295 if( parameterList->isParameter(
"Matching") ){
296 RCP<const ParameterEntryValidator> matching_validator = valid_params->getEntry(
"Matching").validator();
297 parameterList->getEntry(
"Matching").setValidator(matching_validator);
299 sp_->options().set_matching(getIntegralValue<strumpack::MatchingJob>(*parameterList,
"Matching"));
302 if( parameterList->isParameter(
"Ordering") ){
303 RCP<const ParameterEntryValidator> reordering_validator = valid_params->getEntry(
"Ordering").validator();
304 parameterList->getEntry(
"Ordering").setValidator(reordering_validator);
306 sp_->options().set_reordering_method(getIntegralValue<strumpack::ReorderingStrategy>(*parameterList,
"Ordering"));
309 if( parameterList->isParameter(
"ReplaceTinyPivot") ){
310 RCP<const ParameterEntryValidator> replacepivot_validator = valid_params->getEntry(
"ReplaceTinyPivot").validator();
311 parameterList->getEntry(
"ReplaceTinyPivot").setValidator(replacepivot_validator);
313 if( replacepivot_validator) {
314 sp_->options().enable_replace_tiny_pivots();
317 sp_->options().disable_replace_tiny_pivots();
321 if( parameterList->isParameter(
"IterRefine") ){
322 RCP<const ParameterEntryValidator> iter_refine_validator = valid_params->getEntry(
"IterRefine").validator();
323 parameterList->getEntry(
"IterRefine").setValidator(iter_refine_validator);
325 sp_->options().set_Krylov_solver(getIntegralValue<strumpack::KrylovSolver>(*parameterList,
"IterRefine"));
328 if( parameterList->isParameter(
"Compression") ){
329 RCP<const ParameterEntryValidator> compression_validator = valid_params->getEntry(
"Compression").validator();
330 parameterList->getEntry(
"Compression").setValidator(compression_validator);
332 sp_->options().set_compression(getIntegralValue<strumpack::CompressionType>(*parameterList,
"Compression"));
335 TEUCHOS_TEST_FOR_EXCEPTION( this->control_.useTranspose_,
336 std::invalid_argument,
337 "STRUMPACK does not support solving the tranpose system" );
341 template <
class Matrix,
class Vector>
342 Teuchos::RCP<const Teuchos::ParameterList>
346 using Teuchos::tuple;
347 using Teuchos::ParameterList;
348 using Teuchos::EnhancedNumberValidator;
349 using Teuchos::setStringToIntegralParameter;
350 using Teuchos::stringToIntegralParameterEntryValidator;
352 static Teuchos::RCP<const Teuchos::ParameterList> valid_params;
354 if( is_null(valid_params) ){
355 Teuchos::RCP<Teuchos::ParameterList> pl = Teuchos::parameterList();
357 setStringToIntegralParameter<strumpack::MatchingJob>(
"Matching",
"NONE",
358 "Specifies how to permute the "
359 "matrix for numerical stability",
360 tuple<string>(
"NONE",
"MAX_CARDINALITY",
"MAX_SMALLEST_DIAGONAL",
"MAX_SMALLEST_DIAGONAL_2",
"MAX_DIAGONAL_SUM",
"MAX_DIAGONAL_PRODUCT_SCALING",
"COMBBLAS"),
361 tuple<string>(
"NONE",
"MAX_CARDINALITY",
"MAX_SMALLEST_DIAGONAL",
"MAX_SMALLEST_DIAGONAL_2",
"MAX_DIAGONAL_SUM",
"MAX_DIAGONAL_PRODUCT_SCALING",
"COMBBLAS"),
362 tuple<strumpack::MatchingJob>(strumpack::MatchingJob::NONE,
363 strumpack::MatchingJob::MAX_CARDINALITY,
364 strumpack::MatchingJob::MAX_SMALLEST_DIAGONAL,
365 strumpack::MatchingJob::MAX_SMALLEST_DIAGONAL_2,
366 strumpack::MatchingJob::MAX_DIAGONAL_SUM,
367 strumpack::MatchingJob::MAX_DIAGONAL_PRODUCT_SCALING,
368 strumpack::MatchingJob::COMBBLAS),
371 #if defined(STRUMPACK_USE_PARMETIS)
372 std::string default_ordering(
"PARMETIS");
374 std::string default_ordering(
"METIS");
376 setStringToIntegralParameter<strumpack::ReorderingStrategy>(
"Ordering", default_ordering,
377 "Specifies how to permute the "
378 "matrix for sparsity preservation",
379 tuple<string>(
"NATURAL",
"PARMETIS",
"METIS",
"SCOTCH",
"GEOMETRIC",
"PTSCOTCH",
"RCM"),
380 tuple<string>(
"Natural ordering",
384 "Geometric ordering",
387 tuple<strumpack::ReorderingStrategy>(strumpack::ReorderingStrategy::NATURAL,
388 strumpack::ReorderingStrategy::PARMETIS,
389 strumpack::ReorderingStrategy::METIS,
390 strumpack::ReorderingStrategy::SCOTCH,
391 strumpack::ReorderingStrategy::GEOMETRIC,
392 strumpack::ReorderingStrategy::PTSCOTCH,
393 strumpack::ReorderingStrategy::RCM),
396 pl->set(
"ReplaceTinyPivot",
true,
"Specifies whether to replace tiny diagonals during LU factorization");
401 setStringToIntegralParameter<strumpack::KrylovSolver>(
"IterRefine",
"DIRECT",
402 "Type of iterative refinement to use",
403 tuple<string>(
"AUTO",
"DIRECT",
"REFINE",
"PREC_GMRES",
"GMRES",
"PREC_BICGSTAB",
"BICGSTAB"),
404 tuple<string>(
"Use iterative refinement if no compression is used, otherwise use GMRes.",
405 "Single application of the multifrontal solver.",
406 "Iterative refinement.",
407 "Preconditioned GMRes.",
408 "UN-preconditioned GMRes.",
409 "Preconditioned BiCGStab.",
410 "UN-preconditioned BiCGStab."),
411 tuple<strumpack::KrylovSolver>(strumpack::KrylovSolver::AUTO,
412 strumpack::KrylovSolver::DIRECT,
413 strumpack::KrylovSolver::REFINE,
414 strumpack::KrylovSolver::PREC_GMRES,
415 strumpack::KrylovSolver::GMRES,
416 strumpack::KrylovSolver::PREC_BICGSTAB,
417 strumpack::KrylovSolver::BICGSTAB),
422 setStringToIntegralParameter<strumpack::CompressionType>(
"Compression",
"NONE",
423 "Type of compression to use",
424 tuple<string>(
"NONE",
"HSS",
"BLR",
"HODLR",
"LOSSLESS",
"LOSSY"),
425 tuple<string>(
"No compression, purely direct solver.",
426 "HSS compression of frontal matrices.",
427 "Block low-rank compression of fronts.",
428 "Hierarchically Off-diagonal Low-Rank compression of frontal matrices.",
429 "Lossless compresssion.",
430 "Lossy compresssion."),
431 tuple<strumpack::CompressionType>(strumpack::CompressionType::NONE,
432 strumpack::CompressionType::HSS,
433 strumpack::CompressionType::BLR,
434 strumpack::CompressionType::HODLR,
435 strumpack::CompressionType::LOSSLESS,
436 strumpack::CompressionType::LOSSY),
454 template <
class Matrix,
class Vector>
457 using Teuchos::Array;
458 using Teuchos::ArrayView;
459 using Teuchos::ptrInArg;
461 using Teuchos::rcp_dynamic_cast;
465 using Teuchos::MpiComm;
467 #ifdef HAVE_AMESOS2_TIMERS
468 Teuchos::TimeMonitor convTimer(this->timers_.mtxConvTime_);
471 Teuchos::RCP<const MatrixAdapter<Matrix> > redist_mat
472 = this->matrixA_->get(ptrInArg(*strumpack_rowmap_));
474 typedef global_ordinal_type GO;
476 l_nnz = as<GO>(redist_mat->getLocalNNZ());
477 l_rows = as<GO>(redist_mat->getLocalNumRows());
479 RCP<const Comm<int> > comm = this->getComm ();
480 RCP<const MpiComm<int> > mpiComm =
481 rcp_dynamic_cast<
const MpiComm<int> > (comm);
483 const int numProcs = comm->getSize ();
484 const int myRank = comm->getRank ();
485 Array<GO> dist(numProcs+1);
487 dist[myRank+1] = as<GO>(strumpack_rowmap_->getMaxGlobalIndex()) + 1;
489 MPI_Allgather(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, dist.data()+1,
sizeof(GO), MPI_BYTE,
490 mpiComm->getRawMpiComm()->operator()());
491 Kokkos::resize(nzvals_view_, l_nnz);
492 Kokkos::resize(colind_view_, l_nnz);
493 Kokkos::resize(rowptr_view_, l_rows + 1);
498 #ifdef HAVE_AMESOS2_TIMERS
499 Teuchos::TimeMonitor mtxRedistTimer( this->timers_.mtxRedistTime_ );
503 host_value_type_array, host_ordinal_type_array, host_ordinal_type_array >::do_get(
505 nzvals_view_, colind_view_, rowptr_view_,
507 ptrInArg(*strumpack_rowmap_),
513 TEUCHOS_TEST_FOR_EXCEPTION( nnz_ret != l_nnz,
515 "Did not get the expected number of non-zero vals");
518 sp_->set_distributed_csr_matrix
519 (l_rows, rowptr_view_.data(), colind_view_.data(),
520 nzvals_view_.data(), dist.getRawPtr(),
false);
523 #ifdef HAVE_AMESOS2_TIMERS
524 Teuchos::TimeMonitor convTimer(this->timers_.mtxConvTime_);
527 typedef global_ordinal_type GO;
531 Kokkos::resize(nzvals_view_, this->globalNumNonZeros_);
532 Kokkos::resize(colind_view_, this->globalNumNonZeros_);
533 Kokkos::resize(rowptr_view_, this->globalNumRows_ + 1);
536 #ifdef HAVE_AMESOS2_TIMERS
537 Teuchos::TimeMonitor mtxRedistTimer( this->timers_.mtxRedistTime_ );
541 host_value_type_array, host_ordinal_type_array, host_ordinal_type_array >::do_get(
542 this->matrixA_.ptr(),
543 nzvals_view_, colind_view_, rowptr_view_,
546 ARBITRARY, this->rowIndexBase_);
549 TEUCHOS_TEST_FOR_EXCEPTION( nnz_ret != this->globalNumNonZeros_,
551 "Did not get the expected number of non-zero vals");
554 sp_->set_csr_matrix(this->globalNumRows_, rowptr_view_.data(), colind_view_.data(),
555 nzvals_view_.data(),
false);
562 template<
class Matrix,
class Vector>
567 #endif // AMESOS2_STRUMPACK_DEF_HPP
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const override
Return a const parameter list of all of the valid parameters that this->setParameterList(...) will accept.
Definition: Amesos2_SolverCore_def.hpp:524
Amesos2::SolverCore: A templated interface for interaction with third-party direct sparse solvers...
Definition: Amesos2_SolverCore_decl.hpp:71
STRUMPACK(Teuchos::RCP< const Matrix > A, Teuchos::RCP< Vector > X, Teuchos::RCP< const Vector > B)
Initialize from Teuchos::RCP.
Definition: Amesos2_STRUMPACK_def.hpp:32
int preOrdering_impl()
Performs pre-ordering on the matrix to increase efficiency.
Definition: Amesos2_STRUMPACK_def.hpp:102
EPhase
Used to indicate a phase in the direct solution.
Definition: Amesos2_TypeDecl.hpp:31
super_type & setParameters(const Teuchos::RCP< Teuchos::ParameterList > ¶meterList) override
Set/update internal variables and solver options.
Definition: Amesos2_SolverCore_def.hpp:492
void setParameters_impl(const Teuchos::RCP< Teuchos::ParameterList > ¶meterList)
Definition: Amesos2_STRUMPACK_def.hpp:286
bool root_
If true, then this is the root processor.
Definition: Amesos2_SolverCore_decl.hpp:472
global_size_type globalNumRows_
Number of global rows in matrixA_.
Definition: Amesos2_SolverCore_decl.hpp:442
Helper class for getting 1-D copies of multivectors.
Definition: Amesos2_MultiVecAdapter_decl.hpp:233
int symbolicFactorization_impl()
Perform symbolic factorization of the matrix using STRUMPACK.
Definition: Amesos2_STRUMPACK_def.hpp:122
Utility functions for Amesos2.
Control control_
Parameters for solving.
Definition: Amesos2_SolverCore_decl.hpp:460
Teuchos::RCP< const Teuchos::Comm< int > > getComm() const override
Returns a pointer to the Teuchos::Comm communicator with this operator.
Definition: Amesos2_SolverCore_decl.hpp:329
bool matrixShapeOK_impl() const
Determines whether the shape of the matrix is OK for this solver.
Definition: Amesos2_STRUMPACK_def.hpp:268
bool loadA_impl(EPhase current_phase)
Reads matrix data into internal solver structures.
Definition: Amesos2_STRUMPACK_def.hpp:456
Similar to get_ccs_helper , but used to get a CRS representation of the given matrix.
Definition: Amesos2_Util.hpp:625
Amesos2 interface to STRUMPACK direct solver and preconditioner.
Definition: Amesos2_STRUMPACK_decl.hpp:37
~STRUMPACK()
Destructor.
Definition: Amesos2_STRUMPACK_def.hpp:92
Helper class for putting 1-D data arrays into multivectors.
Definition: Amesos2_MultiVecAdapter_decl.hpp:339
A templated MultiVector class adapter for Amesos2.
Definition: Amesos2_MultiVecAdapter_decl.hpp:142
Teuchos::RCP< const MatrixAdapter< Matrix > > matrixA_
The LHS operator.
Definition: Amesos2_SolverCore_decl.hpp:421
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters_impl() const
Definition: Amesos2_STRUMPACK_def.hpp:343
int solve_impl(const Teuchos::Ptr< MultiVecAdapter< Vector > > X, const Teuchos::Ptr< const MultiVecAdapter< Vector > > B) const
STRUMPACK specific solve.
Definition: Amesos2_STRUMPACK_def.hpp:169
int numericFactorization_impl()
STRUMPACK specific numeric factorization.
Definition: Amesos2_STRUMPACK_def.hpp:147