Amesos2_Superlu_decl.hpp

Go to the documentation of this file.

// @HEADER
//
// ***********************************************************************
//
//           Amesos2: Templated Direct Sparse Solver Package
//                  Copyright 2011 Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
//
// ***********************************************************************
//
// @HEADER

#ifndef AMESOS2_SUPERLU_DECL_HPP
#define AMESOS2_SUPERLU_DECL_HPP

#include "Amesos2_SolverTraits.hpp"
#include "Amesos2_SolverCore.hpp"
#include "Amesos2_Superlu_FunctionMap.hpp"

#if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) && defined(KOKKOSKERNELS_ENABLE_TPL_SUPERLU)
#include "KokkosKernels_Handle.hpp"
#endif

namespace Amesos2 {


template <class Matrix,
          class Vector>
class Superlu : public SolverCore<Amesos2::Superlu, Matrix, Vector>
{
  friend class SolverCore<Amesos2::Superlu,Matrix,Vector>; // Give our base access
                                                          // to our private
                                                          // implementation funcs
public:

  static const char* name;      // declaration. Initialization outside.

  typedef Superlu<Matrix,Vector>                                       type;
  typedef SolverCore<Amesos2::Superlu,Matrix,Vector>             super_type;

  // Since typedef's are not inheritted, go grab them
  typedef typename super_type::scalar_type                      scalar_type;
  typedef typename super_type::local_ordinal_type        local_ordinal_type;
  typedef typename super_type::global_ordinal_type      global_ordinal_type;
  typedef typename super_type::global_size_type            global_size_type;

  typedef TypeMap<Amesos2::Superlu,scalar_type>                    type_map;

  /*
   * The SuperLU interface will need two other typedef's, which are:
   * - the superlu type that corresponds to scalar_type and
   * - the corresponding type to use for magnitude
   */
  typedef typename type_map::type                                  slu_type;
  typedef typename type_map::convert_type                  slu_convert_type;
  typedef typename type_map::magnitude_type                  magnitude_type;

  typedef FunctionMap<Amesos2::Superlu,slu_type>               function_map;

#ifdef HAVE_AMESOS2_SUPERLU5_API
  typedef typename function_map::GlobalLU_type                   GlobalLU_t;
#endif



  Superlu(Teuchos::RCP<const Matrix> A,
          Teuchos::RCP<Vector>       X,
          Teuchos::RCP<const Vector> B);


  ~Superlu( );


  std::string description() const;

private:

  int preOrdering_impl();


  int symbolicFactorization_impl();


  int numericFactorization_impl();


  int solve_impl(const Teuchos::Ptr<MultiVecAdapter<Vector> >       X,
                 const Teuchos::Ptr<const MultiVecAdapter<Vector> > B) const;


  bool matrixShapeOK_impl() const;


  void setParameters_impl(
    const Teuchos::RCP<Teuchos::ParameterList> & parameterList );


  Teuchos::RCP<const Teuchos::ParameterList> getValidParameters_impl() const;


  bool loadA_impl(EPhase current_phase);

  typedef Kokkos::DefaultHostExecutionSpace HostExecSpaceType;

  // struct holds all data necessary to make a superlu factorization or solve call
  mutable struct SLUData {
    SLU::SuperMatrix A, B, X, L, U; // matrix A in NCformat
    SLU::SuperMatrix AC;        // permuted matrix A in NCPformat

    SLU::superlu_options_t options;
    SLU::mem_usage_t mem_usage;
#ifdef HAVE_AMESOS2_SUPERLU5_API
    GlobalLU_t lu;      // Use for gssvx and gsisx in SuperLU 5.0
#endif
    SLU::SuperLUStat_t stat;



    typedef Kokkos::View<magnitude_type*, HostExecSpaceType>    host_mag_array;
    typedef Kokkos::View<int*, HostExecSpaceType>               host_int_array;
    host_mag_array berr; 
    host_mag_array ferr; 
    host_int_array perm_r;
    host_int_array perm_c;
    host_int_array etree;
    host_mag_array R;
    host_mag_array C;

#if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) && defined(KOKKOSKERNELS_ENABLE_TPL_SUPERLU)
    host_int_array parents;
#endif

    char equed;
    bool rowequ, colequ;        // flags what type of equilibration
                                // has been performed

    int relax;
    int panel_size;
  } data_;

  typedef int size_type;
  typedef int ordinal_type;
  typedef Kokkos::View<size_type*, HostExecSpaceType>       host_size_type_array;
  typedef Kokkos::View<ordinal_type*, HostExecSpaceType> host_ordinal_type_array;
  typedef Kokkos::View<slu_type*, HostExecSpaceType>       host_value_type_array;

  // The following Arrays are persisting storage arrays for A, X, and B
  host_value_type_array host_nzvals_view_;
  Teuchos::Array<slu_convert_type> convert_nzvals_; // copy to SuperLU native array before calling SuperLU

  host_size_type_array host_rows_view_;
  host_ordinal_type_array host_col_ptr_view_;

  typedef typename Kokkos::View<slu_type**, Kokkos::LayoutLeft, HostExecSpaceType>
    host_solve_array_t;

  mutable host_solve_array_t host_xValues_;
  mutable Teuchos::Array<slu_convert_type> convert_xValues_; // copy to SuperLU native array before calling SuperLU
  int ldx_;

  mutable host_solve_array_t host_bValues_;
  mutable Teuchos::Array<slu_convert_type> convert_bValues_; // copy to SuperLU native array before calling SuperLU
  int ldb_;

#if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) && defined(KOKKOSKERNELS_ENABLE_TPL_SUPERLU)
  typedef Kokkos::DefaultExecutionSpace DeviceExecSpaceType;

  #ifdef KOKKOS_ENABLE_CUDA
    // solver will be UVM off even though Tpetra is CudaUVMSpace
    typedef typename Kokkos::CudaSpace DeviceMemSpaceType;
  #else
    typedef typename DeviceExecSpaceType::memory_space DeviceMemSpaceType;
  #endif

  typedef Kokkos::View<slu_type**, Kokkos::LayoutLeft, DeviceMemSpaceType>
    device_solve_array_t;
  // For triangular solves we have both host and device versions of xValues and
  // bValues because a parameter can turn it on or off.
  mutable device_solve_array_t device_xValues_;
  mutable device_solve_array_t device_bValues_;
  typedef Kokkos::View<int*, DeviceMemSpaceType>              device_int_array;
  device_int_array device_trsv_perm_r_;
  device_int_array device_trsv_perm_c_;
  mutable device_solve_array_t device_trsv_rhs_;
  mutable device_solve_array_t device_trsv_sol_;
  typedef KokkosKernels::Experimental::KokkosKernelsHandle <size_type, ordinal_type, slu_type,
    DeviceExecSpaceType, DeviceMemSpaceType, DeviceMemSpaceType> kernel_handle_type;
  mutable kernel_handle_type device_khL_;
  mutable kernel_handle_type device_khU_;
#endif

  /* Note: In the above, must use "Amesos2::Superlu" rather than
   * "Superlu" because otherwise the compiler references the
   * specialized type of the class, and not the templated type that is
   * required for Amesos2::TypeMap
   */

  /* SuperLU can accept input in either compressed-row or
   * compressed-column storage.  We will store and pass matrices in
   * *compressed-column* format.
   */

  /*
   * Internal flag that is used for the numericFactorization_impl
   * routine.  If true, then the superlu gstrf routine should have
   * SamePattern_SameRowPerm in its options.  Otherwise, it should
   * factor from scratch.
   *
   * This is somewhat of a kludge to get around the fact that the
   * superlu routines all expect something different from the options
   * struct.  The big issue is that we don't want gstrf doing the
   * symbolic factorization if it doesn't need to.  On the other hand,
   * we can't leave options.Fact set to SamePattern_SameRowPerm
   * because the solver driver needs it to be set at FACTORED.  But
   * having it set at FACTORED upon re-entrance into
   * numericFactorization prompts gstrf to redo the symbolic
   * factorization.
   */
  bool same_symbolic_;
  bool ILU_Flag_;

  bool is_contiguous_;
  bool use_triangular_solves_;

  void triangular_solve_factor();

  public: // for GPU
    void triangular_solve() const; // Only for internal use - public to support kernels
};                              // End class Superlu


// Specialize solver_traits struct for SuperLU
template <>
struct solver_traits<Superlu> {
#ifdef HAVE_TEUCHOS_COMPLEX
  typedef Meta::make_list6<float, double,
                           std::complex<float>, std::complex<double>,
                           Kokkos::complex<float>, Kokkos::complex<double>>
                           supported_scalars;
#else
  typedef Meta::make_list2<float, double> supported_scalars;
#endif
};

template <typename Scalar, typename LocalOrdinal, typename ExecutionSpace>
struct solver_supports_matrix<Superlu,
  KokkosSparse::CrsMatrix<Scalar, LocalOrdinal, ExecutionSpace>> {
  static const bool value = true;
};

} // end namespace Amesos2

#endif  // AMESOS2_SUPERLU_DECL_HPP