MueLu_PerfModels_decl.hpp

Go to the documentation of this file.

// @HEADER
//
// ***********************************************************************
//
//        MueLu: A package for multigrid based preconditioning
//                  Copyright 2012 Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact
//                    Jonathan Hu       (jhu@sandia.gov)
//                    Andrey Prokopenko (aprokop@sandia.gov)
//                    Ray Tuminaro      (rstumin@sandia.gov)
//
// ***********************************************************************
//
// @HEADER
#ifndef MUELU_PERFMODELS_HPP
#define MUELU_PERFMODELS_HPP

#include "MueLu_ConfigDefs.hpp"
#include "Xpetra_Import_fwd.hpp"

#include <vector>
#include <ostream>
#include <Teuchos_DefaultComm.hpp>

#include "MueLu_PerfModels_fwd.hpp"

namespace MueLu {

template <class Scalar,
          class LocalOrdinal  = DefaultLocalOrdinal,
          class GlobalOrdinal = DefaultGlobalOrdinal,
          class Node          = DefaultNode>
class PerfModels {
 public:
  PerfModels();

  /* Single Node tests based upon the STREAM benchmark for measuring memory
   * bandwith and computation rate. These processes compute either the addition
   * of two vectors or the multiplication of dense matrices of any given size.
   * Many iterations occur which then return a vector containing the individual
   * lengths of time per iteration.
   *
   * See further here:
   *    - https://www.cs.virginia.edu/stream/ref.html
   *    - https://github.com/UoB-HPC/BabelStream
   */

  /* This version is for table interpolation and works on chars, so the LOG_MAX_SIZE is for bytes */
  void stream_vector_make_table(int KERNEL_REPEATS, int LOG_MAX_SIZE = 20);
  bool has_stream_vector_table() const { return stream_sizes_.size() > 0; }

  /* Lookup in the stream_vector table */
  double stream_vector_copy_lookup(int SIZE_IN_BYTES);
  double stream_vector_add_lookup(int SIZE_IN_BYTES);
  double latency_corrected_stream_vector_copy_lookup(int SIZE_IN_BYTES);
  double latency_corrected_stream_vector_add_lookup(int SIZE_IN_BYTES);

  // Uses the faster of the tables.  The time is then divided by the number of memory transactions
  // per element in the kernel (e.g. 2 for COPY and 3 for ADD).
  double stream_vector_lookup(int SIZE_IN_BYTES);
  double latency_corrected_stream_vector_lookup(int SIZE_IN_BYTES);

  /* Print table */
  void print_stream_vector_table(std::ostream &out, const std::string &prefix = "");
  void print_latency_corrected_stream_vector_table(std::ostream &out, const std::string &prefix = "");

  /* A latency test between two processes based upon the MVAPICH OSU Micro-Benchmarks.
   * The sender process sends a message and then waits for confirmation of reception.
   * Many iterations occur with various message sizes and the average latency values
   * are returned within a map. Utilizes blocking send and recieve.
   *
   * See further: https://mvapich.cse.ohio-state.edu/benchmarks/
   */
  void pingpong_make_table(int KERNEL_REPEATS, int LOG_MAX_SIZE, const RCP<const Teuchos::Comm<int> > &comm);
  bool has_pingpong_table() const { return pingpong_sizes_.size() > 0; }

  /* Lookup in the pingpong_vector table */
  double pingpong_host_lookup(int SIZE_IN_BYTES);
  double pingpong_device_lookup(int SIZE_IN_BYTES);

  /* Print table */
  void print_pingpong_table(std::ostream &out, const std::string &prefix = "");

  /* A halo-exchange based ping-pong, inspired by halo-mode in MPPTEST from ANL.
   * Here we use exactly the communication pattern specified in the import object
   * and send messages accordingly.  We vary the size in bytes sent per message,
   * which should capture max-rate effects to some degree.
   *
   * See further: https://www.mcs.anl.gov/research/projects/mpi/mpptest/
   */
  void halopong_make_table(int KERNEL_REPEATS, int LOG_MAX_SIZE, const RCP<const Xpetra::Import<LocalOrdinal, GlobalOrdinal, Node> > &import);
  bool has_halopong_table() const { return halopong_sizes_.size() > 0; }

  /* Lookup in the halopong_vector table */
  double halopong_host_lookup(int SIZE_IN_BYTES_PER_MESSAGE);
  double halopong_device_lookup(int SIZE_IN_BYTES_PER_MESSAGE);

  /* Print table */
  void print_halopong_table(std::ostream &out, const std::string &prefix = "");

  /* Estimate launch latency based on the cost of submitting an empty Kokkos::parallel_for.
   * This necessary to correct the memory bandwidth costs for models on high latency platforms,
   * e.g., GPUS.
   */
  void launch_latency_make_table(int KERNEL_REPEATS);
  bool has_launch_latency_table() const { return launch_and_wait_latency_ > 0; }

  /* Lookup launch latency */
  double launch_latency_lookup();

  /* Print table */
  void print_launch_latency_table(std::ostream &out, const std::string &prefix = "");

 private:
  void print_stream_vector_table_impl(std::ostream &out, bool use_latency_correction, const std::string &prefix);

  std::vector<int> stream_sizes_;
  std::vector<double> stream_copy_times_;
  std::vector<double> stream_add_times_;
  std::vector<double> latency_corrected_stream_copy_times_;
  std::vector<double> latency_corrected_stream_add_times_;

  std::vector<int> pingpong_sizes_;
  std::vector<double> pingpong_host_times_;
  std::vector<double> pingpong_device_times_;

  std::vector<int> halopong_sizes_;
  std::vector<double> halopong_host_times_;
  std::vector<double> halopong_device_times_;

  double launch_and_wait_latency_;

};  // class PerfModels

}  // namespace MueLu

#endif  // ifndef MUELU_PERFMODELS_HPP