Zoltan2_ComparisonHelper.hpp

Go to the documentation of this file.

// @HEADER
// *****************************************************************************
//   Zoltan2: A package of combinatorial algorithms for scientific computing
//
// Copyright 2012 NTESS and the Zoltan2 contributors.
// SPDX-License-Identifier: BSD-3-Clause
// *****************************************************************************
// @HEADER

#pragma once

#include "Zoltan2_TestHelpers.hpp"
#include "Zoltan2_MetricAnalyzer.hpp"
#include <Zoltan2_Typedefs.hpp>
#include <Zoltan2_EvaluateFactory.hpp>
#include <Zoltan2_ProblemFactory.hpp>
#include <AdapterForTests.hpp>
#include <Teuchos_DefaultComm.hpp>
#include <Teuchos_Time.hpp>
#include <Teuchos_StackedTimer.hpp>

#include <sstream>
#include <string>
#include <map>
#include <iostream>

using Teuchos::Comm;
using Teuchos::RCP;
using Teuchos::ParameterList;
using Teuchos::Time;
using std::string;
using std::map;
using std::pair;
using Teuchos::reduceAll;
using namespace Zoltan2_TestingFramework;

class ComparisonSource
{
public:

  ComparisonSource(Teuchos::RCP<Teuchos::StackedTimer>& stackedTimer,
                   const std::string& problemName)
  : stackedTimer_(stackedTimer)
  , problemName_(problemName) {}

  void startBaseTimer() {
    stackedTimer_->start(problemName_);
  }

  void stopBaseTimer() {
    stackedTimer_->stop(problemName_);
  }

  void startTimer(const std::string& timerName) {
    stackedTimer_->start(timerName);
  }

  void stopTimer(const std::string& timerName) {
    stackedTimer_->stop(timerName);
  }

  std::map<std::string, double> getTimingsMap() const {
    // todo: get all sub-timers
    std::map<std::string, double> timings;
    std::string prefix = stackedTimer_->name()+"@"+problemName_+"@";
    for (auto timerName : {"adapter construction timer", "problem construction time", "solve time"}) {
      auto t = stackedTimer_->findTimer(prefix+timerName);
      timings[timerName] = t.time;
    }
    return timings;
  }

private:
  Teuchos::RCP<Teuchos::StackedTimer> stackedTimer_;
  const std::string problemName_;

public:
  RCP<EvaluateFactory> evaluateFactory;
  RCP<ProblemFactory> problemFactory;
  RCP<AdapterFactory> adapterFactory;
};

class ComparisonHelper
{
public:

  ComparisonHelper(Teuchos::RCP<Teuchos::StackedTimer>& stackedTimer)
  :stackedTimer_(stackedTimer) {}

  /* \brief Compare the solutions, metrics or timers of two Zoltan2 solutions.
   * \param pList is a parameter list defining the comparison
   * \param comm is the process communicator
   */
  bool Compare(const ParameterList &pList, const RCP<const Comm<int> > &comm);
  
  /* \brief Add a new source by name to the comparison source map.
   * \param name is the name of the new source
   * \param source a problem source that to be used for comparison to another source
   */
  RCP<ComparisonSource> AddSource(const string &name);
  
  /* \brief Return the total number of saved sources.
   */
  size_t getNumberOfSources() const
  {
    return this->sources.size();
  }
  
private:
  map<const string,RCP<const ComparisonSource> > sources;

  /* \brief Method called to compare two solutions
   * \param p1 is the name of problem 1
   * \param p2 is the name of problem 2
   * \param comm is the process communicator
   */
  bool CompareSolutions(const string &p1,
                        const string &p2,
                        const RCP<const Comm<int> > &comm);
  
  /* \brief Safely get parts list by adapter type
   * \param problemFactory is the ProblemFactory
   */
  const zpart_t * getPartListView(RCP<ProblemFactory> problemFactory) const;

  /* \brief Method called to compare two paritioning solutions
   * \param sourceA is a ptr to problem A's comparison source
   * \param sourceB is a ptr to problem B's comparison source
   * \param comm is the process communicator
   */
  bool ComparePartitionSolutions(const ComparisonSource * sourceA,
                                 const ComparisonSource * sourceB,
                                 const RCP<const Comm<int> > &comm);
  
  /* \brief Method called to compare two coloring solutions
   * \param sourceA is a ptr to problem A's comparison source
   * \param sourceB is a ptr to problem B's comparison source
   * \param comm is the process communicator
   */
  bool CompareColoringSolutions(const ComparisonSource * sourceA,
                                const ComparisonSource * sourceB,
                                const RCP<const Comm<int> > &comm);
  
  /* \brief Method called to compare two ordering solutions
   * \param sourceA is a ptr to problem A's comparison source
   * \param sourceB is a ptr to problem B's comparison source
   * \param comm is the process communicator
   */
  bool CompareOrderingSolutions(const ComparisonSource * sourceA,
                                const ComparisonSource * sourceB,
                                const RCP<const Comm<int> > &comm);
  
  /* \brief Safely get metric info by adapter type.
   * \param problemFactory is the ProblemFactory
   * \param metricInfo will be filled
   * \param metricsPlist are the parameters to read
   */
  void loadMetricInfo(RCP<EvaluateFactory> problemFactory,
                 std::vector<MetricAnalyzerInfo> & metricInfo,
                 const ParameterList &metricsPlist);

  /* \brief Method called to compare the metrics/timers of two problems.
   * \param metricsPlist is a parameter list defining the comparison
   * \param comm is the process communicator
   */
  bool CompareMetrics(const ParameterList &metricsPlist,
                      const RCP<const Comm<int> > &comm);
  
  /* \brief Method that compares two metrics and returns a pass/fail message.
   * \param[in] comm is the process communicator
   * \param[in] metric is the metric to be compared to a reference metric
   * \param[in] ref_metric is the reference metric for comparison
   * \param[in] metricPlist is the parameter list defining the metric tolerances
   * \param[out] msg is a returned pass/fail message
   *
   * \return boolean value indicated pass/fail status
   */
  static bool
  metricComparisonTest(const RCP<const Comm<int> > &comm,
                       const MetricAnalyzerInfo & metric,
                       const MetricAnalyzerInfo &ref_metric,
                       const Teuchos::ParameterList & metricPlist,
                       std::ostringstream &msg);
  
  /* \brief Method that compares two timers and returns a pass/fail message.
   * \param[in] comm is the process communicator
   * \param[in] time is the timer data to be compared to a reference metric
   * \param[in] ref_time is the reference timer for comparison
   * \param[in] metricPlist is the parameter list defining the timer tolerances
   * \param[out] msg is a returned pass/fail message
   *
   * \return boolean value indicated pass/fail status
   */
  static bool
  timerComparisonTest(const RCP<const Comm<int> > &comm,
                      const double time,
                      const double ref_time,
                      const Teuchos::ParameterList & metricPlist,
                      std::ostringstream &msg);
  
  /* \brief Method for inserting data from all timers to a map of clocked times
   * param[in] timers a map of timers
   *
   * \return a map with clocked times from timers
   */
  static std::map<const string, const double>
  timerDataToMap(const map<const std::string, RCP<Time> > &timers);
  
  
  /* \brief Method for extracting all methods to compare from a parameter list
   * param[in] plist a parameter list defining 1 or more metric/timer comparisons
   *
   * \return a queue of metric comparison definitions
   */
  static std::queue<ParameterList>
  getMetricsToCompare(const ParameterList & pList);
  
  static void
  reduceWithMessage(const RCP<const Comm<int> > &comm,
                    const std::string &msg_in,
                    int &local_status, std::ostringstream &msg);

private:
  RCP<Teuchos::StackedTimer> stackedTimer_;
  
};


RCP<ComparisonSource> ComparisonHelper::AddSource(const string &name)
{
  typedef std::pair<const string &, RCP<const ComparisonSource> > pair_t;
  auto source = Teuchos::rcp(new ComparisonSource(stackedTimer_, name));
  this->sources.insert(pair_t(name, source));
  return source;
}

bool ComparisonHelper::Compare(const ParameterList &pList,
                               const RCP<const Comm<int> > &comm)
{
  if(pList.isParameter("A") && pList.isParameter("B")) {
    // comparing solutions
    string pA = pList.get<string>("A");
    if(this->sources.find(pA) == this->sources.end())
    {
      std::cout << "\nProblem: " + pA + ", was not saved for comparison.";
      std::cout << "\nThis typically indicates that an error ";
      std::cout << "occurred while running the problem.";
      std::cout << "\nSolution comparison FAILED." << std::endl;
      return false;
    }
    
    string pB = pList.get<string>("B");
    if(this->sources.find(pB) == this->sources.end()) {
      std::cout << "\nProblem: " + pB + ", was not saved for comparison.";
      std::cout << "\nThis typically indicates that an error ";
      std::cout << "occurred while running the problem.";
      std::cout << "\nSolution comparison FAILED." << std::endl;
      return false;
    }
    
    bool bResult = this->CompareSolutions(pA, pB, comm);
    return bResult;
  }
  else if (pList.isParameter("Problem") && pList.isParameter("Reference")) {
    // comparing metrics/timers
    string prb = pList.get<string>("Problem");
    if(this->sources.find(prb) == this->sources.end()) {
      std::cout << "\nProblem: " + prb + ", was not saved for comparison.";
      std::cout << "\nThis typically indicates that an error ";
      std::cout << "occurred while running the problem.";
      std::cout << "\nMetric comparison FAILED." << std::endl;
      return false;
    }
    
    string ref = pList.get<string>("Reference");
    if(this->sources.find(ref) == this->sources.end()) {
      std::cout << "\nReference: " + ref + ", was not saved for comparison.";
      std::cout << "\nThis typically indicates that an error ";
      std::cout << "occurred while running the problem.";
      std::cout << "\nMetric comparison FAILED." << std::endl;
      return false;
    }
    
    bool bResult = this->CompareMetrics(pList, comm);
    return bResult;
  }
  else if (pList.isParameter("A") || pList.isParameter("B"))
  {
    if(comm->getRank() == 0)
    {
      std::cout << "Problem A or Problem B is not specified -- check input.";
      std::cout <<"\nSolution comparison FAILED." << std::endl;
    }
  }
  else if (pList.isParameter("Problem") || pList.isParameter("Reference")) {
    if(comm->getRank() == 0) {
      std::cout << "Problem or reference is not specified -- check input.";
      std::cout <<"\nMetric comparison FAILED." << std::endl;
    }
  }
  else {
    if (comm->getRank() == 0) {
      std::cout << "ComparisonHelper did not understand how to read the xml. ";
      std::cout << "Test FAILED." << std::endl;
    }
  }
  return false;
}

bool ComparisonHelper::CompareSolutions(const string &p1,
                                        const string &p2,
                                        const RCP<const Comm<int> > &comm)
{
  if(comm->getRank() == 0) printf( "\nComparing: %s and %s\n",
    p1.c_str(), p2.c_str());
  auto A = this->sources[p1];
  auto B = this->sources[p2];
  if(A->problemFactory->getProblemName() != B->problemFactory->getProblemName()) {
    std::cout << "Problem A and B are of a different kind and cannot be compared.";
    std::cout <<"\nSolution comparison FAILED." << std::endl;
  }
  else {
    if(A->problemFactory->getProblemName() == "partitioning") {
      return this->ComparePartitionSolutions(A.getRawPtr(), B.getRawPtr(), comm);
    }
    else if(A->problemFactory->getProblemName() == "coloring") {
      return this->CompareColoringSolutions(A.getRawPtr(), B.getRawPtr(), comm);
    }
    else if(A->problemFactory->getProblemName() == "ordering"){
      return this->CompareOrderingSolutions(A.getRawPtr(), B.getRawPtr(), comm);
    }
    else {
      std::cout << "Problem kind: " << A->problemFactory->getProblemName() <<
        " not recognized.  Check spelling.";
      std::cout <<"\nSolution comparison FAILED." << std::endl;
    }
  }
  return false;
}

void ComparisonHelper::reduceWithMessage(const RCP<const Comm<int> > &comm,
                                         const std::string &msg_in,
                                         int &local_status,
                                         std::ostringstream &msg) {
  comm->barrier();
  int global_buff;
  Teuchos::Ptr<int> global(&global_buff);
  reduceAll<int,int>(*comm.get(), Teuchos::EReductionType::REDUCE_MAX,
    local_status , global);
  local_status = *global;
  if (local_status == 1) {
    msg << msg_in;
  }
}

const zpart_t * ComparisonHelper::getPartListView(
  RCP<ProblemFactory> problemFactory) const {
  #define GET_PROBLEM_PARTS(adapterClass)                                  \
      return (rcp_dynamic_cast<PartitioningProblem<adapterClass>>(         \
        problemFactory->getProblem()))->getSolution().getPartListView();
  Z2_TEST_UPCAST(problemFactory->getAdapterType(), GET_PROBLEM_PARTS)
}

bool ComparisonHelper::ComparePartitionSolutions(const ComparisonSource * sourceA,
                                                 const ComparisonSource * sourceB,
                                                 const RCP<const Comm<int> > &comm)
{
  int rank = comm->getRank();
  std::ostringstream status;
  int failed = 0;

  if(sourceA->adapterFactory->getMainAdapter()->getLocalNumIDs()
    != sourceB->adapterFactory->getMainAdapter()->getLocalNumIDs()) {
      failed = 1;
  }
  
  ComparisonHelper::reduceWithMessage(comm,
                                  "Number of parts in Solution A != Solution B. \
                                  Partitioning solution comparison FAILED.",
                                  failed, status);
        
  if (!failed) {
    for(size_t i = 0;
      i < sourceA->adapterFactory->getMainAdapter()->getLocalNumIDs(); i++) {
      if(!failed && getPartListView(sourceA->problemFactory)[i] !=
        getPartListView(sourceB->problemFactory)[i]) {
        failed = 1;
        ComparisonHelper::reduceWithMessage(comm,
          "Solution sets A and B have different values for getPartListView(). "
          "Solution comparison FAILED.", failed, status);
      }
    }
  }
  
  if(!failed) {
    status << "Solution sets A and B are the same. ";
    status << "Solution set comparison PASSED.";
  }
  
  if(rank == 0) {
    std::cout << status.str() << std::endl;
  }
  return (failed == 0);
}


bool ComparisonHelper::CompareColoringSolutions(const ComparisonSource * sourceA,
                                                const ComparisonSource * sourceB,
                                                const RCP<const Comm<int> > &comm)
{
  int rank = comm->getRank();
  std::ostringstream status;
  int failed = 0;

  // TO DO - implement coloring comparison
  /*
  if(sourceA->problemFactory->getNumColors()
    != sourceB->problemFactory->getNumColors()) {
      failed = 1;
  }
  
  ComparisonHelper::reduceWithMessage(comm,
                              "Number of colors for Solution A != Solution B. \
                              Coloring solution comparison FAILED.",
                              failed, status);
        
  if (!failed) {
    if(sourceA->problemFactory->getColorsSize()
      != sourceB->problemFactory->getColorsSize()) {
        failed = 1;
    }
    ComparisonHelper::reduceWithMessage(comm,
                              "Size of colors array for Solution A != Solution B. \
                              Coloring solution comparison FAILED.",
                              failed,
                              status);
  }

  if (!failed) {
    for(size_t i = 0; i < sourceA->problemFactory->getColorsSize(); i++) {
      if (sourceA->problemFactory->getColors()[i] !=
        sourceB->problemFactory->getColors()[i]) {
          failed = 1;              // fail
      }
    }
    ComparisonHelper::reduceWithMessage(comm,
                                        "Coloring solution comparison FAILED.",
                                        failed,
                                        status);
  }
  */
  
  if (!failed) {
    status << "Solution sets A and B are the same. ";
    status << "Solution set comparison PASSED.";
  }
  
  if (rank == 0) {
    std::cout << status.str() << std::endl;
  }
  return (failed == 0);
}

bool ComparisonHelper::CompareOrderingSolutions(const ComparisonSource * sourceA,
                                                const ComparisonSource * sourceB,
                                                const RCP<const Comm<int> > &comm)
{
  int rank = comm->getRank();
  std::ostringstream status;
  int failed = 0;
  
  // TO DO - implement ordering comparison

  if(!failed) {
    status << "Solution sets A and B are the same. ";
    status << "Solution set comparison PASSED.";
  }
  
  if(rank == 0) {
    std::cout << status.str() << std::endl;
  }
  return (failed == 0);
}

// Utility function for safe type conversion of adapter
void ComparisonHelper::loadMetricInfo(RCP<EvaluateFactory> evaluateFactory,
  std::vector<MetricAnalyzerInfo> & metricInfo,
  const ParameterList &metricsPlist) {

  #define LOAD_METRIC_INFO(adapterClass, metricAnalyzerClass)                 \
    RCP<EvaluateBaseClass<adapterClass>> pCast =                              \
      rcp_dynamic_cast<EvaluateBaseClass<adapterClass>>(evaluateFactory->getEvaluateClass());  \
    if(pCast == Teuchos::null) throw std::logic_error(                        \
      "Bad evaluate class cast in loadMetricInfo!"  );                        \
      metricAnalyzerClass analyzer(pCast);                                    \
      analyzer.LoadMetricInfo(metricInfo, metricsPlist.sublist("Metrics"));

  #define LOAD_METRIC_INFO_PARTITIONING(adapterClass)                      \
    LOAD_METRIC_INFO(adapterClass, MetricAnalyzerEvaluatePartition<adapterClass>)

  #define LOAD_METRIC_INFO_ORDERING(adapterClass)                          \
    LOAD_METRIC_INFO(adapterClass, MetricAnalyzerEvaluateOrdering<adapterClass>)

  if(evaluateFactory->getProblemName() == "partitioning") {
    Z2_TEST_UPCAST(evaluateFactory->getAdapterType(), LOAD_METRIC_INFO_PARTITIONING)
  }
  else if(evaluateFactory->getProblemName() == "ordering") {
    Z2_TEST_UPCAST(evaluateFactory->getAdapterType(), LOAD_METRIC_INFO_ORDERING)
  }
  else {
    throw std::logic_error(
      "loadMetricInfo not implemented for this problem type!"  );
  }
}

// compare metrics
bool ComparisonHelper::CompareMetrics(const ParameterList &metricsPlist, const RCP<const Comm<int> > &comm)
{
  int rank = comm->getRank();
  
  //get sources for problema nd reference
  const string prb_name = metricsPlist.get<string>("Problem");
  const string ref_name = metricsPlist.get<string>("Reference");
  if(rank == 0) {
    std::cout  << "\nMetric/Timer comparison of: " << prb_name << " and ";
    std::cout << ref_name <<" (reference source)\n";
  }
  
  // get sources
  RCP<const ComparisonSource> sourcePrb = this->sources[prb_name];
  RCP<const ComparisonSource> sourceRef = this->sources[ref_name];

  // get timing data
  const std::map< string, double>& prb_timers = sourcePrb->getTimingsMap();
  const std::map< string, double>& ref_timers = sourceRef->getTimingsMap();

  // get all of the metrics to be tested
  std::queue<ParameterList> metrics = ComparisonHelper::getMetricsToCompare(metricsPlist);

  // run comparison
  int all_tests_pass = 1;
  string metric_name;
  
  while(!metrics.empty()) {
    // print their names...
    std::ostringstream msg;
    metric_name = metrics.front().name();

    if (metric_name == "Metrics") { // special key word means compare the metrics list
      std::vector<MetricAnalyzerInfo> metricInfoSetPrb;
      std::vector<MetricAnalyzerInfo> metricInfoSetRef;

      loadMetricInfo(sourcePrb.get()->evaluateFactory, metricInfoSetPrb, metricsPlist);
      loadMetricInfo(sourceRef.get()->evaluateFactory, metricInfoSetRef, metricsPlist);

      // there is some redundancy here because the metric info holds both the questions and the results
      // this happened because I wanted to reuse the MetricAnalyzer code for loading metric checks or comparisons
      // we can iterate over either to get the questions
      for (size_t n = 0; n < metricInfoSetPrb.size(); ++n) {
        if(!ComparisonHelper::metricComparisonTest(comm, metricInfoSetPrb[n], metricInfoSetRef[n], metrics.front(), msg)) {
          all_tests_pass = 0;
        }
        if(rank == 0) {
          std::cout << msg.str() << std::endl;
        }
      }
    }
    else if(prb_timers.find(metric_name) != prb_timers.end() && ref_timers.find(metric_name) != ref_timers.end()) {
      if(rank == 0) std::cout << "\ncomparing timer: " << metric_name << std::endl;
      if(!ComparisonHelper::timerComparisonTest(comm,
                                                prb_timers.at(metric_name),
                                                ref_timers.at(metric_name),
                                                metrics.front(), msg)) {
        all_tests_pass = 0;
        if (rank == 0) {
          std::cout << "timer comparison test caused a FAILED event." << std::endl;
        }
      }
      
      if(rank == 0) {
        std::cout << msg.str() << std::endl;
      }
    }

    metrics.pop();
  }
  
  if(rank == 0) {
    if(all_tests_pass == 1) {
      std::cout << "\nAll metric/timer comparisons PASSED." << std::endl;
    }
    else {
      std::cout << "\nMetric/timer metric comparisons FAILED." << std::endl;
    }
  }

  return (all_tests_pass == 1);
}

bool ComparisonHelper::metricComparisonTest(const RCP<const Comm<int> > &comm,
                                       const MetricAnalyzerInfo & metric,
                                       const MetricAnalyzerInfo & ref_metric,
                                       const Teuchos::ParameterList & metricPlist,
                                       std::ostringstream &msg)
{
  // run a comparison of min and max against a given metric
  // return an error message on failure
  bool pass = true;
  string test_name = metricPlist.name() + " test";
  double ref_value = ref_metric.theValue;
  double value = metric.theValue;

  if (ref_value == 0) {
    throw std::logic_error( "The parameter list had a 0 value for the reference value so a percentage cannot be calculated." );
  }
  double percentRatio = value / ref_value;
  
  // want to reduce value to max value for all procs
  
  if (ref_metric.bFoundLowerBound) {
    double min = ref_metric.lowerValue;
    if (percentRatio < min) {
      msg << test_name << " FAILED: " << ref_metric.parameterDescription << ": " << value << " is " << percentRatio << " percent of the reference value " << ref_value << ", which less than specified allowable minimum percent, " << min << ".\n";
      pass = false;
    }
    else {
      msg << test_name << " PASSED: " << ref_metric.parameterDescription << ": " << value << " is " << percentRatio << " percent of the reference value " << ref_value << ", which is greater than specified allowable minimum percent, " << min << ".\n";
    }
  }
  
  if (ref_metric.bFoundUpperBound) {
    double max = ref_metric.upperValue;
    if (percentRatio > max) {
      msg << test_name << " FAILED: " << ref_metric.parameterDescription << ": " << value << " is " << percentRatio << " percent of the reference value " << ref_value << ", which is greater than specified allowable maximum percent, " << max << ".\n";
      pass = false;
    }
    else {
      msg << test_name << " PASSED: " << ref_metric.parameterDescription << ": " << value << " is " << percentRatio << " percent of the reference value " << ref_value << ", which is less than specified allowable maximum percent, " << max << ".\n";
    }
  }

  return pass;
}
// BDD, to do: print metrics even for pass
//             reduce max metric to process 0
//             print only on process 0 --- duh.
bool ComparisonHelper::timerComparisonTest(const RCP<const Comm<int> > &comm,
                                           const double time,
                                           const double ref_time,
                                           const Teuchos::ParameterList & metricPlist,
                                           std::ostringstream &msg)
{
  // Reduce time from test
  double global_time;
  Teuchos::Ptr<double> global(&global_time);
  comm->barrier();
  reduceAll<int, double>(*comm.get(),Teuchos::EReductionType::REDUCE_MAX,time,global);
  
  // Reduce time from reference
  double global_ref_time;
  Teuchos::Ptr<double> globalRef(&global_ref_time);
  comm->barrier();
  reduceAll<int, double>(*comm.get(),Teuchos::EReductionType::REDUCE_MAX,ref_time,globalRef);
  
  // run a comparison of min and max against a given metric
  // return an error message on failure
  bool pass = true;
  string test_name = metricPlist.name() + " test";

  if (metricPlist.isParameter("lower")) {
    double min = metricPlist.get<double>("lower")*global_ref_time;
    
    if (global_time < min) {
      msg << test_name << " FAILED: Minimum time, "
      << time <<
      "[s], less than specified allowable minimum time, " << min <<"[s]"<< ".\n";
      pass = false;
    }
    else {
      msg << test_name << " PASSED: Minimum time, "
      << time <<
      "[s], greater than specified allowable minimum time, " << min <<"[s]"<< ".\n";
    }
  }
  
  if (metricPlist.isParameter("upper" ) && pass != false) {
    double max = metricPlist.get<double>("upper") * global_ref_time;
    if (global_time > max) {
      msg << test_name << " FAILED: Maximum time, "
      << global_time <<
      "[s], greater than specified allowable maximum time, " << max <<"[s]"<< ".\n";
      pass = false;
    }
    else {
      msg << test_name << " PASSED: Maximum time, "
      << global_time <<
      "[s], less than specified allowable maximum time, " << max <<"[s]"<< ".\n";
    }
  }
  
  return pass;
}

std::queue<ParameterList> ComparisonHelper::getMetricsToCompare(const ParameterList &pList)
{
  // extract all of the metrics to be tested
  std::queue<ParameterList> metrics;
  for(auto it = pList.begin(); it != pList.end(); ++it) {
    if (pList.isSublist(it->first)) {
      metrics.push(pList.sublist(it->first));
    }
  }
  return metrics;
}