MueLu  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
MueLu_PerfUtils_def.hpp
Go to the documentation of this file.
1 // @HEADER
2 //
3 // ***********************************************************************
4 //
5 // MueLu: A package for multigrid based preconditioning
6 // Copyright 2012 Sandia Corporation
7 //
8 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Redistribution and use in source and binary forms, with or without
12 // modification, are permitted provided that the following conditions are
13 // met:
14 //
15 // 1. Redistributions of source code must retain the above copyright
16 // notice, this list of conditions and the following disclaimer.
17 //
18 // 2. Redistributions in binary form must reproduce the above copyright
19 // notice, this list of conditions and the following disclaimer in the
20 // documentation and/or other materials provided with the distribution.
21 //
22 // 3. Neither the name of the Corporation nor the names of the
23 // contributors may be used to endorse or promote products derived from
24 // this software without specific prior written permission.
25 //
26 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NodeT LIMITED TO, THE
28 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 // PURPOSE ARE DIScalarLAIMED. IN Node EVENT SHALL SANDIA CORPORATION OR THE
30 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NodeT LIMITED TO,
32 // PROCUREMENT OF SUBSTITUTE GlobalOrdinalODS OR SERVICES; LOSS OF USE, DATA, OR
33 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 //
38 // Questions? Contact
39 // Jonathan Hu (jhu@sandia.gov)
40 // Andrey Prokopenko (aprokop@sandia.gov)
41 // Ray Tuminaro (rstumin@sandia.gov)
42 //
43 // ***********************************************************************
44 //
45 // @HEADER
46 #ifndef MUELU_PERFUTILS_DEF_HPP
47 #define MUELU_PERFUTILS_DEF_HPP
48 
49 #include <algorithm>
50 #include <string>
51 
52 #ifdef HAVE_MPI
53 #include <Teuchos_CommHelpers.hpp>
54 #endif
55 
56 #include <Xpetra_Export.hpp>
57 #include <Xpetra_Import.hpp>
58 #include <Xpetra_Matrix.hpp>
59 
60 #include "MueLu_PerfUtils_decl.hpp"
61 
62 //#include "MueLu_Utilities.hpp"
63 
64 namespace MueLu {
65 
66  template<class Type>
67  void calculateStats(Type& minVal, Type& maxVal, double& avgVal, double& devVal, const RCP<const Teuchos::Comm<int> >& comm, int numActiveProcs, const Type& v) {
68 
69  Type sumVal, sum2Val;
70 
71  MueLu_sumAll(comm, v, sumVal);
72  MueLu_sumAll(comm, v*v, sum2Val);
73  MueLu_minAll(comm, v, minVal);
74  MueLu_maxAll(comm, v, maxVal);
75 
76  avgVal = (numActiveProcs > 0 ? as<double>(sumVal) / numActiveProcs : 0);
77  devVal = (numActiveProcs > 1 ? sqrt((sum2Val - sumVal*avgVal)/(numActiveProcs-1)) : 0);
78  }
79 
80  template<class Type>
81  std::string stringStats(const RCP<const Teuchos::Comm<int> >& comm, int numActiveProcs, const Type& v, RCP<ParameterList> paramList = Teuchos::null) {
82  Type minVal, maxVal;
83  double avgVal, devVal;
84  calculateStats<Type>(minVal, maxVal, avgVal, devVal, comm, numActiveProcs, v);
85 
86  char buf[256];
87  if (avgVal && (paramList.is_null() || !paramList->isParameter("print abs") || paramList->get<bool>("print abs") == false))
88  sprintf(buf, "avg = %.2e, dev = %5.1f%%, min = %+6.1f%%, max = %+6.1f%%", avgVal,
89  (devVal/avgVal)*100, (minVal/avgVal-1)*100, (maxVal/avgVal-1)*100);
90  else
91  sprintf(buf, "avg = %8.2f, dev = %6.2f, min = %6.1f , max = %6.1f", avgVal,
92  devVal, as<double>(minVal), as<double>(maxVal));
93  return buf;
94  }
95 
96  template<class Map>
97  bool cmp_less(typename Map::value_type& v1, typename Map::value_type& v2) {
98  return v1.second < v2.second;
99  }
100 
101  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
102  std::string PerfUtils<Scalar, LocalOrdinal, GlobalOrdinal, Node>::PrintMatrixInfo(const Matrix& A, const std::string& msgTag, RCP<const ParameterList> params) {
103  if (!CheckMatrix(A))
104  return "";
105 
107 
108  std::ostringstream ss;
109 
110  ss << msgTag << " size = " << A.getGlobalNumRows() << " x " << A.getGlobalNumCols();
111  if(A.haveGlobalConstants())
112  ss << ", nnz = " << A.getGlobalNumEntries();
113  ss << std::endl;
114 
115  if (params.is_null())
116  return ss.str();
117 
118  bool printLoadBalanceInfo = false, printCommInfo = false;
119  if (params->isParameter("printLoadBalancingInfo") && params->get<bool>("printLoadBalancingInfo"))
120  printLoadBalanceInfo = true;
121  if (params->isParameter("printCommInfo") && params->get<bool>("printCommInfo"))
122  printCommInfo = true;
123 
124  if (!printLoadBalanceInfo && !printCommInfo)
125  return ss.str();
126 
127  RCP<const Import> importer = A.getCrsGraph()->getImporter();
128  RCP<const Export> exporter = A.getCrsGraph()->getExporter();
129 
130  size_t numMyNnz = A.getNodeNumEntries(), numMyRows = A.getNodeNumRows();
131 
132  // Create communicator only for active processes
133  RCP<const Teuchos::Comm<int> > origComm = A.getRowMap()->getComm();
134  bool activeProc = true;
135  int numProc = origComm->getSize();
136  int numActiveProcs = 0;
137 #ifdef HAVE_MPI
138  RCP<const Teuchos::MpiComm<int> > mpiComm = rcp_dynamic_cast<const Teuchos::MpiComm<int> >(origComm);
139  MPI_Comm rawComm = (*mpiComm->getRawMpiComm())();
140 
141  std::vector<size_t> numRowsPerProc(numProc);
142  Teuchos::gatherAll(*origComm, 1, &numMyRows, numProc, &numRowsPerProc[0]);
143 
144  int root = 0;
145  bool rootFlag = true;
146  for (int i = 0; i < numProc; i++) {
147  if (numRowsPerProc[i]) {
148  ++numActiveProcs;
149  if(rootFlag) {
150  root = i;
151  rootFlag = false;
152  }
153  }
154  }
155 
156  if(numMyRows == 0) {activeProc = false; numMyNnz = 0;} // Reset numMyNnz to avoid adding it up in reduceAll
157 #else
158  if(numMyRows == 0) {
159  //FIXME JJH 10-May-2017 Is there any case in serial where numMyRows would be zero?
160  // Reset numMyNnz to avoid adding it up in reduceAll
161  numActiveProcs = 0;
162  activeProc = false;
163  numMyNnz = 0;
164  } else {
165  numActiveProcs = 1;
166  }
167 #endif
168 
169  std::string outstr;
170  ParameterList absList;
171  absList.set("print abs", true);
172 
173  if (printLoadBalanceInfo) {
174  ss << msgTag << " Load balancing info" << std::endl;
175  ss << msgTag << " # active processes: " << numActiveProcs << "/" << numProc << std::endl;
176  ss << msgTag << " # rows per proc : " << stringStats<global_size_t>(origComm, numActiveProcs, numMyRows) << std::endl;
177  ss << msgTag << " # nnz per proc : " << stringStats<global_size_t>(origComm, numActiveProcs, numMyNnz) << std::endl;
178  }
179 
180  if (printCommInfo && numActiveProcs != 1) {
181  typedef std::map<int,size_t> map_type;
182  map_type neighMap;
183  if (!importer.is_null()) {
184  ArrayView<const int> exportPIDs = importer->getExportPIDs();
185  if (exportPIDs.size())
186  for (int i = 0; i < exportPIDs.size(); i++)
187  neighMap[exportPIDs[i]]++;
188  }
189 
190  // Communication volume
191  size_t numExportSend = 0;
192  size_t numImportSend = 0;
193  size_t numMsgs = 0;
194  size_t minMsg = 0;
195  size_t maxMsg = 0;
196 
197  if(activeProc) {
198  numExportSend = (!exporter.is_null() ? exporter->getNumExportIDs() : 0);
199  numImportSend = (!importer.is_null() ? importer->getNumExportIDs() : 0);
200  numMsgs = neighMap.size();
201  map_type::const_iterator it = std::min_element(neighMap.begin(), neighMap.end(), cmp_less<map_type>);
202  minMsg = (it != neighMap.end() ? it->second : 0);
203  it = std::max_element(neighMap.begin(), neighMap.end(), cmp_less<map_type>);
204  maxMsg = (it != neighMap.end() ? it->second : 0);
205  }
206 
207  ss << msgTag << " Communication info" << std::endl;
208  ss << msgTag << " # num export send : " << stringStats<global_size_t>(origComm, numActiveProcs, numExportSend) << std::endl;
209  ss << msgTag << " # num import send : " << stringStats<global_size_t>(origComm, numActiveProcs, numImportSend) << std::endl;
210  ss << msgTag << " # num msgs : " << stringStats<global_size_t>(origComm, numActiveProcs, numMsgs, rcpFromRef(absList)) << std::endl;
211  ss << msgTag << " # min msg size : " << stringStats<global_size_t>(origComm, numActiveProcs, minMsg) << std::endl;
212  ss << msgTag << " # max msg size : " << stringStats<global_size_t>(origComm, numActiveProcs, maxMsg) << std::endl;
213  }
214 
215  outstr = ss.str();
216 
217 #ifdef HAVE_MPI
218  int strLength = outstr.size();
219  MPI_Bcast(&strLength, 1, MPI_INT, root, rawComm);
220  if (origComm->getRank() != root)
221  outstr.resize(strLength);
222  MPI_Bcast(&outstr[0], strLength, MPI_CHAR, root, rawComm);
223 #endif
224 
225  return outstr;
226  }
227 
228  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
230 
232 
233  std::ostringstream ss;
234 
235  // Create communicator only for active processes
236  RCP<const Teuchos::Comm<int> > origComm = importer->getSourceMap()->getComm();
237  bool activeProc = true;
238  int numActiveProcs = origComm->getSize();
239 #ifdef HAVE_MPI
240  RCP<const Teuchos::MpiComm<int> > mpiComm = rcp_dynamic_cast<const Teuchos::MpiComm<int> >(origComm);
241  MPI_Comm rawComm = (*mpiComm->getRawMpiComm())();
242  int root = 0;
243 #endif
244 
245  std::string outstr;
246  ParameterList absList;
247  absList.set("print abs", true);
248 
249  typedef std::map<int,size_t> map_type;
250  map_type neighMap;
251  ArrayView<const int> exportPIDs = importer->getExportPIDs();
252  if (exportPIDs.size())
253  for (int i = 0; i < exportPIDs.size(); i++)
254  neighMap[exportPIDs[i]]++;
255 
256  // Communication volume
257  size_t numImportSend = 0;
258  size_t numMsgs = 0;
259  size_t minMsg = 0;
260  size_t maxMsg = 0;
261 
262  if(activeProc) {
263  numImportSend = importer->getNumExportIDs();
264  numMsgs = neighMap.size();
265  map_type::const_iterator it = std::min_element(neighMap.begin(), neighMap.end(), cmp_less<map_type>);
266  minMsg = (it != neighMap.end() ? it->second : 0);
267  it = std::max_element(neighMap.begin(), neighMap.end(), cmp_less<map_type>);
268  maxMsg = (it != neighMap.end() ? it->second : 0);
269  }
270 
271  ss << msgTag << " Communication info" << std::endl;
272  ss << msgTag << " # num import send : " << stringStats<global_size_t>(origComm, numActiveProcs, numImportSend) << std::endl;
273  ss << msgTag << " # num msgs : " << stringStats<global_size_t>(origComm, numActiveProcs, numMsgs, rcpFromRef(absList)) << std::endl;
274  ss << msgTag << " # min msg size : " << stringStats<global_size_t>(origComm, numActiveProcs, minMsg) << std::endl;
275  ss << msgTag << " # max msg size : " << stringStats<global_size_t>(origComm, numActiveProcs, maxMsg) << std::endl;
276 
277 
278  outstr = ss.str();
279 
280 #ifdef HAVE_MPI
281  int strLength = outstr.size();
282  MPI_Bcast(&strLength, 1, MPI_INT, root, rawComm);
283  if (origComm->getRank() != root)
284  outstr.resize(strLength);
285  MPI_Bcast(&outstr[0], strLength, MPI_CHAR, root, rawComm);
286 #endif
287 
288  return outstr;
289  }
290 
291  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
292  std::string PerfUtils<Scalar, LocalOrdinal, GlobalOrdinal, Node>::CommPattern(const Matrix& A, const std::string& msgTag, RCP<const ParameterList> /* params */) {
293  if (!CheckMatrix(A))
294  return "";
295 
296  std::ostringstream out;
297 
298  RCP<const Teuchos::Comm<int> > comm = A.getRowMap()->getComm();
299  int myRank = comm->getRank();
300 
301  out << msgTag << " " << myRank << ":";
302 
303  RCP<const Import> importer = (A.getCrsGraph() != Teuchos::null ? A.getCrsGraph()->getImporter() : Teuchos::null);
304  if (importer.is_null()) {
305  out << std::endl;
306  return out.str();
307  }
308 
309  ArrayView<const int> exportPIDs = importer->getExportPIDs();
310 
311  if (exportPIDs.size()) {
312  // NodeTE: exportPIDs is sorted but not unique ( 1 1 1 2 2 3 4 4 4 )
313  int neigh = exportPIDs[0];
314  GO weight = 1;
315  for (int i = 1; i < exportPIDs.size(); i++) {
316  if (exportPIDs[i] != exportPIDs[i-1]) {
317  out << " " << neigh << "(" << weight << ")";
318 
319  neigh = exportPIDs[i];
320  weight = 1;
321 
322  } else {
323  weight += 1;
324  }
325  }
326  out << " " << neigh << "(" << weight << ")" << std::endl;
327  }
328 
329  return out.str();
330  }
331 
332  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
334  // We can only print statistics for matrices that have a crs graph. A
335  // potential issue is regarding Xpetra::TpetraBlockCrsMatrix which has no
336  // CrsGraph. It is held as a private data member by Xpetra::CrsMatrix,
337  // which itself is an Xpetra::Matrix. So we check directly whether the
338  // request for the graph throws.
339  bool hasCrsGraph = true;
340  try {
341  A.getCrsGraph();
342 
343  } catch (...) {
344  hasCrsGraph = false;
345  }
346 
347  return hasCrsGraph;
348  }
349 
350 } //namespace MueLu
351 
352 #endif // MUELU_PERFUTILS_DEF_HPP
static bool CheckMatrix(const Matrix &A)
std::string stringStats(const RCP< const Teuchos::Comm< int > > &comm, int numActiveProcs, const Type &v, RCP< ParameterList > paramList=Teuchos::null)
#define MueLu_sumAll(rcpComm, in, out)
#define MueLu_maxAll(rcpComm, in, out)
void calculateStats(Type &minVal, Type &maxVal, double &avgVal, double &devVal, const RCP< const Teuchos::Comm< int > > &comm, int numActiveProcs, const Type &v)
GlobalOrdinal GO
T & get(const std::string &name, T def_value)
ParameterList & set(std::string const &name, T const &value, std::string const &docString="", RCP< const ParameterEntryValidator > const &validator=null)
size_type size() const
#define MueLu_minAll(rcpComm, in, out)
static std::string CommPattern(const Matrix &A, const std::string &msgTag, RCP< const Teuchos::ParameterList > params=Teuchos::null)
static std::string PrintImporterInfo(RCP< const Import > importer, const std::string &msgTag)
bool isParameter(const std::string &name) const
static std::string PrintMatrixInfo(const Matrix &A, const std::string &msgTag, RCP< const Teuchos::ParameterList > params=Teuchos::null)
size_t global_size_t
bool cmp_less(typename Map::value_type &v1, typename Map::value_type &v2)
bool is_null() const