Teuchos - Trilinos Tools Package  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Teuchos_UnitTestRepository.cpp
1 // @HEADER
2 // *****************************************************************************
3 // Teuchos: Common Tools Package
4 //
5 // Copyright 2004 NTESS and the Teuchos contributors.
6 // SPDX-License-Identifier: BSD-3-Clause
7 // *****************************************************************************
8 // @HEADER
9 
10 #include "Teuchos_ConfigDefs.hpp"
12 #include "Teuchos_UnitTestBase.hpp"
14 #include "Teuchos_Array.hpp"
15 #include "Teuchos_Assert.hpp"
16 #include "Teuchos_VerboseObject.hpp"
18 #include "Teuchos_Assert.hpp"
19 #include "Teuchos_Time.hpp"
20 #include "Teuchos_StandardCatchMacros.hpp"
21 
22 
23 namespace Teuchos {
24 
25 
26 struct UnitTestData {
27 
28  const Teuchos::UnitTestBase * unitTest;
29  std::string groupName;
30  std::string testName;
31  int insertionIndex;
32 
33  UnitTestData(
34  Teuchos::UnitTestBase *unitTest_in,
35  const std::string groupName_in,
36  const std::string testName_in
37  )
38  : unitTest(unitTest_in), groupName(groupName_in), testName(testName_in),
39  insertionIndex(insersionIndexCounter_++)
40  {
41 #ifdef TEUCHOS_DEBUG
42  TEUCHOS_ASSERT(unitTest_in);
43 #endif
44  }
45 
46 private:
47  UnitTestData(); // Not defined!
48  static int insersionIndexCounter_;
49 };
50 
51 
52 int UnitTestData::insersionIndexCounter_ = 0;
53 
54 
55 bool operator<(const UnitTestData &a, const UnitTestData &b)
56 {
57  if (a.groupName < b.groupName) {
58  return true;
59  }
60  else if (a.groupName > b.groupName) {
61  return false;
62  }
63  return a.insertionIndex < b.insertionIndex;
64 }
65 
66 
67 
68 std::string getUnitTestName(const std::string groupName,
69  const std::string testName)
70 {
71  std::ostringstream oss;
72  oss << groupName<<"_"<<testName<<"_UnitTest";
73  return oss.str();
74 }
75 
76 
77 enum EShowTestDetails {
78  SHOW_TEST_DETAILS_ALL,
79  SHOW_TEST_DETAILS_TEST_NAMES,
80  SHOW_TEST_DETAILS_FINAL_RESULTS
81 };
82 
83 
84 bool strMatch( const std::string &fullMatchStr, const std::string &str )
85 {
86 
87  const std::string::size_type npos = std::string::npos;
88 
89  const size_t strLen = str.length();
90  const size_t fullMatchStrLen = fullMatchStr.length();
91 
92  if (fullMatchStrLen == 0) {
93  return true;
94  }
95 
96  const bool beginGlob = fullMatchStr[0] == '*';
97  const bool endGlob = fullMatchStr[fullMatchStrLen-1] == '*';
98 
99  const size_t matchStrLen =
100  fullMatchStrLen + (beginGlob ? -1 : 0) + (endGlob ? -1 : 0);
101 
102  if (matchStrLen == 0) {
103  return true;
104  }
105 
106  if (matchStrLen > strLen) {
107  return false;
108  }
109 
110  if (beginGlob && endGlob) {
111  return str.find(fullMatchStr.substr(1, matchStrLen)) != npos;
112  }
113 
114  if (endGlob) {
115  return fullMatchStr.substr(0, matchStrLen) == str.substr(0, matchStrLen);
116  }
117 
118  if (beginGlob) {
119  return fullMatchStr.substr(1, matchStrLen) ==
120  str.substr(strLen-matchStrLen, matchStrLen);
121  }
122 
123  return fullMatchStr == str;
124 
125 }
126 
127 
128 } // namespace Teuchos
129 
130 
131 
132 
133 namespace Teuchos {
134 
135 
136 // Implementation class
137 
138 
139 class UnitTestRepository::InstanceData {
140 public:
141 
142  typedef Teuchos::Array<UnitTestData> unitTests_t;
143 
144  unitTests_t unitTests;
145  CommandLineProcessor clp;
146  EShowTestDetails showTestDetails;
147  bool globallyReduceUnitTestResult;
148  bool showSrcLocation;
149  bool showFailSrcLocation;
150  bool noOp;
151  std::string groupName;
152  std::string testName;
153  std::string notUnitTestName;
154  int testCounter;
155 
156  InstanceData()
157  :clp(false),
158  showTestDetails(SHOW_TEST_DETAILS_TEST_NAMES),
159 #if defined(HAVE_TEUCHOS_GLOBALLY_REDUCE_UNITTEST_RESULTS)
160  globallyReduceUnitTestResult(true),
161 #else
162  globallyReduceUnitTestResult(false),
163 #endif
164  showSrcLocation(false),
165  showFailSrcLocation(true),
166  noOp(false),
167  testCounter(0)
168  {}
169 
170 };
171 
172 
173 // public
174 
175 
177 {
178  return getData().clp;
179 }
180 
181 
183  const bool globallyReduceUnitTestResult)
184 {
185  getData().globallyReduceUnitTestResult = globallyReduceUnitTestResult;
186 }
187 
188 
190 {
191  return getData().globallyReduceUnitTestResult;
192 }
193 
194 
196 {
197 
198  typedef InstanceData::unitTests_t unitTests_t;
199 
200  using std::setprecision;
201 
202  Time overallTimer("overallTimer", true);
203  Time timer("timer");
204 
205  const int timerPrec = 3;
206 
207  out << "\n***\n*** Unit test suite ...\n***\n\n";
208 
209  InstanceData &data = getData();
210 
211  const bool showAll = data.showTestDetails == SHOW_TEST_DETAILS_ALL;
212  const bool showTestNames = data.showTestDetails == SHOW_TEST_DETAILS_TEST_NAMES || showAll;
213 
214  showTestFailureLocation(data.showFailSrcLocation);
215 
216  bool success = true;
217  int testCounter = 0;
218  int numTestsRun = 0;
219  int numTestsFailed = 0;
220 
221  Array<std::string> failedTests;
222 
223  try {
224 
225  out << "\nSorting tests by group name then by the order they were added ...";
226  timer.start(true);
227  std::sort( data.unitTests.begin(), data.unitTests.end() );
228  timer.stop();
229  out << " (time = "<<setprecision(timerPrec)<<timer.totalElapsedTime()<<")\n";
230 
231  out << "\nRunning unit tests ...\n\n";
232  unitTests_t::iterator iter = data.unitTests.begin();
233  for ( ; iter != data.unitTests.end(); ++iter, ++testCounter ) {
234 
235  const UnitTestData &utd = (*iter);
236 
237  const std::string unitTestName = getUnitTestName(utd.groupName, utd.testName);
238 
239  if (
240  (
241  strMatch(data.groupName, utd.groupName)
242  &&
243  strMatch(data.testName, utd.testName)
244  )
245  &&
246  (
247  data.notUnitTestName.length() == 0
248  ||
249  !strMatch(data.notUnitTestName, unitTestName)
250  )
251  )
252  {
253 
254  ++numTestsRun;
255 
256  std::ostringstream testHeaderOSS;
257  testHeaderOSS <<testCounter<<". "<<unitTestName<<" ... ";
258  const std::string testHeader = testHeaderOSS.str();
259 
260  if (showAll)
261  out <<"\n";
262 
263  if (showTestNames)
264  out <<testHeader<<std::flush;
265 
266  {
267 
269  RCP<FancyOStream> localOut;
270  if (showAll) {
271  out << "\n";
272  localOut = rcpFromRef(out);
273  }
274  else {
275  oss = rcp(new std::ostringstream);
276  localOut = fancyOStream(rcp_implicit_cast<std::ostream>(oss));
277  }
278 
279  OSTab tab(out);
280 
281  if (!data.noOp) {
282 
283  timer.start(true);
284  const bool result = runUnitTestImpl(*utd.unitTest, *localOut);
285  timer.stop();
286 
287  if (!result) {
288 
289  failedTests.push_back(testHeader);
290 
291  if (!showTestNames)
292  out <<testHeader<<"\n"<<std::flush;
293  else if (!showAll)
294  out <<"\n";
295 
296  if (!is_null(oss))
297  out << oss->str();
298 
299  out
300  <<"[FAILED] "
301  <<" "<<setprecision(timerPrec)<<"("<<timer.totalElapsedTime()<< " sec)"
302  <<" "<<unitTestName<<"\n"
303  <<"Location: "<<utd.unitTest->unitTestFile()<<":"
304  <<utd.unitTest->unitTestFileLineNumber()<<"\n";
305 
306  if (!is_null(oss))
307  out << "\n";
308 
309  success = false;
310 
311  ++numTestsFailed;
312 
313  }
314  else {
315 
316  if (showTestNames)
317  out << "[Passed] "
318  << setprecision(timerPrec)<<"("<<timer.totalElapsedTime()<<" sec)\n";
319 
320  if (showAll && data.showSrcLocation)
321  out
322  << "Location: "<<utd.unitTest->unitTestFile()<<":"
323  <<utd.unitTest->unitTestFileLineNumber()<<"\n";
324 
325  }
326 
327  }
328  else {
329 
330  if (showTestNames)
331  out << "[Not Run]\n";
332 
333  }
334 
335  }
336 
337  }
338 
339  }
340 
341  TEUCHOS_ASSERT_EQUALITY(testCounter, as<int>(data.unitTests.size()));
342 
343  }
344  TEUCHOS_STANDARD_CATCH_STATEMENTS(true, out, success);
345 
346  if (failedTests.size()) {
347  out << "\nThe following tests FAILED:\n";
348  for (Teuchos_Ordinal i = 0; i < failedTests.size(); ++i)
349  out << " " << failedTests[i] << "\n";
350  }
351 
352  overallTimer.stop();
353  out << "\nTotal Time: " << setprecision(timerPrec)
354  << overallTimer.totalElapsedTime() << " sec\n";
355 
356  out
357  << "\nSummary: total = " << testCounter
358  << ", run = " << numTestsRun;
359 
360  if (!data.noOp) {
361  out
362  << ", passed = " << (numTestsRun-numTestsFailed)
363  << ", failed = " << numTestsFailed << "\n";
364  }
365  else {
366  out
367  << ", passed = ???"
368  << ", failed = ???\n";
369  }
370 
371  return success;
372 
373 }
374 
375 
376 int UnitTestRepository::runUnitTestsFromMain( int argc, char* argv[] )
377 {
378 
380 
381  CommandLineProcessor &clp = getData().clp;
382  setUpCLP(outArg(clp));
384  clp.parse(argc,argv);
385  if ( parse_return != CommandLineProcessor::PARSE_SUCCESSFUL ) {
386  *out << "\nEnd Result: TEST FAILED" << std::endl;
387  return parse_return;
388  }
389 
390  const bool success = runUnitTests(*out);
391 
392  if (success)
393  *out << "\nEnd Result: TEST PASSED" << std::endl;
394  else
395  *out << "\nEnd Result: TEST FAILED" << std::endl;
396 
397  clp.printFinalTimerSummary(out.ptr());
398 
399  return (success ? 0 : 1);
400 
401 }
402 
403 
405  const std::string groupName, const std::string testName_in )
406 {
407  InstanceData &data = getData();
408  std::string testName = testName_in;
409  data.unitTests.push_back(UnitTestData(unitTest, groupName, testName));
410 }
411 
412 
414 {
415  return (getData().showTestDetails == SHOW_TEST_DETAILS_ALL);
416 }
417 
418 
419 // private:
420 
421 
422 UnitTestRepository::UnitTestRepository()
423 {}
424 
425 
426 void UnitTestRepository::setUpCLP(const Ptr<CommandLineProcessor>& clp)
427 {
428 
429  clp->addOutputSetupOptions(true);
430 
431  const int numShowTestDetails = 3;
432  const EShowTestDetails showTestDetailsValues[numShowTestDetails] =
433  { SHOW_TEST_DETAILS_ALL,
434  SHOW_TEST_DETAILS_TEST_NAMES,
435  SHOW_TEST_DETAILS_FINAL_RESULTS
436  };
437  const char* showTestDetailsNames[numShowTestDetails] =
438  { "ALL",
439  "TEST_NAMES",
440  "FINAL_RESULTS"
441  };
442  clp->setOption(
443  "show-test-details", &getData().showTestDetails,
444  numShowTestDetails, showTestDetailsValues, showTestDetailsNames,
445  "Level of detail to show in the tests"
446  );
447  clp->setOption(
448  "details", &getData().showTestDetails,
449  numShowTestDetails, showTestDetailsValues, showTestDetailsNames,
450  "Short for --show-test-details"
451  );
452 
453  clp->setOption(
454  "show-src-location", "no-show-src-location", &getData().showSrcLocation,
455  "If true, then the location of the unit test source code is shown."
456  " Only meaningful if --show-test-details=ALL."
457  );
458 
459  clp->setOption(
460  "show-fail-src-location", "no-show-fail-src-location", &getData().showFailSrcLocation,
461  "If true, then the location of every failed unit test check is printed."
462  );
463 
464  clp->setOption(
465  "globally-reduce-test-result", "no-globally-reduce-test-result",
466  &getData().globallyReduceUnitTestResult,
467  "If true, individual unit test pass/fail is globally reduced across MPI processes."
468  );
469 
470  clp->setOption(
471  "group-name", &getData().groupName,
472  "If specified, selects only tests that match the group name glob." );
473  clp->setOption(
474  "group", &getData().groupName,
475  "Short for --group-name." );
476 
477  clp->setOption(
478  "test-name", &getData().testName,
479  "If specified, selects only tests that match the test name glob." );
480  clp->setOption(
481  "test", &getData().testName,
482  "Short for --test-name." );
483 
484  clp->setOption(
485  "not-unit-test", &getData().notUnitTestName,
486  "If specified, full unit tests with glob matches will *not* be run." );
487 
488  clp->setOption(
489  "no-op", "do-op", &getData().noOp,
490  "If --no-op, then only the names of the tests that would be run are run."
491  );
492 
493 }
494 
495 
496 UnitTestRepository::InstanceData& UnitTestRepository::getData()
497 {
498  static UnitTestRepository::InstanceData data;
499  return data;
500 }
501 
502 
503 bool UnitTestRepository::runUnitTestImpl(const UnitTestBase &unitTest,
504  FancyOStream &out)
505 {
506  const bool result = unitTest.runUnitTest(out);
507  if (getData().globallyReduceUnitTestResult) {
508  const int globalSum = GlobalMPISession::sum(result ? 0 : 1);
509  if (globalSum == 0) {
510  return true;
511  }
512  else {
513  // Only print that there are failures on processes where the local
514  // unit test actally passed. On processes where the local unit test
515  // fails, users already know that test failed so there is no need to
516  // exlain it.
517  if (result) {
518  out << "NOTE: Global reduction shows failures on other processes!\n"
519  << "(rerun with --output-to-root-rank-only=-1 to see output\n"
520  << "from other processes to see what process failed!)\n";
521  }
522  else {
523  // The test failed on the root process so the user already knows it failed!
524  }
525  // Determine what processes have failing tests
526  const int numProcs = GlobalMPISession::getNProc();
527  Array<int> passFailFlags(numProcs);
528  GlobalMPISession::allGather( result ? 0 : 1, passFailFlags());
529  Array<int> procsThatFailed;
530  for ( int proc_k = 0; proc_k < numProcs; ++proc_k ) {
531  if (passFailFlags[proc_k] != 0) {
532  procsThatFailed.push_back(proc_k);
533  }
534  }
535  // Print what processes have the failing tests. If there is only one
536  // processes, don't print anything.
537  if (numProcs > 1) {
538  if (procsThatFailed.size() == numProcs) {
539  out << "NOTE: Unit test failed on all processes!\n";
540  // NOTE: when all the processes are failing it is useless to print
541  // out a list of all of the processes.
542  }
543  else {
544  out << "NOTE: Unit test failed on processes = " << procsThatFailed << "\n"
545  << "(rerun with --output-to-root-rank-only=<procID> to see output\n"
546  << "from individual processes where the unit test is failing!)\n";
547  }
548  }
549  return false;
550  }
551  }
552  return result;
553 }
554 
555 
556 } // namespace Teuchos
static bool verboseUnitTests()
Returns if unit tests are verbose or not.
static int getNProc()
The number of processes in MPI_COMM_WORLD.
static CommandLineProcessor & getCLP()
Return the CLP to add options to.
bool is_null(const std::shared_ptr< T > &p)
Returns true if p.get()==NULL.
Basic wall-clock timer class.
Unit testing support.
basic_FancyOStream< char > FancyOStream
Teuchos header file which uses auto-configuration information to include necessary C++ headers...
Tabbing class for helping to create formated, indented output for a basic_FancyOStream object...
static void addUnitTest(UnitTestBase *unitTest, const std::string groupName, const std::string testName)
Add an unit test (called indirectly through macros.
Unit testing support.
void showTestFailureLocation(bool)
Set if TEUCHOS_PASS_FAIL(...) should print test failure location.
static int runUnitTestsFromMain(int argc, char *argv[])
Run the unit tests from main() passing in (argc, argv).
void start(bool reset=false)
Start the timer, if the timer is enabled (see disable()).
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)
Deprecated.
double stop()
Stop the timer, if the timer is enabled (see disable()).
std::ostream subclass that performs the magic of indenting data sent to an std::ostream object among ...
Wall-clock timer.
Ptr< T > ptr() const
Get a safer wrapper raw C++ pointer to the underlying object.
static RCP< FancyOStream > getDefaultOStream()
Get the default output stream object.
EParseCommandLineReturn parse(int argc, char *argv[], std::ostream *errout=&std::cerr) const
Parse a command line.
static void setGloballyReduceTestResult(const bool globallyReduceUnitTestResult)
Set if the unit tests should reduce pass/fail across processes.
static int sum(int localVal)
Sum a set of integers across processes.
EParseCommandLineReturn
Return value for CommandLineProcessor::parse(). Note: These enums are all given non-negative values s...
Templated array class derived from the STL std::vector.
Utilities to make writing tests easier.
static bool runUnitTests(FancyOStream &out)
Run the registered unit tests.
void push_back(const value_type &x)
Basic command line parser for input from (argc,argv[])
static bool getGloballyReduceTestResult()
Get if the unit tests should reduce across processes or not.
size_type size() const
double totalElapsedTime(bool readCurrentTime=false) const
The total time in seconds accumulated by this timer.
static void allGather(int localVal, const ArrayView< int > &allVals)
Global all-to-all of a set of integers across processes.
void printFinalTimerSummary(const Ptr< std::ostream > &out=null)
Call to print timers so that they don&#39;t get printed in the destructor.
#define TEUCHOS_STANDARD_CATCH_STATEMENTS(VERBOSE, ERR_STREAM, SUCCESS_FLAG)
Simple macro that catches and reports standard exceptions and other exceptions.
#define TEUCHOS_ASSERT(assertion_test)
This macro is throws when an assert fails.
#define TEUCHOS_ASSERT_EQUALITY(val1, val2)
This macro is checks that to numbers are equal and if not then throws an exception with a good error ...
Unit test base class.
Class that helps parse command line input arguments from (argc,argv[]) and set options.