Teuchos - Trilinos Tools Package  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Teuchos_UnitTestRepository.cpp
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Teuchos: Common Tools Package
5 // Copyright (2004) Sandia Corporation
6 //
7 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8 // license for use of this work by or on behalf of the U.S. Government.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ***********************************************************************
40 // @HEADER
41 
42 
43 #include "Teuchos_ConfigDefs.hpp"
45 #include "Teuchos_UnitTestBase.hpp"
47 #include "Teuchos_Array.hpp"
48 #include "Teuchos_Assert.hpp"
49 #include "Teuchos_VerboseObject.hpp"
51 #include "Teuchos_Assert.hpp"
52 #include "Teuchos_Time.hpp"
53 #include "Teuchos_StandardCatchMacros.hpp"
54 
55 
56 namespace Teuchos {
57 
58 
59 struct UnitTestData {
60 
61  const Teuchos::UnitTestBase * unitTest;
62  std::string groupName;
63  std::string testName;
64  int insertionIndex;
65 
66  UnitTestData(
67  Teuchos::UnitTestBase *unitTest_in,
68  const std::string groupName_in,
69  const std::string testName_in
70  )
71  : unitTest(unitTest_in), groupName(groupName_in), testName(testName_in),
72  insertionIndex(insersionIndexCounter_++)
73  {
74 #ifdef TEUCHOS_DEBUG
75  TEUCHOS_ASSERT(unitTest_in);
76 #endif
77  }
78 
79 private:
80  UnitTestData(); // Not defined!
81  static int insersionIndexCounter_;
82 };
83 
84 
85 int UnitTestData::insersionIndexCounter_ = 0;
86 
87 
88 bool operator<(const UnitTestData &a, const UnitTestData &b)
89 {
90  if (a.groupName < b.groupName) {
91  return true;
92  }
93  else if (a.groupName > b.groupName) {
94  return false;
95  }
96  return a.insertionIndex < b.insertionIndex;
97 }
98 
99 
100 
101 std::string getUnitTestName(const std::string groupName,
102  const std::string testName)
103 {
104  std::ostringstream oss;
105  oss << groupName<<"_"<<testName<<"_UnitTest";
106  return oss.str();
107 }
108 
109 
110 enum EShowTestDetails {
111  SHOW_TEST_DETAILS_ALL,
112  SHOW_TEST_DETAILS_TEST_NAMES,
113  SHOW_TEST_DETAILS_FINAL_RESULTS
114 };
115 
116 
117 bool strMatch( const std::string &fullMatchStr, const std::string &str )
118 {
119 
120  const std::string::size_type npos = std::string::npos;
121 
122  const size_t strLen = str.length();
123  const size_t fullMatchStrLen = fullMatchStr.length();
124 
125  if (fullMatchStrLen == 0) {
126  return true;
127  }
128 
129  const bool beginGlob = fullMatchStr[0] == '*';
130  const bool endGlob = fullMatchStr[fullMatchStrLen-1] == '*';
131 
132  const size_t matchStrLen =
133  fullMatchStrLen + (beginGlob ? -1 : 0) + (endGlob ? -1 : 0);
134 
135  if (matchStrLen == 0) {
136  return true;
137  }
138 
139  if (matchStrLen > strLen) {
140  return false;
141  }
142 
143  if (beginGlob && endGlob) {
144  return str.find(fullMatchStr.substr(1, matchStrLen)) != npos;
145  }
146 
147  if (endGlob) {
148  return fullMatchStr.substr(0, matchStrLen) == str.substr(0, matchStrLen);
149  }
150 
151  if (beginGlob) {
152  return fullMatchStr.substr(1, matchStrLen) ==
153  str.substr(strLen-matchStrLen, matchStrLen);
154  }
155 
156  return fullMatchStr == str;
157 
158 }
159 
160 
161 } // namespace Teuchos
162 
163 
164 
165 
166 namespace Teuchos {
167 
168 
169 // Implementation class
170 
171 
172 class UnitTestRepository::InstanceData {
173 public:
174 
175  typedef Teuchos::Array<UnitTestData> unitTests_t;
176 
177  unitTests_t unitTests;
178  CommandLineProcessor clp;
179  EShowTestDetails showTestDetails;
180  bool globallyReduceUnitTestResult;
181  bool showSrcLocation;
182  bool showFailSrcLocation;
183  bool noOp;
184  std::string groupName;
185  std::string testName;
186  std::string notUnitTestName;
187  int testCounter;
188 
189  InstanceData()
190  :clp(false),
191  showTestDetails(SHOW_TEST_DETAILS_TEST_NAMES),
192 #if defined(HAVE_TEUCHOS_GLOBALLY_REDUCE_UNITTEST_RESULTS)
193  globallyReduceUnitTestResult(true),
194 #else
195  globallyReduceUnitTestResult(false),
196 #endif
197  showSrcLocation(false),
198  showFailSrcLocation(true),
199  noOp(false),
200  testCounter(0)
201  {}
202 
203 };
204 
205 
206 // public
207 
208 
210 {
211  return getData().clp;
212 }
213 
214 
216  const bool globallyReduceUnitTestResult)
217 {
218  getData().globallyReduceUnitTestResult = globallyReduceUnitTestResult;
219 }
220 
221 
223 {
224  return getData().globallyReduceUnitTestResult;
225 }
226 
227 
229 {
230 
231  typedef InstanceData::unitTests_t unitTests_t;
232 
233  using std::setprecision;
234 
235  Time overallTimer("overallTimer", true);
236  Time timer("timer");
237 
238  const int timerPrec = 3;
239 
240  out << "\n***\n*** Unit test suite ...\n***\n\n";
241 
242  InstanceData &data = getData();
243 
244  const bool showAll = data.showTestDetails == SHOW_TEST_DETAILS_ALL;
245  const bool showTestNames = data.showTestDetails == SHOW_TEST_DETAILS_TEST_NAMES || showAll;
246 
247  showTestFailureLocation(data.showFailSrcLocation);
248 
249  bool success = true;
250  int testCounter = 0;
251  int numTestsRun = 0;
252  int numTestsFailed = 0;
253 
254  Array<std::string> failedTests;
255 
256  try {
257 
258  out << "\nSorting tests by group name then by the order they were added ...";
259  timer.start(true);
260  std::sort( data.unitTests.begin(), data.unitTests.end() );
261  timer.stop();
262  out << " (time = "<<setprecision(timerPrec)<<timer.totalElapsedTime()<<")\n";
263 
264  out << "\nRunning unit tests ...\n\n";
265  unitTests_t::iterator iter = data.unitTests.begin();
266  for ( ; iter != data.unitTests.end(); ++iter, ++testCounter ) {
267 
268  const UnitTestData &utd = (*iter);
269 
270  const std::string unitTestName = getUnitTestName(utd.groupName, utd.testName);
271 
272  if (
273  (
274  strMatch(data.groupName, utd.groupName)
275  &&
276  strMatch(data.testName, utd.testName)
277  )
278  &&
279  (
280  data.notUnitTestName.length() == 0
281  ||
282  !strMatch(data.notUnitTestName, unitTestName)
283  )
284  )
285  {
286 
287  ++numTestsRun;
288 
289  std::ostringstream testHeaderOSS;
290  testHeaderOSS <<testCounter<<". "<<unitTestName<<" ... ";
291  const std::string testHeader = testHeaderOSS.str();
292 
293  if (showAll)
294  out <<"\n";
295 
296  if (showTestNames)
297  out <<testHeader<<std::flush;
298 
299  {
300 
302  RCP<FancyOStream> localOut;
303  if (showAll) {
304  out << "\n";
305  localOut = rcpFromRef(out);
306  }
307  else {
308  oss = rcp(new std::ostringstream);
309  localOut = fancyOStream(rcp_implicit_cast<std::ostream>(oss));
310  }
311 
312  OSTab tab(out);
313 
314  if (!data.noOp) {
315 
316  timer.start(true);
317  const bool result = runUnitTestImpl(*utd.unitTest, *localOut);
318  timer.stop();
319 
320  if (!result) {
321 
322  failedTests.push_back(testHeader);
323 
324  if (!showTestNames)
325  out <<testHeader<<"\n"<<std::flush;
326  else if (!showAll)
327  out <<"\n";
328 
329  if (!is_null(oss))
330  out << oss->str();
331 
332  out
333  <<"[FAILED] "
334  <<" "<<setprecision(timerPrec)<<"("<<timer.totalElapsedTime()<< " sec)"
335  <<" "<<unitTestName<<"\n"
336  <<"Location: "<<utd.unitTest->unitTestFile()<<":"
337  <<utd.unitTest->unitTestFileLineNumber()<<"\n";
338 
339  if (!is_null(oss))
340  out << "\n";
341 
342  success = false;
343 
344  ++numTestsFailed;
345 
346  }
347  else {
348 
349  if (showTestNames)
350  out << "[Passed] "
351  << setprecision(timerPrec)<<"("<<timer.totalElapsedTime()<<" sec)\n";
352 
353  if (showAll && data.showSrcLocation)
354  out
355  << "Location: "<<utd.unitTest->unitTestFile()<<":"
356  <<utd.unitTest->unitTestFileLineNumber()<<"\n";
357 
358  }
359 
360  }
361  else {
362 
363  if (showTestNames)
364  out << "[Not Run]\n";
365 
366  }
367 
368  }
369 
370  }
371 
372  }
373 
374  TEUCHOS_ASSERT_EQUALITY(testCounter, as<int>(data.unitTests.size()));
375 
376  }
377  TEUCHOS_STANDARD_CATCH_STATEMENTS(true, out, success);
378 
379  if (failedTests.size()) {
380  out << "\nThe following tests FAILED:\n";
381  for (Teuchos_Ordinal i = 0; i < failedTests.size(); ++i)
382  out << " " << failedTests[i] << "\n";
383  }
384 
385  overallTimer.stop();
386  out << "\nTotal Time: " << setprecision(timerPrec)
387  << overallTimer.totalElapsedTime() << " sec\n";
388 
389  out
390  << "\nSummary: total = " << testCounter
391  << ", run = " << numTestsRun;
392 
393  if (!data.noOp) {
394  out
395  << ", passed = " << (numTestsRun-numTestsFailed)
396  << ", failed = " << numTestsFailed << "\n";
397  }
398  else {
399  out
400  << ", passed = ???"
401  << ", failed = ???\n";
402  }
403 
404  return success;
405 
406 }
407 
408 
409 int UnitTestRepository::runUnitTestsFromMain( int argc, char* argv[] )
410 {
411 
413 
414  CommandLineProcessor &clp = getData().clp;
415  setUpCLP(outArg(clp));
417  clp.parse(argc,argv);
418  if ( parse_return != CommandLineProcessor::PARSE_SUCCESSFUL ) {
419  *out << "\nEnd Result: TEST FAILED" << std::endl;
420  return parse_return;
421  }
422 
423  const bool success = runUnitTests(*out);
424 
425  if (success)
426  *out << "\nEnd Result: TEST PASSED" << std::endl;
427  else
428  *out << "\nEnd Result: TEST FAILED" << std::endl;
429 
430  clp.printFinalTimerSummary(out.ptr());
431 
432  return (success ? 0 : 1);
433 
434 }
435 
436 
438  const std::string groupName, const std::string testName_in )
439 {
440  InstanceData &data = getData();
441  std::string testName = testName_in;
442  data.unitTests.push_back(UnitTestData(unitTest, groupName, testName));
443 }
444 
445 
447 {
448  return (getData().showTestDetails == SHOW_TEST_DETAILS_ALL);
449 }
450 
451 
452 // private:
453 
454 
455 UnitTestRepository::UnitTestRepository()
456 {}
457 
458 
459 void UnitTestRepository::setUpCLP(const Ptr<CommandLineProcessor>& clp)
460 {
461 
462  clp->addOutputSetupOptions(true);
463 
464  const int numShowTestDetails = 3;
465  const EShowTestDetails showTestDetailsValues[numShowTestDetails] =
466  { SHOW_TEST_DETAILS_ALL,
467  SHOW_TEST_DETAILS_TEST_NAMES,
468  SHOW_TEST_DETAILS_FINAL_RESULTS
469  };
470  const char* showTestDetailsNames[numShowTestDetails] =
471  { "ALL",
472  "TEST_NAMES",
473  "FINAL_RESULTS"
474  };
475  clp->setOption(
476  "show-test-details", &getData().showTestDetails,
477  numShowTestDetails, showTestDetailsValues, showTestDetailsNames,
478  "Level of detail to show in the tests"
479  );
480  clp->setOption(
481  "details", &getData().showTestDetails,
482  numShowTestDetails, showTestDetailsValues, showTestDetailsNames,
483  "Short for --show-test-details"
484  );
485 
486  clp->setOption(
487  "show-src-location", "no-show-src-location", &getData().showSrcLocation,
488  "If true, then the location of the unit test source code is shown."
489  " Only meaningful if --show-test-details=ALL."
490  );
491 
492  clp->setOption(
493  "show-fail-src-location", "no-show-fail-src-location", &getData().showFailSrcLocation,
494  "If true, then the location of every failed unit test check is printed."
495  );
496 
497  clp->setOption(
498  "globally-reduce-test-result", "no-globally-reduce-test-result",
499  &getData().globallyReduceUnitTestResult,
500  "If true, individual unit test pass/fail is globally reduced across MPI processes."
501  );
502 
503  clp->setOption(
504  "group-name", &getData().groupName,
505  "If specified, selects only tests that match the group name glob." );
506  clp->setOption(
507  "group", &getData().groupName,
508  "Short for --group-name." );
509 
510  clp->setOption(
511  "test-name", &getData().testName,
512  "If specified, selects only tests that match the test name glob." );
513  clp->setOption(
514  "test", &getData().testName,
515  "Short for --test-name." );
516 
517  clp->setOption(
518  "not-unit-test", &getData().notUnitTestName,
519  "If specified, full unit tests with glob matches will *not* be run." );
520 
521  clp->setOption(
522  "no-op", "do-op", &getData().noOp,
523  "If --no-op, then only the names of the tests that would be run are run."
524  );
525 
526 }
527 
528 
529 UnitTestRepository::InstanceData& UnitTestRepository::getData()
530 {
531  static UnitTestRepository::InstanceData data;
532  return data;
533 }
534 
535 
536 bool UnitTestRepository::runUnitTestImpl(const UnitTestBase &unitTest,
537  FancyOStream &out)
538 {
539  const bool result = unitTest.runUnitTest(out);
540  if (getData().globallyReduceUnitTestResult) {
541  const int globalSum = GlobalMPISession::sum(result ? 0 : 1);
542  if (globalSum == 0) {
543  return true;
544  }
545  else {
546  // Only print that there are failures on processes where the local
547  // unit test actally passed. On processes where the local unit test
548  // fails, users already know that test failed so there is no need to
549  // exlain it.
550  if (result) {
551  out << "NOTE: Global reduction shows failures on other processes!\n"
552  << "(rerun with --output-to-root-rank-only=-1 to see output\n"
553  << "from other processes to see what process failed!)\n";
554  }
555  else {
556  // The test failed on the root process so the user already knows it failed!
557  }
558  // Determine what processes have failing tests
559  const int numProcs = GlobalMPISession::getNProc();
560  Array<int> passFailFlags(numProcs);
561  GlobalMPISession::allGather( result ? 0 : 1, passFailFlags());
562  Array<int> procsThatFailed;
563  for ( int proc_k = 0; proc_k < numProcs; ++proc_k ) {
564  if (passFailFlags[proc_k] != 0) {
565  procsThatFailed.push_back(proc_k);
566  }
567  }
568  // Print what processes have the failing tests. If there is only one
569  // processes, don't print anything.
570  if (numProcs > 1) {
571  if (procsThatFailed.size() == numProcs) {
572  out << "NOTE: Unit test failed on all processes!\n";
573  // NOTE: when all the processes are failing it is useless to print
574  // out a list of all of the processes.
575  }
576  else {
577  out << "NOTE: Unit test failed on processes = " << procsThatFailed << "\n"
578  << "(rerun with --output-to-root-rank-only=<procID> to see output\n"
579  << "from individual processes where the unit test is failing!)\n";
580  }
581  }
582  return false;
583  }
584  }
585  return result;
586 }
587 
588 
589 } // namespace Teuchos
static bool verboseUnitTests()
Returns if unit tests are verbose or not.
static int getNProc()
The number of processes in MPI_COMM_WORLD.
static CommandLineProcessor & getCLP()
Return the CLP to add options to.
bool is_null(const std::shared_ptr< T > &p)
Returns true if p.get()==NULL.
Basic wall-clock timer class.
Unit testing support.
basic_FancyOStream< char > FancyOStream
Teuchos header file which uses auto-configuration information to include necessary C++ headers...
Tabbing class for helping to create formated, indented output for a basic_FancyOStream object...
static void addUnitTest(UnitTestBase *unitTest, const std::string groupName, const std::string testName)
Add an unit test (called indirectly through macros.
Unit testing support.
void showTestFailureLocation(bool)
Set if TEUCHOS_PASS_FAIL(...) should print test failure location.
static int runUnitTestsFromMain(int argc, char *argv[])
Run the unit tests from main() passing in (argc, argv).
void start(bool reset=false)
Start the timer, if the timer is enabled (see disable()).
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)
Deprecated.
double stop()
Stop the timer, if the timer is enabled (see disable()).
std::ostream subclass that performs the magic of indenting data sent to an std::ostream object among ...
Wall-clock timer.
Ptr< T > ptr() const
Get a safer wrapper raw C++ pointer to the underlying object.
static RCP< FancyOStream > getDefaultOStream()
Get the default output stream object.
EParseCommandLineReturn parse(int argc, char *argv[], std::ostream *errout=&std::cerr) const
Parse a command line.
static void setGloballyReduceTestResult(const bool globallyReduceUnitTestResult)
Set if the unit tests should reduce pass/fail across processes.
static int sum(int localVal)
Sum a set of integers across processes.
EParseCommandLineReturn
Return value for CommandLineProcessor::parse(). Note: These enums are all given non-negative values s...
Templated array class derived from the STL std::vector.
Utilities to make writing tests easier.
static bool runUnitTests(FancyOStream &out)
Run the registered unit tests.
void push_back(const value_type &x)
Basic command line parser for input from (argc,argv[])
static bool getGloballyReduceTestResult()
Get if the unit tests should reduce across processes or not.
size_type size() const
double totalElapsedTime(bool readCurrentTime=false) const
The total time in seconds accumulated by this timer.
static void allGather(int localVal, const ArrayView< int > &allVals)
Global all-to-all of a set of integers across processes.
void printFinalTimerSummary(const Ptr< std::ostream > &out=null)
Call to print timers so that they don&#39;t get printed in the destructor.
#define TEUCHOS_STANDARD_CATCH_STATEMENTS(VERBOSE, ERR_STREAM, SUCCESS_FLAG)
Simple macro that catches and reports standard exceptions and other exceptions.
#define TEUCHOS_ASSERT(assertion_test)
This macro is throws when an assert fails.
#define TEUCHOS_ASSERT_EQUALITY(val1, val2)
This macro is checks that to numbers are equal and if not then throws an exception with a good error ...
Unit test base class.
Class that helps parse command line input arguments from (argc,argv[]) and set options.