googletest/test/googletest-failfast-unittest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2020 Google Inc. All Rights Reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Unit test for Google Test fail_fast.
  32
  33 A user can specify if a Google Test program should continue test execution
  34 after a test failure via the GTEST_FAIL_FAST environment variable or the
  35 --gtest_fail_fast flag. The default value of the flag can also be changed
  36 by Bazel fail fast environment variable TESTBRIDGE_TEST_RUNNER_FAIL_FAST.
  37
  38 This script tests such functionality by invoking googletest-failfast-unittest_
  39 (a program written with Google Test) with different environments and command
  40 line flags.
  41 """
  42
  43 import os
  44 from googletest.test import gtest_test_utils
  45
  46 # Constants.
  47
  48 # Bazel testbridge environment variable for fail fast
  49 BAZEL_FAIL_FAST_ENV_VAR = 'TESTBRIDGE_TEST_RUNNER_FAIL_FAST'
  50
  51 # The environment variable for specifying fail fast.
  52 FAIL_FAST_ENV_VAR = 'GTEST_FAIL_FAST'
  53
  54 # The command line flag for specifying fail fast.
  55 FAIL_FAST_FLAG = 'gtest_fail_fast'
  56
  57 # The command line flag to run disabled tests.
  58 RUN_DISABLED_FLAG = 'gtest_also_run_disabled_tests'
  59
  60 # The command line flag for specifying a filter.
  61 FILTER_FLAG = 'gtest_filter'
  62
  63 # Command to run the googletest-failfast-unittest_ program.
  64 COMMAND = gtest_test_utils.GetTestExecutablePath(
  65     'googletest-failfast-unittest_')
  66
  67 # The command line flag to tell Google Test to output the list of tests it
  68 # will run.
  69 LIST_TESTS_FLAG = '--gtest_list_tests'
  70
  71 # Indicates whether Google Test supports death tests.
  72 SUPPORTS_DEATH_TESTS = 'HasDeathTest' in gtest_test_utils.Subprocess(
  73     [COMMAND, LIST_TESTS_FLAG]).output
  74
  75 # Utilities.
  76
  77 environ = os.environ.copy()
  78
  79
  80 def SetEnvVar(env_var, value):
  81   """Sets the env variable to 'value'; unsets it when 'value' is None."""
  82
  83   if value is not None:
  84     environ[env_var] = value
  85   elif env_var in environ:
  86     del environ[env_var]
  87
  88
  89 def RunAndReturnOutput(test_suite=None, fail_fast=None, run_disabled=False):
  90   """Runs the test program and returns its output."""
  91
  92   args = []
  93   xml_path = os.path.join(gtest_test_utils.GetTempDir(),
  94                           '.GTestFailFastUnitTest.xml')
  95   args += ['--gtest_output=xml:' + xml_path]
  96   if fail_fast is not None:
  97     if isinstance(fail_fast, str):
  98       args += ['--%s=%s' % (FAIL_FAST_FLAG, fail_fast)]
  99     elif fail_fast:
 100       args += ['--%s' % FAIL_FAST_FLAG]
 101     else:
 102       args += ['--no%s' % FAIL_FAST_FLAG]
 103   if test_suite:
 104     args += ['--%s=%s.*' % (FILTER_FLAG, test_suite)]
 105   if run_disabled:
 106     args += ['--%s' % RUN_DISABLED_FLAG]
 107   txt_out = gtest_test_utils.Subprocess([COMMAND] + args, env=environ).output
 108   with open(xml_path) as xml_file:
 109     return txt_out, xml_file.read()
 110
 111
 112 # The unit test.
 113 class GTestFailFastUnitTest(gtest_test_utils.TestCase):
 114   """Tests the env variable or the command line flag for fail_fast."""
 115
 116   def testDefaultBehavior(self):
 117     """Tests the behavior of not specifying the fail_fast."""
 118
 119     txt, _ = RunAndReturnOutput()
 120     self.assertIn('22 FAILED TEST', txt)
 121
 122   def testGoogletestFlag(self):
 123     txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=True)
 124     self.assertIn('1 FAILED TEST', txt)
 125     self.assertIn('[  SKIPPED ] 3 tests', txt)
 126
 127     txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=False)
 128     self.assertIn('4 FAILED TEST', txt)
 129     self.assertNotIn('[  SKIPPED ]', txt)
 130
 131   def testGoogletestEnvVar(self):
 132     """Tests the behavior of specifying fail_fast via Googletest env var."""
 133
 134     try:
 135       SetEnvVar(FAIL_FAST_ENV_VAR, '1')
 136       txt, _ = RunAndReturnOutput('HasSimpleTest')
 137       self.assertIn('1 FAILED TEST', txt)
 138       self.assertIn('[  SKIPPED ] 3 tests', txt)
 139
 140       SetEnvVar(FAIL_FAST_ENV_VAR, '0')
 141       txt, _ = RunAndReturnOutput('HasSimpleTest')
 142       self.assertIn('4 FAILED TEST', txt)
 143       self.assertNotIn('[  SKIPPED ]', txt)
 144     finally:
 145       SetEnvVar(FAIL_FAST_ENV_VAR, None)
 146
 147   def testBazelEnvVar(self):
 148     """Tests the behavior of specifying fail_fast via Bazel testbridge."""
 149
 150     try:
 151       SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '1')
 152       txt, _ = RunAndReturnOutput('HasSimpleTest')
 153       self.assertIn('1 FAILED TEST', txt)
 154       self.assertIn('[  SKIPPED ] 3 tests', txt)
 155
 156       SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0')
 157       txt, _ = RunAndReturnOutput('HasSimpleTest')
 158       self.assertIn('4 FAILED TEST', txt)
 159       self.assertNotIn('[  SKIPPED ]', txt)
 160     finally:
 161       SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None)
 162
 163   def testFlagOverridesEnvVar(self):
 164     """Tests precedence of flag over env var."""
 165
 166     try:
 167       SetEnvVar(FAIL_FAST_ENV_VAR, '0')
 168       txt, _ = RunAndReturnOutput('HasSimpleTest', True)
 169       self.assertIn('1 FAILED TEST', txt)
 170       self.assertIn('[  SKIPPED ] 3 tests', txt)
 171     finally:
 172       SetEnvVar(FAIL_FAST_ENV_VAR, None)
 173
 174   def testGoogletestEnvVarOverridesBazelEnvVar(self):
 175     """Tests that the Googletest native env var over Bazel testbridge."""
 176
 177     try:
 178       SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0')
 179       SetEnvVar(FAIL_FAST_ENV_VAR, '1')
 180       txt, _ = RunAndReturnOutput('HasSimpleTest')
 181       self.assertIn('1 FAILED TEST', txt)
 182       self.assertIn('[  SKIPPED ] 3 tests', txt)
 183     finally:
 184       SetEnvVar(FAIL_FAST_ENV_VAR, None)
 185       SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None)
 186
 187   def testEventListener(self):
 188     txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=True)
 189     self.assertIn('1 FAILED TEST', txt)
 190     self.assertIn('[  SKIPPED ] 3 tests', txt)
 191     for expected_count, callback in [(1, 'OnTestSuiteStart'),
 192                                      (5, 'OnTestStart'),
 193                                      (5, 'OnTestEnd'),
 194                                      (5, 'OnTestPartResult'),
 195                                      (1, 'OnTestSuiteEnd')]:
 196       self.assertEqual(
 197           expected_count, txt.count(callback),
 198           'Expected %d calls to callback %s match count on output: %s ' %
 199           (expected_count, callback, txt))
 200
 201     txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=False)
 202     self.assertIn('3 FAILED TEST', txt)
 203     self.assertIn('[  SKIPPED ] 1 test', txt)
 204     for expected_count, callback in [(1, 'OnTestSuiteStart'),
 205                                      (5, 'OnTestStart'),
 206                                      (5, 'OnTestEnd'),
 207                                      (5, 'OnTestPartResult'),
 208                                      (1, 'OnTestSuiteEnd')]:
 209       self.assertEqual(
 210           expected_count, txt.count(callback),
 211           'Expected %d calls to callback %s match count on output: %s ' %
 212           (expected_count, callback, txt))
 213
 214   def assertXmlResultCount(self, result, count, xml):
 215     self.assertEqual(
 216         count, xml.count('result="%s"' % result),
 217         'Expected \'result="%s"\' match count of %s: %s ' %
 218         (result, count, xml))
 219
 220   def assertXmlStatusCount(self, status, count, xml):
 221     self.assertEqual(
 222         count, xml.count('status="%s"' % status),
 223         'Expected \'status="%s"\' match count of %s: %s ' %
 224         (status, count, xml))
 225
 226   def assertFailFastXmlAndTxtOutput(self,
 227                                     fail_fast,
 228                                     test_suite,
 229                                     passed_count,
 230                                     failure_count,
 231                                     skipped_count,
 232                                     suppressed_count,
 233                                     run_disabled=False):
 234     """Assert XML and text output of a test execution."""
 235
 236     txt, xml = RunAndReturnOutput(test_suite, fail_fast, run_disabled)
 237     if failure_count > 0:
 238       self.assertIn('%s FAILED TEST' % failure_count, txt)
 239     if suppressed_count > 0:
 240       self.assertIn('%s DISABLED TEST' % suppressed_count, txt)
 241     if skipped_count > 0:
 242       self.assertIn('[  SKIPPED ] %s tests' % skipped_count, txt)
 243     self.assertXmlStatusCount('run',
 244                               passed_count + failure_count + skipped_count, xml)
 245     self.assertXmlStatusCount('notrun', suppressed_count, xml)
 246     self.assertXmlResultCount('completed', passed_count + failure_count, xml)
 247     self.assertXmlResultCount('skipped', skipped_count, xml)
 248     self.assertXmlResultCount('suppressed', suppressed_count, xml)
 249
 250   def assertFailFastBehavior(self,
 251                              test_suite,
 252                              passed_count,
 253                              failure_count,
 254                              skipped_count,
 255                              suppressed_count,
 256                              run_disabled=False):
 257     """Assert --fail_fast via flag."""
 258
 259     for fail_fast in ('true', '1', 't', True):
 260       self.assertFailFastXmlAndTxtOutput(fail_fast, test_suite, passed_count,
 261                                          failure_count, skipped_count,
 262                                          suppressed_count, run_disabled)
 263
 264   def assertNotFailFastBehavior(self,
 265                                 test_suite,
 266                                 passed_count,
 267                                 failure_count,
 268                                 skipped_count,
 269                                 suppressed_count,
 270                                 run_disabled=False):
 271     """Assert --nofail_fast via flag."""
 272
 273     for fail_fast in ('false', '0', 'f', False):
 274       self.assertFailFastXmlAndTxtOutput(fail_fast, test_suite, passed_count,
 275                                          failure_count, skipped_count,
 276                                          suppressed_count, run_disabled)
 277
 278   def testFlag_HasFixtureTest(self):
 279     """Tests the behavior of fail_fast and TEST_F."""
 280     self.assertFailFastBehavior(
 281         test_suite='HasFixtureTest',
 282         passed_count=1,
 283         failure_count=1,
 284         skipped_count=3,
 285         suppressed_count=0)
 286     self.assertNotFailFastBehavior(
 287         test_suite='HasFixtureTest',
 288         passed_count=1,
 289         failure_count=4,
 290         skipped_count=0,
 291         suppressed_count=0)
 292
 293   def testFlag_HasSimpleTest(self):
 294     """Tests the behavior of fail_fast and TEST."""
 295     self.assertFailFastBehavior(
 296         test_suite='HasSimpleTest',
 297         passed_count=1,
 298         failure_count=1,
 299         skipped_count=3,
 300         suppressed_count=0)
 301     self.assertNotFailFastBehavior(
 302         test_suite='HasSimpleTest',
 303         passed_count=1,
 304         failure_count=4,
 305         skipped_count=0,
 306         suppressed_count=0)
 307
 308   def testFlag_HasParametersTest(self):
 309     """Tests the behavior of fail_fast and TEST_P."""
 310     self.assertFailFastBehavior(
 311         test_suite='HasParametersSuite/HasParametersTest',
 312         passed_count=0,
 313         failure_count=1,
 314         skipped_count=3,
 315         suppressed_count=0)
 316     self.assertNotFailFastBehavior(
 317         test_suite='HasParametersSuite/HasParametersTest',
 318         passed_count=0,
 319         failure_count=4,
 320         skipped_count=0,
 321         suppressed_count=0)
 322
 323   def testFlag_HasDisabledTest(self):
 324     """Tests the behavior of fail_fast and Disabled test cases."""
 325     self.assertFailFastBehavior(
 326         test_suite='HasDisabledTest',
 327         passed_count=1,
 328         failure_count=1,
 329         skipped_count=2,
 330         suppressed_count=1,
 331         run_disabled=False)
 332     self.assertNotFailFastBehavior(
 333         test_suite='HasDisabledTest',
 334         passed_count=1,
 335         failure_count=3,
 336         skipped_count=0,
 337         suppressed_count=1,
 338         run_disabled=False)
 339
 340   def testFlag_HasDisabledRunDisabledTest(self):
 341     """Tests the behavior of fail_fast and Disabled test cases enabled."""
 342     self.assertFailFastBehavior(
 343         test_suite='HasDisabledTest',
 344         passed_count=1,
 345         failure_count=1,
 346         skipped_count=3,
 347         suppressed_count=0,
 348         run_disabled=True)
 349     self.assertNotFailFastBehavior(
 350         test_suite='HasDisabledTest',
 351         passed_count=1,
 352         failure_count=4,
 353         skipped_count=0,
 354         suppressed_count=0,
 355         run_disabled=True)
 356
 357   def testFlag_HasDisabledSuiteTest(self):
 358     """Tests the behavior of fail_fast and Disabled test suites."""
 359     self.assertFailFastBehavior(
 360         test_suite='DISABLED_HasDisabledSuite',
 361         passed_count=0,
 362         failure_count=0,
 363         skipped_count=0,
 364         suppressed_count=5,
 365         run_disabled=False)
 366     self.assertNotFailFastBehavior(
 367         test_suite='DISABLED_HasDisabledSuite',
 368         passed_count=0,
 369         failure_count=0,
 370         skipped_count=0,
 371         suppressed_count=5,
 372         run_disabled=False)
 373
 374   def testFlag_HasDisabledSuiteRunDisabledTest(self):
 375     """Tests the behavior of fail_fast and Disabled test suites enabled."""
 376     self.assertFailFastBehavior(
 377         test_suite='DISABLED_HasDisabledSuite',
 378         passed_count=1,
 379         failure_count=1,
 380         skipped_count=3,
 381         suppressed_count=0,
 382         run_disabled=True)
 383     self.assertNotFailFastBehavior(
 384         test_suite='DISABLED_HasDisabledSuite',
 385         passed_count=1,
 386         failure_count=4,
 387         skipped_count=0,
 388         suppressed_count=0,
 389         run_disabled=True)
 390
 391   if SUPPORTS_DEATH_TESTS:
 392
 393     def testFlag_HasDeathTest(self):
 394       """Tests the behavior of fail_fast and death tests."""
 395       self.assertFailFastBehavior(
 396           test_suite='HasDeathTest',
 397           passed_count=1,
 398           failure_count=1,
 399           skipped_count=3,
 400           suppressed_count=0)
 401       self.assertNotFailFastBehavior(
 402           test_suite='HasDeathTest',
 403           passed_count=1,
 404           failure_count=4,
 405           skipped_count=0,
 406           suppressed_count=0)
 407
 408
 409 if __name__ == '__main__':
 410   gtest_test_utils.Main()