googletest/test/googletest-json-output-unittest.py

   1 #!/usr/bin/env python
   2 # Copyright 2018, Google Inc.
   3 # All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Unit test for the gtest_json_output module."""
  32
  33 import datetime
  34 import errno
  35 import json
  36 import os
  37 import re
  38 import sys
  39
  40 import gtest_json_test_utils
  41 import gtest_test_utils
  42
  43 GTEST_FILTER_FLAG = '--gtest_filter'
  44 GTEST_LIST_TESTS_FLAG = '--gtest_list_tests'
  45 GTEST_OUTPUT_FLAG = '--gtest_output'
  46 GTEST_DEFAULT_OUTPUT_FILE = 'test_detail.json'
  47 GTEST_PROGRAM_NAME = 'gtest_xml_output_unittest_'
  48
  49 # The flag indicating stacktraces are not supported
  50 NO_STACKTRACE_SUPPORT_FLAG = '--no_stacktrace_support'
  51
  52 SUPPORTS_STACK_TRACES = NO_STACKTRACE_SUPPORT_FLAG not in sys.argv
  53
  54 if SUPPORTS_STACK_TRACES:
  55   STACK_TRACE_TEMPLATE = '\nStack trace:\n*'
  56 else:
  57   STACK_TRACE_TEMPLATE = ''
  58
  59 EXPECTED_NON_EMPTY = {
  60     u'tests':
  61         26,
  62     u'failures':
  63         5,
  64     u'disabled':
  65         2,
  66     u'errors':
  67         0,
  68     u'timestamp':
  69         u'*',
  70     u'time':
  71         u'*',
  72     u'ad_hoc_property':
  73         u'42',
  74     u'name':
  75         u'AllTests',
  76     u'testsuites': [{
  77         u'name':
  78             u'SuccessfulTest',
  79         u'tests':
  80             1,
  81         u'failures':
  82             0,
  83         u'disabled':
  84             0,
  85         u'errors':
  86             0,
  87         u'time':
  88             u'*',
  89         u'timestamp':
  90             u'*',
  91         u'testsuite': [{
  92             u'name': u'Succeeds',
  93             u'status': u'RUN',
  94             u'result': u'COMPLETED',
  95             u'time': u'*',
  96             u'timestamp': u'*',
  97             u'classname': u'SuccessfulTest'
  98         }]
  99     }, {
 100         u'name':
 101             u'FailedTest',
 102         u'tests':
 103             1,
 104         u'failures':
 105             1,
 106         u'disabled':
 107             0,
 108         u'errors':
 109             0,
 110         u'time':
 111             u'*',
 112         u'timestamp':
 113             u'*',
 114         u'testsuite': [{
 115             u'name':
 116                 u'Fails',
 117             u'status':
 118                 u'RUN',
 119             u'result':
 120                 u'COMPLETED',
 121             u'time':
 122                 u'*',
 123             u'timestamp':
 124                 u'*',
 125             u'classname':
 126                 u'FailedTest',
 127             u'failures': [{
 128                 u'failure': u'gtest_xml_output_unittest_.cc:*\n'
 129                             u'Expected equality of these values:\n'
 130                             u'  1\n  2' + STACK_TRACE_TEMPLATE,
 131                 u'type': u''
 132             }]
 133         }]
 134     }, {
 135         u'name':
 136             u'DisabledTest',
 137         u'tests':
 138             1,
 139         u'failures':
 140             0,
 141         u'disabled':
 142             1,
 143         u'errors':
 144             0,
 145         u'time':
 146             u'*',
 147         u'timestamp':
 148             u'*',
 149         u'testsuite': [{
 150             u'name': u'DISABLED_test_not_run',
 151             u'status': u'NOTRUN',
 152             u'result': u'SUPPRESSED',
 153             u'time': u'*',
 154             u'timestamp': u'*',
 155             u'classname': u'DisabledTest'
 156         }]
 157     }, {
 158         u'name':
 159             u'SkippedTest',
 160         u'tests':
 161             3,
 162         u'failures':
 163             1,
 164         u'disabled':
 165             0,
 166         u'errors':
 167             0,
 168         u'time':
 169             u'*',
 170         u'timestamp':
 171             u'*',
 172         u'testsuite': [{
 173             u'name': u'Skipped',
 174             u'status': u'RUN',
 175             u'result': u'SKIPPED',
 176             u'time': u'*',
 177             u'timestamp': u'*',
 178             u'classname': u'SkippedTest'
 179         }, {
 180             u'name': u'SkippedWithMessage',
 181             u'status': u'RUN',
 182             u'result': u'SKIPPED',
 183             u'time': u'*',
 184             u'timestamp': u'*',
 185             u'classname': u'SkippedTest'
 186         }, {
 187             u'name':
 188                 u'SkippedAfterFailure',
 189             u'status':
 190                 u'RUN',
 191             u'result':
 192                 u'COMPLETED',
 193             u'time':
 194                 u'*',
 195             u'timestamp':
 196                 u'*',
 197             u'classname':
 198                 u'SkippedTest',
 199             u'failures': [{
 200                 u'failure': u'gtest_xml_output_unittest_.cc:*\n'
 201                             u'Expected equality of these values:\n'
 202                             u'  1\n  2' + STACK_TRACE_TEMPLATE,
 203                 u'type': u''
 204             }]
 205         }]
 206     }, {
 207         u'name':
 208             u'MixedResultTest',
 209         u'tests':
 210             3,
 211         u'failures':
 212             1,
 213         u'disabled':
 214             1,
 215         u'errors':
 216             0,
 217         u'time':
 218             u'*',
 219         u'timestamp':
 220             u'*',
 221         u'testsuite': [{
 222             u'name': u'Succeeds',
 223             u'status': u'RUN',
 224             u'result': u'COMPLETED',
 225             u'time': u'*',
 226             u'timestamp': u'*',
 227             u'classname': u'MixedResultTest'
 228         }, {
 229             u'name':
 230                 u'Fails',
 231             u'status':
 232                 u'RUN',
 233             u'result':
 234                 u'COMPLETED',
 235             u'time':
 236                 u'*',
 237             u'timestamp':
 238                 u'*',
 239             u'classname':
 240                 u'MixedResultTest',
 241             u'failures': [{
 242                 u'failure': u'gtest_xml_output_unittest_.cc:*\n'
 243                             u'Expected equality of these values:\n'
 244                             u'  1\n  2' + STACK_TRACE_TEMPLATE,
 245                 u'type': u''
 246             }, {
 247                 u'failure': u'gtest_xml_output_unittest_.cc:*\n'
 248                             u'Expected equality of these values:\n'
 249                             u'  2\n  3' + STACK_TRACE_TEMPLATE,
 250                 u'type': u''
 251             }]
 252         }, {
 253             u'name': u'DISABLED_test',
 254             u'status': u'NOTRUN',
 255             u'result': u'SUPPRESSED',
 256             u'time': u'*',
 257             u'timestamp': u'*',
 258             u'classname': u'MixedResultTest'
 259         }]
 260     }, {
 261         u'name':
 262             u'XmlQuotingTest',
 263         u'tests':
 264             1,
 265         u'failures':
 266             1,
 267         u'disabled':
 268             0,
 269         u'errors':
 270             0,
 271         u'time':
 272             u'*',
 273         u'timestamp':
 274             u'*',
 275         u'testsuite': [{
 276             u'name':
 277                 u'OutputsCData',
 278             u'status':
 279                 u'RUN',
 280             u'result':
 281                 u'COMPLETED',
 282             u'time':
 283                 u'*',
 284             u'timestamp':
 285                 u'*',
 286             u'classname':
 287                 u'XmlQuotingTest',
 288             u'failures': [{
 289                 u'failure': u'gtest_xml_output_unittest_.cc:*\n'
 290                             u'Failed\nXML output: <?xml encoding="utf-8">'
 291                             u'<top><![CDATA[cdata text]]></top>' +
 292                             STACK_TRACE_TEMPLATE,
 293                 u'type': u''
 294             }]
 295         }]
 296     }, {
 297         u'name':
 298             u'InvalidCharactersTest',
 299         u'tests':
 300             1,
 301         u'failures':
 302             1,
 303         u'disabled':
 304             0,
 305         u'errors':
 306             0,
 307         u'time':
 308             u'*',
 309         u'timestamp':
 310             u'*',
 311         u'testsuite': [{
 312             u'name':
 313                 u'InvalidCharactersInMessage',
 314             u'status':
 315                 u'RUN',
 316             u'result':
 317                 u'COMPLETED',
 318             u'time':
 319                 u'*',
 320             u'timestamp':
 321                 u'*',
 322             u'classname':
 323                 u'InvalidCharactersTest',
 324             u'failures': [{
 325                 u'failure': u'gtest_xml_output_unittest_.cc:*\n'
 326                             u'Failed\nInvalid characters in brackets'
 327                             u' [\x01\x02]' + STACK_TRACE_TEMPLATE,
 328                 u'type': u''
 329             }]
 330         }]
 331     }, {
 332         u'name':
 333             u'PropertyRecordingTest',
 334         u'tests':
 335             4,
 336         u'failures':
 337             0,
 338         u'disabled':
 339             0,
 340         u'errors':
 341             0,
 342         u'time':
 343             u'*',
 344         u'timestamp':
 345             u'*',
 346         u'SetUpTestSuite':
 347             u'yes',
 348         u'TearDownTestSuite':
 349             u'aye',
 350         u'testsuite': [{
 351             u'name': u'OneProperty',
 352             u'status': u'RUN',
 353             u'result': u'COMPLETED',
 354             u'time': u'*',
 355             u'timestamp': u'*',
 356             u'classname': u'PropertyRecordingTest',
 357             u'key_1': u'1'
 358         }, {
 359             u'name': u'IntValuedProperty',
 360             u'status': u'RUN',
 361             u'result': u'COMPLETED',
 362             u'time': u'*',
 363             u'timestamp': u'*',
 364             u'classname': u'PropertyRecordingTest',
 365             u'key_int': u'1'
 366         }, {
 367             u'name': u'ThreeProperties',
 368             u'status': u'RUN',
 369             u'result': u'COMPLETED',
 370             u'time': u'*',
 371             u'timestamp': u'*',
 372             u'classname': u'PropertyRecordingTest',
 373             u'key_1': u'1',
 374             u'key_2': u'2',
 375             u'key_3': u'3'
 376         }, {
 377             u'name': u'TwoValuesForOneKeyUsesLastValue',
 378             u'status': u'RUN',
 379             u'result': u'COMPLETED',
 380             u'time': u'*',
 381             u'timestamp': u'*',
 382             u'classname': u'PropertyRecordingTest',
 383             u'key_1': u'2'
 384         }]
 385     }, {
 386         u'name':
 387             u'NoFixtureTest',
 388         u'tests':
 389             3,
 390         u'failures':
 391             0,
 392         u'disabled':
 393             0,
 394         u'errors':
 395             0,
 396         u'time':
 397             u'*',
 398         u'timestamp':
 399             u'*',
 400         u'testsuite': [{
 401             u'name': u'RecordProperty',
 402             u'status': u'RUN',
 403             u'result': u'COMPLETED',
 404             u'time': u'*',
 405             u'timestamp': u'*',
 406             u'classname': u'NoFixtureTest',
 407             u'key': u'1'
 408         }, {
 409             u'name': u'ExternalUtilityThatCallsRecordIntValuedProperty',
 410             u'status': u'RUN',
 411             u'result': u'COMPLETED',
 412             u'time': u'*',
 413             u'timestamp': u'*',
 414             u'classname': u'NoFixtureTest',
 415             u'key_for_utility_int': u'1'
 416         }, {
 417             u'name': u'ExternalUtilityThatCallsRecordStringValuedProperty',
 418             u'status': u'RUN',
 419             u'result': u'COMPLETED',
 420             u'time': u'*',
 421             u'timestamp': u'*',
 422             u'classname': u'NoFixtureTest',
 423             u'key_for_utility_string': u'1'
 424         }]
 425     }, {
 426         u'name':
 427             u'TypedTest/0',
 428         u'tests':
 429             1,
 430         u'failures':
 431             0,
 432         u'disabled':
 433             0,
 434         u'errors':
 435             0,
 436         u'time':
 437             u'*',
 438         u'timestamp':
 439             u'*',
 440         u'testsuite': [{
 441             u'name': u'HasTypeParamAttribute',
 442             u'type_param': u'int',
 443             u'status': u'RUN',
 444             u'result': u'COMPLETED',
 445             u'time': u'*',
 446             u'timestamp': u'*',
 447             u'classname': u'TypedTest/0'
 448         }]
 449     }, {
 450         u'name':
 451             u'TypedTest/1',
 452         u'tests':
 453             1,
 454         u'failures':
 455             0,
 456         u'disabled':
 457             0,
 458         u'errors':
 459             0,
 460         u'time':
 461             u'*',
 462         u'timestamp':
 463             u'*',
 464         u'testsuite': [{
 465             u'name': u'HasTypeParamAttribute',
 466             u'type_param': u'long',
 467             u'status': u'RUN',
 468             u'result': u'COMPLETED',
 469             u'time': u'*',
 470             u'timestamp': u'*',
 471             u'classname': u'TypedTest/1'
 472         }]
 473     }, {
 474         u'name':
 475             u'Single/TypeParameterizedTestSuite/0',
 476         u'tests':
 477             1,
 478         u'failures':
 479             0,
 480         u'disabled':
 481             0,
 482         u'errors':
 483             0,
 484         u'time':
 485             u'*',
 486         u'timestamp':
 487             u'*',
 488         u'testsuite': [{
 489             u'name': u'HasTypeParamAttribute',
 490             u'type_param': u'int',
 491             u'status': u'RUN',
 492             u'result': u'COMPLETED',
 493             u'time': u'*',
 494             u'timestamp': u'*',
 495             u'classname': u'Single/TypeParameterizedTestSuite/0'
 496         }]
 497     }, {
 498         u'name':
 499             u'Single/TypeParameterizedTestSuite/1',
 500         u'tests':
 501             1,
 502         u'failures':
 503             0,
 504         u'disabled':
 505             0,
 506         u'errors':
 507             0,
 508         u'time':
 509             u'*',
 510         u'timestamp':
 511             u'*',
 512         u'testsuite': [{
 513             u'name': u'HasTypeParamAttribute',
 514             u'type_param': u'long',
 515             u'status': u'RUN',
 516             u'result': u'COMPLETED',
 517             u'time': u'*',
 518             u'timestamp': u'*',
 519             u'classname': u'Single/TypeParameterizedTestSuite/1'
 520         }]
 521     }, {
 522         u'name':
 523             u'Single/ValueParamTest',
 524         u'tests':
 525             4,
 526         u'failures':
 527             0,
 528         u'disabled':
 529             0,
 530         u'errors':
 531             0,
 532         u'time':
 533             u'*',
 534         u'timestamp':
 535             u'*',
 536         u'testsuite': [{
 537             u'name': u'HasValueParamAttribute/0',
 538             u'value_param': u'33',
 539             u'status': u'RUN',
 540             u'result': u'COMPLETED',
 541             u'time': u'*',
 542             u'timestamp': u'*',
 543             u'classname': u'Single/ValueParamTest'
 544         }, {
 545             u'name': u'HasValueParamAttribute/1',
 546             u'value_param': u'42',
 547             u'status': u'RUN',
 548             u'result': u'COMPLETED',
 549             u'time': u'*',
 550             u'timestamp': u'*',
 551             u'classname': u'Single/ValueParamTest'
 552         }, {
 553             u'name': u'AnotherTestThatHasValueParamAttribute/0',
 554             u'value_param': u'33',
 555             u'status': u'RUN',
 556             u'result': u'COMPLETED',
 557             u'time': u'*',
 558             u'timestamp': u'*',
 559             u'classname': u'Single/ValueParamTest'
 560         }, {
 561             u'name': u'AnotherTestThatHasValueParamAttribute/1',
 562             u'value_param': u'42',
 563             u'status': u'RUN',
 564             u'result': u'COMPLETED',
 565             u'time': u'*',
 566             u'timestamp': u'*',
 567             u'classname': u'Single/ValueParamTest'
 568         }]
 569     }]
 570 }
 571
 572 EXPECTED_FILTERED = {
 573     u'tests':
 574         1,
 575     u'failures':
 576         0,
 577     u'disabled':
 578         0,
 579     u'errors':
 580         0,
 581     u'time':
 582         u'*',
 583     u'timestamp':
 584         u'*',
 585     u'name':
 586         u'AllTests',
 587     u'ad_hoc_property':
 588         u'42',
 589     u'testsuites': [{
 590         u'name':
 591             u'SuccessfulTest',
 592         u'tests':
 593             1,
 594         u'failures':
 595             0,
 596         u'disabled':
 597             0,
 598         u'errors':
 599             0,
 600         u'time':
 601             u'*',
 602         u'timestamp':
 603             u'*',
 604         u'testsuite': [{
 605             u'name': u'Succeeds',
 606             u'status': u'RUN',
 607             u'result': u'COMPLETED',
 608             u'time': u'*',
 609             u'timestamp': u'*',
 610             u'classname': u'SuccessfulTest',
 611         }]
 612     }],
 613 }
 614
 615 EXPECTED_NO_TEST = {
 616     u'tests':
 617         0,
 618     u'failures':
 619         0,
 620     u'disabled':
 621         0,
 622     u'errors':
 623         0,
 624     u'time':
 625         u'*',
 626     u'timestamp':
 627         u'*',
 628     u'name':
 629         u'AllTests',
 630     u'testsuites': [{
 631         u'name':
 632             u'NonTestSuiteFailure',
 633         u'tests':
 634             1,
 635         u'failures':
 636             1,
 637         u'disabled':
 638             0,
 639         u'skipped':
 640             0,
 641         u'errors':
 642             0,
 643         u'time':
 644             u'*',
 645         u'timestamp':
 646             u'*',
 647         u'testsuite': [{
 648             u'name':
 649                 u'',
 650             u'status':
 651                 u'RUN',
 652             u'result':
 653                 u'COMPLETED',
 654             u'time':
 655                 u'*',
 656             u'timestamp':
 657                 u'*',
 658             u'classname':
 659                 u'',
 660             u'failures': [{
 661                 u'failure': u'gtest_no_test_unittest.cc:*\n'
 662                             u'Expected equality of these values:\n'
 663                             u'  1\n  2' + STACK_TRACE_TEMPLATE,
 664                 u'type': u'',
 665             }]
 666         }]
 667     }],
 668 }
 669
 670 GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME)
 671
 672 SUPPORTS_TYPED_TESTS = 'TypedTest' in gtest_test_utils.Subprocess(
 673     [GTEST_PROGRAM_PATH, GTEST_LIST_TESTS_FLAG], capture_stderr=False).output
 674
 675
 676 class GTestJsonOutputUnitTest(gtest_test_utils.TestCase):
 677   """Unit test for Google Test's JSON output functionality.
 678   """
 679
 680   # This test currently breaks on platforms that do not support typed and
 681   # type-parameterized tests, so we don't run it under them.
 682   if SUPPORTS_TYPED_TESTS:
 683
 684     def testNonEmptyJsonOutput(self):
 685       """Verifies JSON output for a Google Test binary with non-empty output.
 686
 687       Runs a test program that generates a non-empty JSON output, and
 688       tests that the JSON output is expected.
 689       """
 690       self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_NON_EMPTY, 1)
 691
 692   def testNoTestJsonOutput(self):
 693     """Verifies JSON output for a Google Test binary without actual tests.
 694
 695     Runs a test program that generates an JSON output for a binary with no
 696     tests, and tests that the JSON output is expected.
 697     """
 698
 699     self._TestJsonOutput('gtest_no_test_unittest', EXPECTED_NO_TEST, 0)
 700
 701   def testTimestampValue(self):
 702     """Checks whether the timestamp attribute in the JSON output is valid.
 703
 704     Runs a test program that generates an empty JSON output, and checks if
 705     the timestamp attribute in the testsuites tag is valid.
 706     """
 707     actual = self._GetJsonOutput('gtest_no_test_unittest', [], 0)
 708     date_time_str = actual['timestamp']
 709     # datetime.strptime() is only available in Python 2.5+ so we have to
 710     # parse the expected datetime manually.
 711     match = re.match(r'(\d+)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)', date_time_str)
 712     self.assertTrue(
 713         re.match,
 714         'JSON datettime string %s has incorrect format' % date_time_str)
 715     date_time_from_json = datetime.datetime(
 716         year=int(match.group(1)), month=int(match.group(2)),
 717         day=int(match.group(3)), hour=int(match.group(4)),
 718         minute=int(match.group(5)), second=int(match.group(6)))
 719
 720     time_delta = abs(datetime.datetime.now() - date_time_from_json)
 721     # timestamp value should be near the current local time
 722     self.assertTrue(time_delta < datetime.timedelta(seconds=600),
 723                     'time_delta is %s' % time_delta)
 724
 725   def testDefaultOutputFile(self):
 726     """Verifies the default output file name.
 727
 728     Confirms that Google Test produces an JSON output file with the expected
 729     default name if no name is explicitly specified.
 730     """
 731     output_file = os.path.join(gtest_test_utils.GetTempDir(),
 732                                GTEST_DEFAULT_OUTPUT_FILE)
 733     gtest_prog_path = gtest_test_utils.GetTestExecutablePath(
 734         'gtest_no_test_unittest')
 735     try:
 736       os.remove(output_file)
 737     except OSError:
 738       e = sys.exc_info()[1]
 739       if e.errno != errno.ENOENT:
 740         raise
 741
 742     p = gtest_test_utils.Subprocess(
 743         [gtest_prog_path, '%s=json' % GTEST_OUTPUT_FLAG],
 744         working_dir=gtest_test_utils.GetTempDir())
 745     self.assert_(p.exited)
 746     self.assertEquals(0, p.exit_code)
 747     self.assert_(os.path.isfile(output_file))
 748
 749   def testSuppressedJsonOutput(self):
 750     """Verifies that no JSON output is generated.
 751
 752     Tests that no JSON file is generated if the default JSON listener is
 753     shut down before RUN_ALL_TESTS is invoked.
 754     """
 755
 756     json_path = os.path.join(gtest_test_utils.GetTempDir(),
 757                              GTEST_PROGRAM_NAME + 'out.json')
 758     if os.path.isfile(json_path):
 759       os.remove(json_path)
 760
 761     command = [GTEST_PROGRAM_PATH,
 762                '%s=json:%s' % (GTEST_OUTPUT_FLAG, json_path),
 763                '--shut_down_xml']
 764     p = gtest_test_utils.Subprocess(command)
 765     if p.terminated_by_signal:
 766       # p.signal is available only if p.terminated_by_signal is True.
 767       self.assertFalse(
 768           p.terminated_by_signal,
 769           '%s was killed by signal %d' % (GTEST_PROGRAM_NAME, p.signal))
 770     else:
 771       self.assert_(p.exited)
 772       self.assertEquals(1, p.exit_code,
 773                         "'%s' exited with code %s, which doesn't match "
 774                         'the expected exit code %s.'
 775                         % (command, p.exit_code, 1))
 776
 777     self.assert_(not os.path.isfile(json_path))
 778
 779   def testFilteredTestJsonOutput(self):
 780     """Verifies JSON output when a filter is applied.
 781
 782     Runs a test program that executes only some tests and verifies that
 783     non-selected tests do not show up in the JSON output.
 784     """
 785
 786     self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_FILTERED, 0,
 787                          extra_args=['%s=SuccessfulTest.*' % GTEST_FILTER_FLAG])
 788
 789   def _GetJsonOutput(self, gtest_prog_name, extra_args, expected_exit_code):
 790     """Returns the JSON output generated by running the program gtest_prog_name.
 791
 792     Furthermore, the program's exit code must be expected_exit_code.
 793
 794     Args:
 795       gtest_prog_name: Google Test binary name.
 796       extra_args: extra arguments to binary invocation.
 797       expected_exit_code: program's exit code.
 798     """
 799     json_path = os.path.join(gtest_test_utils.GetTempDir(),
 800                              gtest_prog_name + 'out.json')
 801     gtest_prog_path = gtest_test_utils.GetTestExecutablePath(gtest_prog_name)
 802
 803     command = (
 804         [gtest_prog_path, '%s=json:%s' % (GTEST_OUTPUT_FLAG, json_path)] +
 805         extra_args
 806     )
 807     p = gtest_test_utils.Subprocess(command)
 808     if p.terminated_by_signal:
 809       self.assert_(False,
 810                    '%s was killed by signal %d' % (gtest_prog_name, p.signal))
 811     else:
 812       self.assert_(p.exited)
 813       self.assertEquals(expected_exit_code, p.exit_code,
 814                         "'%s' exited with code %s, which doesn't match "
 815                         'the expected exit code %s.'
 816                         % (command, p.exit_code, expected_exit_code))
 817     with open(json_path) as f:
 818       actual = json.load(f)
 819     return actual
 820
 821   def _TestJsonOutput(self, gtest_prog_name, expected,
 822                       expected_exit_code, extra_args=None):
 823     """Checks the JSON output generated by the Google Test binary.
 824
 825     Asserts that the JSON document generated by running the program
 826     gtest_prog_name matches expected_json, a string containing another
 827     JSON document.  Furthermore, the program's exit code must be
 828     expected_exit_code.
 829
 830     Args:
 831       gtest_prog_name: Google Test binary name.
 832       expected: expected output.
 833       expected_exit_code: program's exit code.
 834       extra_args: extra arguments to binary invocation.
 835     """
 836
 837     actual = self._GetJsonOutput(gtest_prog_name, extra_args or [],
 838                                  expected_exit_code)
 839     self.assertEqual(expected, gtest_json_test_utils.normalize(actual))
 840
 841
 842 if __name__ == '__main__':
 843   if NO_STACKTRACE_SUPPORT_FLAG in sys.argv:
 844     # unittest.main() can't handle unknown flags
 845     sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG)
 846
 847   os.environ['GTEST_STACK_TRACE_DEPTH'] = '1'
 848   gtest_test_utils.Main()