From c7a7c8acca83b440ad106f0fc48e45b9d9ea5f29 Mon Sep 17 00:00:00 2001 From: Zachary Turner Date: Thu, 28 May 2015 19:56:26 +0000 Subject: [PATCH] Refactor test runner to print sub-test-case pass/fail rate. llvm-svn: 238467 --- lldb/test/api/multithreaded/Makefile | 2 +- lldb/test/dosep.py | 68 +++++++++++++++++++++++++----------- lldb/test/unittest2/result.py | 2 ++ lldb/test/unittest2/runner.py | 35 +++++++++---------- 4 files changed, 67 insertions(+), 40 deletions(-) diff --git a/lldb/test/api/multithreaded/Makefile b/lldb/test/api/multithreaded/Makefile index 8b2b6cf..55f21cb 100644 --- a/lldb/test/api/multithreaded/Makefile +++ b/lldb/test/api/multithreaded/Makefile @@ -6,4 +6,4 @@ CXX_SOURCES := main.cpp include $(LEVEL)/Makefile.rules clean:: - rm -rf *.o *.d *.dSYM + rm $(wildcard -rf *.o *.d *.dSYM) diff --git a/lldb/test/dosep.py b/lldb/test/dosep.py index de42ea0..b8a017e 100755 --- a/lldb/test/dosep.py +++ b/lldb/test/dosep.py @@ -66,27 +66,47 @@ default_timeout = os.getenv("LLDB_TEST_TIMEOUT") or "10m" # Status codes for running command with timeout. eTimedOut, ePassed, eFailed = 124, 0, 1 +def parse_test_results(output): + passes = 0 + failures = 0 + for result in output: + pass_count = re.search("^RESULT:.*([0-9]+) passes", result, re.MULTILINE) + fail_count = re.search("^RESULT:.*([0-9]+) failures", result, re.MULTILINE) + error_count = re.search("^RESULT:.*([0-9]+) errors", result, re.MULTILINE) + this_fail_count = 0 + this_error_count = 0 + if pass_count != None: + passes = passes + int(pass_count.group(1)) + if fail_count != None: + failures = failures + int(fail_count.group(1)) + if error_count != None: + failures = failures + int(error_count.group(1)) + pass + return passes, failures + def call_with_timeout(command, timeout): """Run command with a timeout if possible.""" """-s QUIT will create a coredump if they are enabled on your system""" + process = None + if timeout_command and timeout != "0": + command = [timeout_command, '-s', 'QUIT', timeout] + command + # Specifying a value for close_fds is unsupported on Windows when using subprocess.PIPE if os.name != "nt": - if timeout_command and timeout != "0": - return subprocess.call([timeout_command, '-s', 'QUIT', timeout] + command, - stdin=subprocess.PIPE, close_fds=True) - return (ePassed if subprocess.call(command, stdin=subprocess.PIPE, close_fds=True) == 0 - else eFailed) + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) else: - if timeout_command and timeout != "0": - return subprocess.call([timeout_command, '-s', 'QUIT', timeout] + command, - stdin=subprocess.PIPE) - return (ePassed if subprocess.call(command, stdin=subprocess.PIPE) == 0 - else eFailed) + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output = process.communicate() + exit_status = process.returncode + passes, failures = parse_test_results(output) + return exit_status, passes, failures def process_dir(root, files, test_root, dotest_argv): """Examine a directory for tests, and invoke any found within it.""" timed_out = [] failed = [] passed = [] + pass_sub_count = 0 + fail_sub_count = 0 for name in files: path = os.path.join(root, name) @@ -107,15 +127,18 @@ def process_dir(root, files, test_root, dotest_argv): timeout = os.getenv("LLDB_%s_TIMEOUT" % timeout_name) or default_timeout - exit_status = call_with_timeout(command, timeout) + exit_status, pass_count, fail_count = call_with_timeout(command, timeout) - if ePassed == exit_status: + pass_sub_count = pass_sub_count + pass_count + fail_sub_count = fail_sub_count + fail_count + + if exit_status == ePassed: passed.append(name) else: if eTimedOut == exit_status: timed_out.append(name) failed.append(name) - return (timed_out, failed, passed) + return (timed_out, failed, passed, fail_sub_count, pass_sub_count) in_q = None out_q = None @@ -154,14 +177,18 @@ def walk_and_invoke(test_directory, test_subdir, dotest_argv, num_threads): timed_out = [] failed = [] passed = [] + fail_sub_count = 0 + pass_sub_count = 0 for test_result in test_results: - (dir_timed_out, dir_failed, dir_passed) = test_result + (dir_timed_out, dir_failed, dir_passed, dir_fail_sub_count, dir_pass_sub_count) = test_result timed_out += dir_timed_out failed += dir_failed passed += dir_passed + fail_sub_count = fail_sub_count + dir_fail_sub_count + pass_sub_count = pass_sub_count + dir_pass_sub_count - return (timed_out, failed, passed) + return (timed_out, failed, passed, fail_sub_count, pass_sub_count) def getExpectedTimeouts(platform_name): # returns a set of test filenames that might timeout @@ -294,10 +321,11 @@ Run lldb test suite using a separate process for each test file. num_threads = 1 system_info = " ".join(platform.uname()) - (timed_out, failed, passed) = walk_and_invoke(test_directory, test_subdir, dotest_argv, - num_threads) + (timed_out, failed, passed, all_fails, all_passes) = walk_and_invoke(test_directory, test_subdir, dotest_argv, num_threads) + timed_out = set(timed_out) - num_tests = len(failed) + len(passed) + num_test_files = len(failed) + len(passed) + num_tests = all_fails + all_passes # move core files into session dir cores = find('core.*', test_subdir) @@ -322,10 +350,10 @@ Run lldb test suite using a separate process for each test file. test_name = os.path.splitext(xtime)[0] touch(os.path.join(session_dir, "{}-{}".format(result, test_name))) - print "Ran %d tests." % num_tests + print "Ran %d test suites (%d failed) (%f%%)" % (num_test_files, len(failed), 100.0*len(failed)/num_test_files) + print "Ran %d test cases (%d failed) (%f%%)" % (num_tests, all_fails, 100.0*all_fails/num_tests) if len(failed) > 0: failed.sort() - print "Failing Tests (%d)" % len(failed) for f in failed: print "%s: LLDB (suite) :: %s (%s)" % ( "TIMEOUT" if f in timed_out else "FAIL", f, system_info diff --git a/lldb/test/unittest2/result.py b/lldb/test/unittest2/result.py index cacc0ee..72c6ccb 100644 --- a/lldb/test/unittest2/result.py +++ b/lldb/test/unittest2/result.py @@ -40,6 +40,7 @@ class TestResult(unittest.TestResult): def __init__(self): self.failfast = False self.failures = [] + self.passes = [] self.errors = [] self.testsRun = 0 self.skipped = [] @@ -117,6 +118,7 @@ class TestResult(unittest.TestResult): def addSuccess(self, test): "Called when a test has completed successfully" + self.passes.append(test) pass def addSkip(self, test, reason): diff --git a/lldb/test/unittest2/runner.py b/lldb/test/unittest2/runner.py index 4468f71..db0f89d 100644 --- a/lldb/test/unittest2/runner.py +++ b/lldb/test/unittest2/runner.py @@ -175,32 +175,29 @@ class TextTestRunner(unittest.TextTestRunner): (run, run != 1 and "s" or "", timeTaken)) self.stream.writeln() - expectedFails = unexpectedSuccesses = skipped = 0 + expectedFails = unexpectedSuccesses = skipped = passed = failed = errored = 0 try: results = map(len, (result.expectedFailures, result.unexpectedSuccesses, - result.skipped)) - expectedFails, unexpectedSuccesses, skipped = results + result.skipped, + result.passes, + result.failures, + result.errors)) + expectedFails, unexpectedSuccesses, skipped, passed, failed, errored = results except AttributeError: pass infos = [] + infos.append("%d passes" % passed) + infos.append("%d failures" % failed) + infos.append("%d errors" % errored) + infos.append("%d skipped" % skipped) + infos.append("%d expected failures" % expectedFails) + infos.append("%d unexpected successes" % unexpectedSuccesses) + self.stream.write("RESULT: ") if not result.wasSuccessful(): self.stream.write("FAILED") - failed, errored = map(len, (result.failures, result.errors)) - if failed: - infos.append("failures=%d" % failed) - if errored: - infos.append("errors=%d" % errored) else: - self.stream.write("OK") - if skipped: - infos.append("skipped=%d" % skipped) - if expectedFails: - infos.append("expected failures=%d" % expectedFails) - if unexpectedSuccesses: - infos.append("unexpected successes=%d" % unexpectedSuccesses) - if infos: - self.stream.writeln(" (%s)" % (", ".join(infos),)) - else: - self.stream.write("\n") + self.stream.write("PASSED") + + self.stream.writeln(" (%s)" % (", ".join(infos),)) return result -- 2.7.4