From b00658d30872c49df93af880dd70ecc5794a9454 Mon Sep 17 00:00:00 2001 From: Stef Walter Date: Wed, 5 Mar 2014 10:12:00 +0100 Subject: [PATCH] test-driver: Use a single driver which supports both tap and simple Rename our GTest tap compiler so it's clearer what's going on. --- Makefile.am | 8 +- build/tap-driver | 287 ----------------------- build/{tap-compiler => tap-gtester} | 12 +- build/test-driver | 454 ++++++++++++++++++++++++++---------- 4 files changed, 342 insertions(+), 419 deletions(-) delete mode 100755 build/tap-driver rename build/{tap-compiler => tap-gtester} (97%) diff --git a/Makefile.am b/Makefile.am index 53976a9..72d1f9d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -147,14 +147,14 @@ CLEANFILES = \ EXTRA_DIST = \ COPYING.TESTS -LOG_DRIVER = $(srcdir)/build/tap-driver -LOG_COMPILER = $(srcdir)/build/tap-compiler +LOG_DRIVER = $(srcdir)/build/test-driver --format=tap +LOG_COMPILER = $(srcdir)/build/tap-gtester TESTS_ENVIRONMENT = LD_LIBRARY_PATH=$(builddir)/.libs GI_TYPELIB_PATH=$(builddir) TEST_EXTENSIONS = .py .js -PY_LOG_DRIVER = $(srcdir)/build/tap-driver +PY_LOG_DRIVER = $(srcdir)/build/test-driver --format=tap PY_LOG_COMPILER = $(srcdir)/build/tap-unittest -JS_LOG_DRIVER = $(srcdir)/build/test-driver +JS_LOG_DRIVER = $(srcdir)/build/test-driver --format=simple JS_LOG_COMPILER = gjs include build/Makefile.am diff --git a/build/tap-driver b/build/tap-driver deleted file mode 100755 index 1c5af40..0000000 --- a/build/tap-driver +++ /dev/null @@ -1,287 +0,0 @@ -#!/usr/bin/python - -# Copyright (C) 2013 Red Hat, Inc. -# -# Cockpit is free software; you can redistribute it and/or modify it -# under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation; either version 2.1 of the License, or -# (at your option) any later version. -# -# Cockpit is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Cockpit; If not, see . - -# -# This is a TAP driver for automake -# -# In particular it leaves stderr untouched, and is cleaner than the -# one implemented in shell that is making the rounds. -# -# This implements the automake "Custom Test Driver" protocol: -# https://www.gnu.org/software/automake/manual/html_node/Custom-Test-Drivers.html -# -# This consumes the Test Anything Protocol (ie: TAP) -# https://metacpan.org/pod/release/PETDANCE/Test-Harness-2.64/lib/Test/Harness/TAP.pod -# - -import argparse -import os -import select -import subprocess -import sys - -class Driver: - def __init__(self, command, args): - self.argv = command - self.output = "" - self.test_name = args.test_name - self.log = open(args.log_file, "w") - self.trs = open(args.trs_file, "w") - self.color_tests = args.color_tests - self.expect_failure = args.expect_failure - self.reported = { } - self.test_plan = None - self.late_plan = False - self.errored = False - self.bail_out = False - - def report(self, code, num, *args): - CODES = { - "XPASS": '\x1b[0;31m', # red - "FAIL": '\x1b[0;31m', # red - "PASS": '\x1b[0;32m', # grn - "XFAIL": '\x1b[1;32m', # lgn - "SKIP": '\x1b[1;34m', # blu - "ERROR": '\x1b[0;35m', # mgn - } - - # Print out to console - if self.color_tests: - if code in CODES: - sys.stdout.write(CODES[code]) - sys.stdout.write(code) - if self.color_tests: - sys.stdout.write('\x1b[m') - sys.stdout.write(": ") - sys.stdout.write(self.test_name) - sys.stdout.write(" ") - if num: - sys.stdout.write(str(num)) - sys.stdout.write(" ") - for arg in args: - sys.stdout.write(str(arg)) - sys.stdout.write("\n") - sys.stdout.flush() - - # Book keeping - if code in CODES: - if num != None: - self.reported[num] = code - self.trs.write(":test-result: %s\n" % code) - if code == "ERROR": - self.errored = True - - def result_pass(self, num, description): - if self.expect_failure: - self.report("XPASS", num, description) - else: - self.report("PASS", num, description) - - def result_fail(self, num, description): - if self.expect_failure: - self.report("XFAIL", num, description) - else: - self.report("FAIL", num, description) - - def result_skip(self, num, description, ok): - if self.expect_failure: - self.report("XFAIL", num, description) - else: - self.report("SKIP", num, description) - - def report_error(self, problem): - self.report("ERROR", None, problem) - - def consume_test_line(self, ok, data): - # It's an error if the caller sends a test plan in the middle of tests - if self.late_plan: - self.report_error("Got tests after late TAP test plan") - self.late_plan = False - - # Parse out a number and then description - (num, unused, description) = data.partition(" ") - try: - num = int(num) - except ValueError: - self.report_error("Invalid test number: %s" % data) - return - description = description.lstrip() - - # Special case if description starts with this, then skip - if description.lower().startswith("# skip"): - self.result_skip(num, description, ok) - elif ok: - self.result_pass(num, description) - else: - self.result_fail(num, description) - - def consume_test_plan(self, first, last): - # Only one test plan is supported - if self.test_plan: - self.report_error("Get a second TAP test plan") - return - - try: - first = int(first) - last = int(last) - except ValueError: - self.report_error("Invalid test plan: %s..%s" % (first, last)) - return - - self.test_plan = (first, last) - self.late_plan = self.reported and True or False - - def consume_bail_out(self, line): - self.bail_out = True - self.report("SKIP", 0, line) - - def drain(self): - (ready, unused, self.output) = self.output.rpartition("\n") - for line in ready.split("\n"): - self.log.write(line) - self.log.write("\n") - - if line.startswith("ok "): - self.consume_test_line(True, line[3:]) - elif line.startswith("not ok "): - self.consume_test_line(False, line[7:]) - elif line and line[0].isdigit() and ".." in line: - (first, unused, last) = line.partition("..") - self.consume_test_plan(first, last) - elif line.lower().startswith("bail out!"): - self.consume_bail_out(line) - - def execute(self): - try: - proc = subprocess.Popen(self.argv, close_fds=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except OSError, ex: - self.report_error("Couldn't run %s: %s" % (self.argv[0], str(ex))) - return - - outf = proc.stdout.fileno() - errf = proc.stderr.fileno() - rset = [outf, errf] - while len(rset) > 0: - ret = select.select(rset, [], [], 10) - if outf in ret[0]: - data = os.read(outf, 1024) - if data == "": - if self.output: - self.output += "\n" - rset.remove(outf) - else: - self.output += data - self.drain() - if errf in ret[0]: - data = os.read(errf, 1024) - if data == "": - rset.remove(errf) - self.log.write(data) - sys.stderr.write(data) - - proc.wait() - self.returncode = proc.returncode - - def run(self): - self.execute() - - failed = False - skipped = True - - # Basic collation of results - for (num, code) in self.reported.items(): - if code == "ERROR": - self.errored = True - elif code == "FAIL" or code == "XPASS": - failed = True - if code != "SKIP": - skipped = False - - # Check the plan - if not self.errored: - if not self.test_plan: - if not self.bail_out: - if self.returncode: - self.report_error("Test process failed: %d" % self.returncode) - else: - self.report_error("Didn't receive a TAP test plan") - else: - for i in range(self.test_plan[0], self.test_plan[1] + 1): - if i not in self.reported: - if self.bail_out: - self.report("SKIP", i, "- bailed out") - else: - self.report("ERROR", i, "- missing test") - skipped = False - self.errored = True - - if self.errored: - self.trs.write(":global-test-result: ERROR\n") - self.trs.write(":test-global-result: ERROR\n") - self.trs.write(":recheck: no\n") - elif failed: - self.trs.write(":global-test-result: FAIL\n") - self.trs.write(":test-global-result: FAIL\n") - self.trs.write(":recheck: yes\n") - elif skipped: - self.trs.write(":global-test-result: SKIP\n") - self.trs.write(":test-global-result: SKIP\n") - self.trs.write(":recheck: no\n") - if self.errored or failed: - self.trs.write(":copy-in-global-log: yes\n") - - # Process result code - return self.errored and 1 or 0 - -class YesNoAction(argparse.Action): - def __init__(self, option_strings, dest, **kwargs): - argparse.Action.__init__(self, option_strings, dest, **kwargs) - self.metavar = "[yes|no]" - def __call__(self, parser, namespace, values, option_string=None): - if not values or "yes" in values: - setattr(namespace, self.dest, True) - else: - setattr(namespace, self.dest, False) - -def main(argv): - parser = argparse.ArgumentParser(description='Automake TAP driver') - parser.add_argument('--test-name', metavar='NAME', - help='The name of the test') - parser.add_argument('--log-file', metavar='PATH.log', required=True, - help='The .log file the driver creates') - parser.add_argument('--trs-file', metavar='PATH.trs', required=True, - help='The .trs file the driver creates') - parser.add_argument('--color-tests', default=True, action=YesNoAction, - help='Whether the console output should be colorized or not') - parser.add_argument('--expect-failure', default=False, action=YesNoAction, - help="Whether the tested program is expected to fail") - parser.add_argument('--enable-hard-errors', default=False, action=YesNoAction, - help="Whether hard errors in the tested program are treated differently") - parser.add_argument('command', nargs='+', - help="A test command line to run") - args = parser.parse_args(argv[1:]) - - if not args.test_name: - args.test_name = os.path.basename(args.command[0]) - - driver = Driver(args.command, args) - return driver.run() - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/build/tap-compiler b/build/tap-gtester similarity index 97% rename from build/tap-compiler rename to build/tap-gtester index 76b3171..5179c99 100755 --- a/build/tap-compiler +++ b/build/tap-gtester @@ -137,7 +137,7 @@ class GTestCompiler(NullCompiler): def main(argv): parser = argparse.ArgumentParser(description='Automake TAP compiler') - parser.add_argument('--format', metavar='FORMAT', choices=[ "auto", "GTest", "TAP" ], + parser.add_argument('--format', metavar='FORMAT', choices=[ "auto", "gtest", "tap" ], default="auto", help='The input format to compile') parser.add_argument('--verbose', action='store_true', default=True, help='Verbose mode (ignored)') @@ -149,21 +149,21 @@ def main(argv): cmd = args.command proc = None - if format in ["auto", "GTest"]: + if format in ["auto", "gtest"]: list_cmd = cmd + ["-l", "--verbose"] proc = subprocess.Popen(list_cmd, close_fds=True, stdout=subprocess.PIPE) output = proc.stdout.readline() # Smell whether we're dealing with GTest list output from first line if "random seed" in output or "GTest" in output or output.startswith("/"): - format = "GTest" + format = "gtest" else: - format = "TAP" + format = "tap" else: proc = subprocess.Popen(cmd, close_fds=True, stdout=subprocess.PIPE) - if format == "GTest": + if format == "gtest": compiler = GTestCompiler(cmd) - elif format == "TAP": + elif format == "tap": compiler = NullCompiler(cmd) else: assert False, "not reached" diff --git a/build/test-driver b/build/test-driver index 32bf39e..2d24c8b 100755 --- a/build/test-driver +++ b/build/test-driver @@ -1,127 +1,337 @@ -#! /bin/sh -# test-driver - basic testsuite driver script. +#!/usr/bin/python -scriptversion=2012-06-27.10; # UTC +# Copyright (C) 2013 Red Hat, Inc. +# +# Cockpit is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# Cockpit is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Cockpit; If not, see . -# Copyright (C) 2011-2013 Free Software Foundation, Inc. # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. +# This is a TAP driver for automake # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# In particular it leaves stderr untouched, and is cleaner than the +# one implemented in shell that is making the rounds. # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# As a special exception to the GNU General Public License, if you -# distribute this file as part of a program that contains a -# configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. - -# This file is maintained in Automake, please report -# bugs to or send patches to -# . - -# Make unconditional expansion of undefined variables an error. This -# helps a lot in preventing typo-related bugs. -set -u - -usage_error () -{ - echo "$0: $*" >&2 - print_usage >&2 - exit 2 -} - -print_usage () -{ - cat <$log_file 2>&1 -estatus=$? -if test $enable_hard_errors = no && test $estatus -eq 99; then - estatus=1 -fi - -case $estatus:$expect_failure in - 0:yes) col=$red res=XPASS recheck=yes gcopy=yes;; - 0:*) col=$grn res=PASS recheck=no gcopy=no;; - 77:*) col=$blu res=SKIP recheck=no gcopy=yes;; - 99:*) col=$mgn res=ERROR recheck=yes gcopy=yes;; - *:yes) col=$lgn res=XFAIL recheck=no gcopy=yes;; - *:*) col=$red res=FAIL recheck=yes gcopy=yes;; -esac - -# Report outcome to console. -echo "${col}${res}${std}: $test_name" - -# Register the test result, and other relevant metadata. -echo ":test-result: $res" > $trs_file -echo ":global-test-result: $res" >> $trs_file -echo ":recheck: $recheck" >> $trs_file -echo ":copy-in-global-log: $gcopy" >> $trs_file - -# Local Variables: -# mode: shell-script -# sh-indentation: 2 -# eval: (add-hook 'write-file-hooks 'time-stamp) -# time-stamp-start: "scriptversion=" -# time-stamp-format: "%:y-%02m-%02d.%02H" -# time-stamp-time-zone: "UTC" -# time-stamp-end: "; # UTC" -# End: +# This implements the automake "Custom Test Driver" protocol: +# https://www.gnu.org/software/automake/manual/html_node/Custom-Test-Drivers.html +# +# This consumes the Test Anything Protocol (ie: TAP) +# https://metacpan.org/pod/release/PETDANCE/Test-Harness-2.64/lib/Test/Harness/TAP.pod +# + +import argparse +import os +import select +import subprocess +import sys + +class Driver: + def __init__(self, args): + self.argv = args.command + self.test_name = args.test_name + self.log = open(args.log_file, "w") + self.trs = open(args.trs_file, "w") + self.color_tests = args.color_tests + self.expect_failure = args.expect_failure + + def report(self, code, *args): + CODES = { + "XPASS": '\x1b[0;31m', # red + "FAIL": '\x1b[0;31m', # red + "PASS": '\x1b[0;32m', # grn + "XFAIL": '\x1b[1;32m', # lgn + "SKIP": '\x1b[1;34m', # blu + "ERROR": '\x1b[0;35m', # mgn + } + + # Print out to console + if self.color_tests: + if code in CODES: + sys.stdout.write(CODES[code]) + sys.stdout.write(code) + if self.color_tests: + sys.stdout.write('\x1b[m') + sys.stdout.write(": ") + sys.stdout.write(self.test_name) + sys.stdout.write(" ") + for arg in args: + sys.stdout.write(str(arg)) + sys.stdout.write("\n") + sys.stdout.flush() + + # Book keeping + if code in CODES: + self.trs.write(":test-result: %s\n" % code) + + def result_pass(self, *args): + if self.expect_failure: + self.report("XPASS", *args) + else: + self.report("PASS", *args) + + def result_fail(self, *args): + if self.expect_failure: + self.report("XFAIL", *args) + else: + self.report("FAIL", *args) + + def result_skip(self, *args): + if self.expect_failure: + self.report("XFAIL", *args) + else: + self.report("SKIP", *args) + + def report_error(self, description): + self.report("ERROR", "", description) + + def process(self, output): + pass + + def execute(self): + try: + proc = subprocess.Popen(self.argv, close_fds=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except OSError, ex: + self.report_error("Couldn't run %s: %s" % (self.argv[0], str(ex))) + return + + outf = proc.stdout.fileno() + errf = proc.stderr.fileno() + rset = [outf, errf] + while len(rset) > 0: + ret = select.select(rset, [], [], 10) + if outf in ret[0]: + data = os.read(outf, 1024) + if data == "": + rset.remove(outf) + self.log.write(data) + self.process(data) + if errf in ret[0]: + data = os.read(errf, 1024) + if data == "": + rset.remove(errf) + self.log.write(data) + sys.stderr.write(data) + + proc.wait() + return proc.returncode + +class TapDriver(Driver): + def __init__(self, args): + Driver.__init__(self, args) + self.output = "" + self.reported = { } + self.test_plan = None + self.late_plan = False + self.errored = False + self.bail_out = False + + def report(self, code, num, *args): + if num: + Driver.report(self, code, num, " ", *args) + self.reported[num] = code + else: + Driver.report(self, code, *args) + if code == "ERROR": + self.errored = True + + def consume_test_line(self, ok, data): + # It's an error if the caller sends a test plan in the middle of tests + if self.late_plan: + self.report_error("Got tests after late TAP test plan") + self.late_plan = False + + # Parse out a number and then description + (num, unused, description) = data.partition(" ") + try: + num = int(num) + except ValueError: + self.report_error("Invalid test number: %s" % data) + return + description = description.lstrip() + + # Special case if description starts with this, then skip + if description.lower().startswith("# skip"): + self.result_skip(num, description) + elif ok: + self.result_pass(num, description) + else: + self.result_fail(num, description) + + def consume_test_plan(self, first, last): + # Only one test plan is supported + if self.test_plan: + self.report_error("Get a second TAP test plan") + return + + try: + first = int(first) + last = int(last) + except ValueError: + self.report_error("Invalid test plan: %s..%s" % (first, last)) + return + + self.test_plan = (first, last) + self.late_plan = self.reported and True or False + + def consume_bail_out(self, line): + self.bail_out = True + self.report("SKIP", 0, line) + + def process(self, output): + if output: + self.output += output + elif self.output: + self.output += "\n" + (ready, unused, self.output) = self.output.rpartition("\n") + for line in ready.split("\n"): + self.log.write(line) + self.log.write("\n") + + if line.startswith("ok "): + self.consume_test_line(True, line[3:]) + elif line.startswith("not ok "): + self.consume_test_line(False, line[7:]) + elif line and line[0].isdigit() and ".." in line: + (first, unused, last) = line.partition("..") + self.consume_test_plan(first, last) + elif line.lower().startswith("bail out!"): + self.consume_bail_out(line) + + def run(self): + returncode = self.execute() + + failed = False + skipped = True + + # Basic collation of results + for (num, code) in self.reported.items(): + if code == "ERROR": + self.errored = True + elif code == "FAIL" or code == "XPASS": + failed = True + if code != "SKIP": + skipped = False + + # Check the plan + if not self.errored: + if not self.test_plan: + if not self.bail_out: + if returncode: + self.report_error("Test process failed: %d" % returncode) + else: + self.report_error("Didn't receive a TAP test plan") + else: + for i in range(self.test_plan[0], self.test_plan[1] + 1): + if i not in self.reported: + if self.bail_out: + self.report("SKIP", i, "- bailed out") + else: + self.report("ERROR", i, "- missing test") + skipped = False + self.errored = True + + if self.errored: + self.trs.write(":global-test-result: ERROR\n") + self.trs.write(":test-global-result: ERROR\n") + self.trs.write(":recheck: no\n") + elif failed: + self.trs.write(":global-test-result: FAIL\n") + self.trs.write(":test-global-result: FAIL\n") + self.trs.write(":recheck: yes\n") + elif skipped: + self.trs.write(":global-test-result: SKIP\n") + self.trs.write(":test-global-result: SKIP\n") + self.trs.write(":recheck: no\n") + if self.errored or failed: + self.trs.write(":copy-in-global-log: yes\n") + + # Process result code + return self.errored and 1 or 0 + +class SimpleDriver(Driver): + def __init__(self, args): + Driver.__init__(self, args) + + def run(self): + returncode = self.execute() + if returncode == 0: + self.result_pass() + self.trs.write(":global-test-result: PASS\n") + self.trs.write(":test-global-result: PASS\n") + self.trs.write(":recheck: no\n") + return 0 + elif returncode == 77: + self.result_skip() + self.trs.write(":global-test-result: SKIP\n") + self.trs.write(":test-global-result: SKIP\n") + self.trs.write(":recheck: no\n") + return 0 + elif returncode == 99: + self.result_error() + self.trs.write(":global-test-result: ERROR\n") + self.trs.write(":test-global-result: ERROR\n") + self.trs.write(":copy-in-global-log: yes\n") + self.trs.write(":recheck: no\n") + return 1 + else: + self.result_fail() + self.trs.write(":global-test-result: FAIL\n") + self.trs.write(":test-global-result: FAIL\n") + self.trs.write(":copy-in-global-log: yes\n") + self.trs.write(":recheck: yes\n") + return 0 + + # Process result code + return self.errored and 1 or 0 + +class YesNoAction(argparse.Action): + def __init__(self, option_strings, dest, **kwargs): + argparse.Action.__init__(self, option_strings, dest, **kwargs) + self.metavar = "[yes|no]" + def __call__(self, parser, namespace, values, option_string=None): + if not values or "yes" in values: + setattr(namespace, self.dest, True) + else: + setattr(namespace, self.dest, False) + +def main(argv): + parser = argparse.ArgumentParser(description='Automake TAP driver') + parser.add_argument('--format', metavar='FORMAT', choices=[ "simple", "tap" ], + default="simple", help='The type of test to drive') + parser.add_argument('--test-name', metavar='NAME', + help='The name of the test') + parser.add_argument('--log-file', metavar='PATH.log', required=True, + help='The .log file the driver creates') + parser.add_argument('--trs-file', metavar='PATH.trs', required=True, + help='The .trs file the driver creates') + parser.add_argument('--color-tests', default=True, action=YesNoAction, + help='Whether the console output should be colorized or not') + parser.add_argument('--expect-failure', default=False, action=YesNoAction, + help="Whether the tested program is expected to fail") + parser.add_argument('--enable-hard-errors', default=False, action=YesNoAction, + help="Whether hard errors in the tested program are treated differently") + parser.add_argument('command', nargs='+', + help="A test command line to run") + args = parser.parse_args(argv[1:]) + + if not args.test_name: + args.test_name = os.path.basename(args.command[0]) + if args.format == "simple": + driver = SimpleDriver(args) + elif args.format == "tap": + driver = TapDriver(args) + return driver.run() + +if __name__ == "__main__": + sys.exit(main(sys.argv)) -- 2.7.4