From f089662a9f107079127b53ebb9460449207b08e6 Mon Sep 17 00:00:00 2001 From: Stef Walter Date: Wed, 5 Mar 2014 11:39:42 +0100 Subject: [PATCH] Makefile.am: Add check-memory target for valgrind testing Update the test-driver to accomodate this and figure out how best to implement this. Fix up 'make recheck' as well. --- .gitignore | 1 + Makefile.am | 78 ++++++++++++++++++++++++--------------- build/Makefile.am | 23 ------------ build/{test-driver => tap-driver} | 56 +++++++++++++++++++--------- build/tap-gtester | 8 ++-- 5 files changed, 92 insertions(+), 74 deletions(-) delete mode 100644 build/Makefile.am rename build/{test-driver => tap-driver} (90%) diff --git a/.gitignore b/.gitignore index f57066d..0c8c394 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ stamp* /build/m4/* !/build/m4/vapigen.m4 /build/valgrind-suppressions +/build/test-driver /docs/man/secret-tool.1 /docs/reference/libsecret/version.xml diff --git a/Makefile.am b/Makefile.am index 72d1f9d..ba5b6bc 100644 --- a/Makefile.am +++ b/Makefile.am @@ -30,12 +30,6 @@ dist-hook: echo A git clone is required to generate a ChangeLog >&2; \ fi -check-memory: - @for subdir in $(SUBDIRS); do \ - test -d $(builddir)/$$subdir/tests && \ - make -C $(builddir)/$$subdir/tests check-memory; \ - done - upload-release: $(DIST_ARCHIVES) scp $(DIST_ARCHIVES) master.gnome.org: @@ -56,18 +50,47 @@ dist-hook: dist-check-valac distcleancheck_listfiles = \ find . -name '*.gc[dn][oa]' -prune -o -type f -print -TEST_SUPPRESSIONS = $(top_builddir)/build/valgrind-suppressions +TESTS_ENVIRONMENT = LD_LIBRARY_PATH=$(builddir)/.libs GI_TYPELIB_PATH=$(builddir) +TEST_EXTENSIONS = .py .js + +# Default executable tests +LOG_DRIVER = $(srcdir)/build/tap-driver +LOG_DRIVER_FLAGS = --format=tap +LOG_COMPILER = $(srcdir)/build/tap-gtester + +# Python tests +PY_LOG_DRIVER = $(srcdir)/build/tap-driver +PY_LOG_DRIVER_FLAGS = --format=tap +PY_LOG_COMPILER = $(srcdir)/build/tap-unittest + +# Javascript tests +JS_LOG_DRIVER = $(srcdir)/build/tap-driver +JS_LOG_DRIVER_FLAGS = --format=simple +JS_LOG_COMPILER = gjs -perform-memcheck: $(TEST_PROGS) $(TEST_SUPPRESSIONS) - @make -C $(top_builddir)/build all - @for test in $(TEST_PROGS); do \ - G_SLICE=always-malloc libtool --mode=execute \ - valgrind --trace-children=no --gen-suppressions=all \ - --suppressions=$(TEST_SUPPRESSIONS) \ - --leak-check=full --show-reachable=yes --num-callers=16 \ - --quiet --error-exitcode=33 \ - $(builddir)/$$test; \ - done +VALGRIND_ARGS = --trace-children=no --quiet --error-exitcode=33 \ + --suppressions=valgrind-suppressions --gen-suppressions=all \ + --num-callers=16 +VALGRIND_SUPPRESSIONS = \ + build/gcrypt.supp \ + build/glib.supp \ + build/pthread.supp \ + build/unknown.supp \ + $(NULL) + +valgrind-suppressions: $(VALGRIND_SUPPRESSIONS) + $(AM_V_GEN) cat $^ > $@ + +check-memory: valgrind-suppressions + $(MAKE) LOG_FLAGS="-- libtool --mode=execute valgrind $(VALGRIND_ARGS)" \ + PY_LOG_DRIVER_FLAGS="--missing=no-python-valgrind" \ + JS_LOG_DRIVER_FLAGS="--missing=no-gjs-valgrind" \ + $(AM_MAKEFLAGS) check +recheck-memory: valgrind-suppressions + $(MAKE) LOG_FLAGS="-- libtool --mode=execute valgrind $(VALGRIND_ARGS)" \ + PY_LOG_DRIVER_FLAGS="--missing=no-python-valgrind" \ + JS_LOG_DRIVER_FLAGS="--missing=no-gjs-valgrind" \ + $(AM_MAKEFLAGS) recheck if WITH_COVERAGE coverage: @@ -142,22 +165,17 @@ CLEANFILES = \ $(typelibs_DATA) \ $(BUILT_SOURCES) \ $(vapi_DATA) \ - $(man_MANS) + $(man_MANS) \ + valgrind-suppressions EXTRA_DIST = \ - COPYING.TESTS - -LOG_DRIVER = $(srcdir)/build/test-driver --format=tap -LOG_COMPILER = $(srcdir)/build/tap-gtester - -TESTS_ENVIRONMENT = LD_LIBRARY_PATH=$(builddir)/.libs GI_TYPELIB_PATH=$(builddir) -TEST_EXTENSIONS = .py .js -PY_LOG_DRIVER = $(srcdir)/build/test-driver --format=tap -PY_LOG_COMPILER = $(srcdir)/build/tap-unittest -JS_LOG_DRIVER = $(srcdir)/build/test-driver --format=simple -JS_LOG_COMPILER = gjs + COPYING.TESTS \ + build/valgrind \ + build/tap-gtester \ + build/tap-driver \ + build/tap-unittest \ + $(VALGRIND_SUPPRESSIONS) -include build/Makefile.am include egg/Makefile.am include libsecret/Makefile.am include tool/Makefile.am diff --git a/build/Makefile.am b/build/Makefile.am deleted file mode 100644 index b73bf65..0000000 --- a/build/Makefile.am +++ /dev/null @@ -1,23 +0,0 @@ -SUPPRESSIONS = \ - build/gcrypt.supp \ - build/glib.supp \ - build/pthread.supp \ - build/unknown.supp \ - $(NULL) - -valgrind-suppressions: $(SUPPRESSIONS) - $(AM_V_GEN) cat $^ > $@ - -EXTRA_DIST += \ - build/valgrind \ - build/tap-compiler \ - build/tap-driver \ - build/tap-unittest \ - build/test-driver \ - $(SUPPRESSIONS) - -CLEANFILES += \ - valgrind-suppressions \ - $(NULL) - -nodist_noinst_DATA += valgrind-suppressions diff --git a/build/test-driver b/build/tap-driver similarity index 90% rename from build/test-driver rename to build/tap-driver index 2d24c8b..3906b40 100755 --- a/build/test-driver +++ b/build/tap-driver @@ -39,6 +39,7 @@ class Driver: self.argv = args.command self.test_name = args.test_name self.log = open(args.log_file, "w") + self.log.write("# %s\n" % " ".join(sys.argv)) self.trs = open(args.trs_file, "w") self.color_tests = args.color_tests self.expect_failure = args.expect_failure @@ -90,7 +91,7 @@ class Driver: else: self.report("SKIP", *args) - def report_error(self, description): + def report_error(self, description=""): self.report("ERROR", "", description) def process(self, output): @@ -126,6 +127,7 @@ class Driver: proc.wait() return proc.returncode + class TapDriver(Driver): def __init__(self, args): Driver.__init__(self, args) @@ -195,9 +197,6 @@ class TapDriver(Driver): self.output += "\n" (ready, unused, self.output) = self.output.rpartition("\n") for line in ready.split("\n"): - self.log.write(line) - self.log.write("\n") - if line.startswith("ok "): self.consume_test_line(True, line[3:]) elif line.startswith("not ok "): @@ -223,14 +222,15 @@ class TapDriver(Driver): if code != "SKIP": skipped = False + if not self.errored and returncode: + self.report_error("process failed: %d" % returncode) + self.errored = True + # Check the plan if not self.errored: if not self.test_plan: if not self.bail_out: - if returncode: - self.report_error("Test process failed: %d" % returncode) - else: - self.report_error("Didn't receive a TAP test plan") + self.report_error("Didn't receive a TAP test plan") else: for i in range(self.test_plan[0], self.test_plan[1] + 1): if i not in self.reported: @@ -244,7 +244,7 @@ class TapDriver(Driver): if self.errored: self.trs.write(":global-test-result: ERROR\n") self.trs.write(":test-global-result: ERROR\n") - self.trs.write(":recheck: no\n") + self.trs.write(":recheck: yes\n") elif failed: self.trs.write(":global-test-result: FAIL\n") self.trs.write(":test-global-result: FAIL\n") @@ -253,11 +253,16 @@ class TapDriver(Driver): self.trs.write(":global-test-result: SKIP\n") self.trs.write(":test-global-result: SKIP\n") self.trs.write(":recheck: no\n") + else: + self.trs.write(":global-test-result: PASS\n") + self.trs.write(":test-global-result: PASS\n") + self.trs.write(":recheck: no\n") if self.errored or failed: self.trs.write(":copy-in-global-log: yes\n") # Process result code - return self.errored and 1 or 0 + return 0 + class SimpleDriver(Driver): def __init__(self, args): @@ -270,30 +275,40 @@ class SimpleDriver(Driver): self.trs.write(":global-test-result: PASS\n") self.trs.write(":test-global-result: PASS\n") self.trs.write(":recheck: no\n") - return 0 elif returncode == 77: self.result_skip() self.trs.write(":global-test-result: SKIP\n") self.trs.write(":test-global-result: SKIP\n") self.trs.write(":recheck: no\n") - return 0 elif returncode == 99: - self.result_error() + self.report_error() self.trs.write(":global-test-result: ERROR\n") self.trs.write(":test-global-result: ERROR\n") self.trs.write(":copy-in-global-log: yes\n") - self.trs.write(":recheck: no\n") - return 1 + self.trs.write(":recheck: yes\n") else: self.result_fail() self.trs.write(":global-test-result: FAIL\n") self.trs.write(":test-global-result: FAIL\n") self.trs.write(":copy-in-global-log: yes\n") self.trs.write(":recheck: yes\n") - return 0 # Process result code - return self.errored and 1 or 0 + return 0 + + +class MissingDriver(Driver): + def __init__(self, args): + Driver.__init__(self, args) + self.missing = args.missing + + def run(self): + self.result_skip("skipping due to: ", self.missing) + self.trs.write(":global-test-result: SKIP\n") + self.trs.write(":test-global-result: SKIP\n") + self.trs.write(":recheck: no\n") + return 0 + class YesNoAction(argparse.Action): def __init__(self, option_strings, dest, **kwargs): @@ -305,10 +320,13 @@ class YesNoAction(argparse.Action): else: setattr(namespace, self.dest, False) + def main(argv): parser = argparse.ArgumentParser(description='Automake TAP driver') parser.add_argument('--format', metavar='FORMAT', choices=[ "simple", "tap" ], default="simple", help='The type of test to drive') + parser.add_argument('--missing', metavar="TOOL", nargs='?', + help="Force the test to skip due to missing tool") parser.add_argument('--test-name', metavar='NAME', help='The name of the test') parser.add_argument('--log-file', metavar='PATH.log', required=True, @@ -327,7 +345,9 @@ def main(argv): if not args.test_name: args.test_name = os.path.basename(args.command[0]) - if args.format == "simple": + if args.missing: + driver = MissingDriver(args) + elif args.format == "simple": driver = SimpleDriver(args) elif args.format == "tap": driver = TapDriver(args) diff --git a/build/tap-gtester b/build/tap-gtester index 5179c99..ff7506a 100755 --- a/build/tap-gtester +++ b/build/tap-gtester @@ -80,7 +80,8 @@ class GTestCompiler(NullCompiler): print "not ok %d %s", (self.test_num, self.test_name) self.test_name = None elif cmd == "skipping": - print "ok %d # skip -- %s" % (self.test_num, self.test_name) + if "/subprocess" not in data: + print "ok %d # skip -- %s" % (self.test_num, data) self.test_name = None elif data: print "# %s: %s" % (cmd, data) @@ -97,7 +98,8 @@ class GTestCompiler(NullCompiler): output += proc.stdout.read() proc.wait() if proc.returncode: - raise subprocess.CalledProcessError(proc.returncode, self.command) + sys.stderr.write("tap-gtester: listing GTest tests failed: %d\n" % proc.returncode) + return proc.returncode self.test_remaining = [] for line in output.split("\n"): if line.startswith("/"): @@ -130,7 +132,7 @@ class GTestCompiler(NullCompiler): result = self.process(proc) # The various exit codes and signals we continue for - if result not in [ 0, 1, -4, -5, -6, -7, -8, -11 ]: + if result not in [ 0, 1, -4, -5, -6, -7, -8, -11, 33 ]: break return result -- 2.7.4