[valgrind] Use libtool and support run-subset-fuzzer-tests (#1668)

diff --git a/test/fuzzing/Makefile.am b/test/fuzzing/Makefile.am
index a77df70..5bd2d7e 100644 (file)
--- a/test/fuzzing/Makefile.am
+++ b/test/fuzzing/Makefile.am
@@ -55,8 +55,8 @@ hb_subset_fuzzer_CPPFLAGS = $(AM_CPPFLAGS)
 hb_subset_fuzzer_DEPENDENCIES = $(top_builddir)/src/libharfbuzz-subset.la
 
 check:
-       EXEEXT="$(EXEEXT)" srcdir="$(srcdir)" builddir="$(builddir)" $(srcdir)/run-shape-fuzzer-tests.py
-       EXEEXT="$(EXEEXT)" srcdir="$(srcdir)" builddir="$(builddir)" $(srcdir)/run-subset-fuzzer-tests.py
+       EXEEXT="$(EXEEXT)" srcdir="$(srcdir)" builddir="$(builddir)" LIBTOOL="$(LIBTOOL)" $(srcdir)/run-shape-fuzzer-tests.py
+       EXEEXT="$(EXEEXT)" srcdir="$(srcdir)" builddir="$(builddir)" LIBTOOL="$(LIBTOOL)" $(srcdir)/run-subset-fuzzer-tests.py
 check-valgrind:
        $(AM_V_at)RUN_VALGRIND=1 $(MAKE) $(AM_MAKEFLGS) check
 

diff --git a/test/fuzzing/run-shape-fuzzer-tests.py b/test/fuzzing/run-shape-fuzzer-tests.py
index 90ed509..ba480dd 100755 (executable)
--- a/test/fuzzing/run-shape-fuzzer-tests.py
+++ b/test/fuzzing/run-shape-fuzzer-tests.py
@@ -67,36 +67,36 @@ please provide it as the first argument to the tool""")
 print ('hb_shape_fuzzer:', hb_shape_fuzzer)
 fails = 0
 
+libtool = os.environ.get('LIBTOOL')
 valgrind = None
 if os.environ.get('RUN_VALGRIND', ''):
        valgrind = which ('valgrind')
        if valgrind is None:
                print ("""Valgrind requested but not found.""")
                sys.exit (1)
+       if libtool is None:
+               print ("""Valgrind support is currently autotools only and needs libtool but not found.""")
+
 
 parent_path = os.path.join (srcdir, "fonts")
 for file in os.listdir (parent_path):
        path = os.path.join(parent_path, file)
 
-       text, returncode = cmd ([hb_shape_fuzzer, path])
-       if text.strip ():
+       if valgrind:
+               text, returncode = cmd (libtool.split(' ') + ['--mode=execute', valgrind + ' --leak-check=full --error-exitcode=1', '--', hb_shape_fuzzer, path])
+       else:
+               text, returncode = cmd ([hb_shape_fuzzer, path])
+               if 'error' in text:
+                       returncode = 1
+
+       if not valgrind and text.strip ():
                print (text)
 
-       failed = False
-       if returncode != 0 or 'error' in text:
+       if returncode != 0:
                print ('failure on %s' % file)
-               failed = True
-
-       if valgrind:
-               text, returncode = cmd ([valgrind, '--error-exitcode=1', '--leak-check=full', hb_shape_fuzzer, path])
-               if returncode:
-                       print (text)
-                       print ('failure on %s' % file)
-                       failed = True
-
-       if failed:
                fails = fails + 1
 
+
 if fails:
        print ("%i shape fuzzer related tests failed." % fails)
        sys.exit (1)

diff --git a/test/fuzzing/run-subset-fuzzer-tests.py b/test/fuzzing/run-subset-fuzzer-tests.py
index 7392a92..3ac2288 100755 (executable)
--- a/test/fuzzing/run-subset-fuzzer-tests.py
+++ b/test/fuzzing/run-subset-fuzzer-tests.py
@@ -2,7 +2,54 @@
 
 from __future__ import print_function, division, absolute_import
 
-import sys, os, subprocess
+import sys, os, subprocess, tempfile, threading
+
+
+def which(program):
+       # https://stackoverflow.com/a/377028
+       def is_exe(fpath):
+               return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+       fpath, _ = os.path.split(program)
+       if fpath:
+               if is_exe(program):
+                       return program
+       else:
+               for path in os.environ["PATH"].split(os.pathsep):
+                       exe_file = os.path.join(path, program)
+                       if is_exe(exe_file):
+                               return exe_file
+
+       return None
+
+
+def cmd(command):
+       # https://stackoverflow.com/a/4408409
+       # https://stackoverflow.com/a/10012262
+       with tempfile.TemporaryFile() as tempf:
+               p = subprocess.Popen (command, stderr=tempf)
+               is_killed = {'value': False}
+
+               def timeout(p, is_killed):
+                       is_killed['value'] = True
+                       p.kill()
+               timer = threading.Timer (2, timeout, [p, is_killed])
+
+               try:
+                       timer.start()
+                       p.wait ()
+                       tempf.seek (0)
+                       text = tempf.read().decode ("utf-8").strip ()
+                       returncode = p.returncode
+               finally:
+                       timer.cancel()
+
+               if is_killed['value']:
+                       text = 'error: timeout, ' + text
+                       returncode = 1
+
+               return text, returncode
+
 
 srcdir = os.environ.get ("srcdir", ".")
 EXEEXT = os.environ.get ("EXEEXT", "")
@@ -20,22 +67,38 @@ please provide it as the first argument to the tool""")
 print ('hb_subset_fuzzer:', hb_subset_fuzzer)
 fails = 0
 
+libtool = os.environ.get('LIBTOOL')
+valgrind = None
+if os.environ.get('RUN_VALGRIND', ''):
+       valgrind = which ('valgrind')
+       if valgrind is None:
+               print ("""Valgrind requested but not found.""")
+               sys.exit (1)
+       if libtool is None:
+               print ("""Valgrind support is currently autotools only and needs libtool but not found.""")
+
+
 def run_dir (parent_path):
        global fails
        for file in os.listdir (parent_path):
                path = os.path.join(parent_path, file)
 
                print ("running subset fuzzer against %s" % path)
-               p = subprocess.Popen ([hb_subset_fuzzer, path])
+               if valgrind:
+                       text, returncode = cmd (libtool.split(' ') + ['--mode=execute', valgrind + ' --leak-check=full --show-leak-kinds=all --error-exitcode=1', '--', hb_subset_fuzzer, path])
+               else:
+                       text, returncode = cmd ([hb_subset_fuzzer, path])
+                       if 'error' in text:
+                               returncode = 1
 
-               if p.wait () != 0:
-                       print ("failed for %s" % path)
-                       fails = fails + 1
+               if not valgrind and text.strip ():
+                       print (text)
 
-               if p.wait () != 0:
+               if returncode != 0:
                        print ("failed for %s" % path)
                        fails = fails + 1
 
+
 run_dir (os.path.join (srcdir, "..", "subset", "data", "fonts"))
 # TODO running these tests very slow tests.  Fix and re-enable
 #run_dir (os.path.join (srcdir, "fonts"))