test: xkeyboard-config: add a multiprocessing.Pool() to speed up the test

Collect all options into a dictionary, then process that as async actions
through a process pool. This of course requires collecting the various print
statements to avoid mangled output.

This dropped the time to completion from around 14 min to 8 min on my local
machine (unscientific single run only for the original timing).

Signed-off-by: Peter Hutterer <peter.hutterer@who-t.net>

diff --git a/test/xkeyboard-config-test.py.in b/test/xkeyboard-config-test.py.in
index 9516b2c..bdbb4e2 100755 (executable)
--- a/test/xkeyboard-config-test.py.in
+++ b/test/xkeyboard-config-test.py.in
@@ -3,7 +3,9 @@ import argparse
 import sys
 import subprocess
 import os
+import io
 import xml.etree.ElementTree as ET
+from multiprocessing import Pool
 
 
 verbose = True
@@ -27,7 +29,12 @@ if os.isatty(sys.stdout.fileno()):
         pass
 
 
-def xkbcommontool(r='evdev', m='pc105', l='us', v=None, o=None):
+def xkbcommontool(rmlvo):
+    r = rmlvo.get('r', 'evdev')
+    m = rmlvo.get('m', 'pc105')
+    l = rmlvo.get('l', 'us')
+    v = rmlvo.get('v', None)
+    o = rmlvo.get('o', None)
     args = [
         'rmlvo-to-keymap',
         '--rules', r,
@@ -39,20 +46,29 @@ def xkbcommontool(r='evdev', m='pc105', l='us', v=None, o=None):
     if o is not None:
         args += ['--options', o]
 
+    success = True
+    out = io.StringIO()
     if verbose:
-        print(':: {}'.format(' '.join(args)))
+        print(':: {}'.format(' '.join(args)), file=out)
 
     try:
         output = subprocess.check_output(args, stderr=subprocess.STDOUT)
         if verbose:
-            print(output.decode('utf-8'))
+            print(output.decode('utf-8'), file=out)
     except subprocess.CalledProcessError as err:
-        print('ERROR: Failed to compile: {}'.format(' '.join(args)))
-        print(err.output.decode('utf-8'))
-        sys.exit(1)
+        print('ERROR: Failed to compile: {}'.format(' '.join(args)), file=out)
+        print(err.output.decode('utf-8'), file=out)
+        success = False
 
+    return success, out.getvalue()
 
-def xkbcomp(r='evdev', m='pc105', l='us', v='', o=''):
+
+def xkbcomp(rmlvo):
+    r = rmlvo.get('r', 'evdev')
+    m = rmlvo.get('m', 'pc105')
+    l = rmlvo.get('l', 'us')
+    v = rmlvo.get('v', None)
+    o = rmlvo.get('o', None)
     args = ['setxkbmap', '-print']
     if r is not None:
         args.append('-rules')
@@ -67,30 +83,37 @@ def xkbcomp(r='evdev', m='pc105', l='us', v='', o=''):
         args.append('-option')
         args.append('{}'.format(o))
 
+    success = True
+    out = io.StringIO()
     if verbose:
-        print(':: {}'.format(' '.join(args)))
+        print(':: {}'.format(' '.join(args)), file=out)
 
     try:
         xkbcomp_args = ['xkbcomp', '-xkb', '-', '-']
 
         setxkbmap = subprocess.Popen(args, stdout=subprocess.PIPE)
         xkbcomp = subprocess.Popen(xkbcomp_args, stdin=setxkbmap.stdout,
-                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         setxkbmap.stdout.close()
         stdout, stderr = xkbcomp.communicate()
         if xkbcomp.returncode != 0:
-            print('ERROR: Failed to compile: {}'.format(' '.join(args)))
+            print('ERROR: Failed to compile: {}'.format(' '.join(args)), file=out)
+            success = False
         if xkbcomp.returncode != 0 or verbose:
-            print(stdout.decode('utf-8'))
-            print(stderr.decode('utf-8'))
+            print(stdout.decode('utf-8'), file=out)
+            print(stderr.decode('utf-8'), file=out)
 
     # This catches setxkbmap errors.
     except subprocess.CalledProcessError as err:
-        print('ERROR: Failed to compile: {}'.format(' '.join(args)))
-        print(err.output.decode('utf-8'))
+        print('ERROR: Failed to compile: {}'.format(' '.join(args)), file=out)
+        print(err.output.decode('utf-8'), file=out)
+        success = False
+
+    return success, out.getvalue()
 
 
-def parse(root, tool):
+def parse(path):
+    root = ET.fromstring(open(path).read())
     layouts = root.findall('layoutList/layout')
 
     options = [
@@ -98,17 +121,34 @@ def parse(root, tool):
         for e in root.findall('optionList/group/option/configItem/name')
     ]
 
-    for l in progress_bar(layouts, 'layout '):
+    combos = []
+    for l in layouts:
         layout = l.find('configItem/name').text
-        tool(l=layout)
+        combos.append({'l': layout})
 
         variants = l.findall('variantList/variant')
-        for v in progress_bar(variants, 'variant'):
+        for v in variants:
             variant = v.find('configItem/name').text
-            tool(l=layout, v=variant)
 
-            for option in progress_bar(options, 'option '):
-                tool(l=layout, v=variant, o=option)
+            combos.append({'l': layout, 'v': variant})
+            for option in options:
+                combos.append({'l': layout, 'v': variant, 'o': option})
+
+    return combos
+
+
+def run(combos, tool, njobs):
+    failed = False
+    with Pool(njobs) as p:
+        results = p.imap_unordered(tool, combos)
+        for r in progress_bar(results, 'testing'):
+            success, output = r
+            if not success:
+                failed = True
+            if output:
+                print(output)
+
+    return failed
 
 
 def main(args):
@@ -127,13 +167,16 @@ def main(args):
     parser.add_argument('--tool', choices=tools.keys(),
                         type=str, default='libxkbcommon',
                         help='parsing tool to use')
+    parser.add_argument('--jobs', '-j', type=int,
+                        default=os.cpu_count() * 4,
+                        help='number of processes to use')
     args = parser.parse_args()
 
     tool = tools[args.tool]
 
-    with open(args.path) as f:
-        root = ET.fromstring(f.read())
-        parse(root, tool)
+    combos = parse(args.path)
+    failed = run(combos, tool, args.jobs)
+    sys.exit(failed)
 
 
 if __name__ == '__main__':