From f53838477d10b68581af18f71ac08c2eded0698a Mon Sep 17 00:00:00 2001 From: Peter Hutterer Date: Tue, 29 Oct 2019 16:06:10 +1000 Subject: [PATCH] test: xkeyboard-config: add a multiprocessing.Pool() to speed up the test Collect all options into a dictionary, then process that as async actions through a process pool. This of course requires collecting the various print statements to avoid mangled output. This dropped the time to completion from around 14 min to 8 min on my local machine (unscientific single run only for the original timing). Signed-off-by: Peter Hutterer --- test/xkeyboard-config-test.py.in | 91 +++++++++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 24 deletions(-) diff --git a/test/xkeyboard-config-test.py.in b/test/xkeyboard-config-test.py.in index 9516b2c..bdbb4e2 100755 --- a/test/xkeyboard-config-test.py.in +++ b/test/xkeyboard-config-test.py.in @@ -3,7 +3,9 @@ import argparse import sys import subprocess import os +import io import xml.etree.ElementTree as ET +from multiprocessing import Pool verbose = True @@ -27,7 +29,12 @@ if os.isatty(sys.stdout.fileno()): pass -def xkbcommontool(r='evdev', m='pc105', l='us', v=None, o=None): +def xkbcommontool(rmlvo): + r = rmlvo.get('r', 'evdev') + m = rmlvo.get('m', 'pc105') + l = rmlvo.get('l', 'us') + v = rmlvo.get('v', None) + o = rmlvo.get('o', None) args = [ 'rmlvo-to-keymap', '--rules', r, @@ -39,20 +46,29 @@ def xkbcommontool(r='evdev', m='pc105', l='us', v=None, o=None): if o is not None: args += ['--options', o] + success = True + out = io.StringIO() if verbose: - print(':: {}'.format(' '.join(args))) + print(':: {}'.format(' '.join(args)), file=out) try: output = subprocess.check_output(args, stderr=subprocess.STDOUT) if verbose: - print(output.decode('utf-8')) + print(output.decode('utf-8'), file=out) except subprocess.CalledProcessError as err: - print('ERROR: Failed to compile: {}'.format(' '.join(args))) - print(err.output.decode('utf-8')) - sys.exit(1) + print('ERROR: Failed to compile: {}'.format(' '.join(args)), file=out) + print(err.output.decode('utf-8'), file=out) + success = False + return success, out.getvalue() -def xkbcomp(r='evdev', m='pc105', l='us', v='', o=''): + +def xkbcomp(rmlvo): + r = rmlvo.get('r', 'evdev') + m = rmlvo.get('m', 'pc105') + l = rmlvo.get('l', 'us') + v = rmlvo.get('v', None) + o = rmlvo.get('o', None) args = ['setxkbmap', '-print'] if r is not None: args.append('-rules') @@ -67,30 +83,37 @@ def xkbcomp(r='evdev', m='pc105', l='us', v='', o=''): args.append('-option') args.append('{}'.format(o)) + success = True + out = io.StringIO() if verbose: - print(':: {}'.format(' '.join(args))) + print(':: {}'.format(' '.join(args)), file=out) try: xkbcomp_args = ['xkbcomp', '-xkb', '-', '-'] setxkbmap = subprocess.Popen(args, stdout=subprocess.PIPE) xkbcomp = subprocess.Popen(xkbcomp_args, stdin=setxkbmap.stdout, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, stderr=subprocess.PIPE) setxkbmap.stdout.close() stdout, stderr = xkbcomp.communicate() if xkbcomp.returncode != 0: - print('ERROR: Failed to compile: {}'.format(' '.join(args))) + print('ERROR: Failed to compile: {}'.format(' '.join(args)), file=out) + success = False if xkbcomp.returncode != 0 or verbose: - print(stdout.decode('utf-8')) - print(stderr.decode('utf-8')) + print(stdout.decode('utf-8'), file=out) + print(stderr.decode('utf-8'), file=out) # This catches setxkbmap errors. except subprocess.CalledProcessError as err: - print('ERROR: Failed to compile: {}'.format(' '.join(args))) - print(err.output.decode('utf-8')) + print('ERROR: Failed to compile: {}'.format(' '.join(args)), file=out) + print(err.output.decode('utf-8'), file=out) + success = False + + return success, out.getvalue() -def parse(root, tool): +def parse(path): + root = ET.fromstring(open(path).read()) layouts = root.findall('layoutList/layout') options = [ @@ -98,17 +121,34 @@ def parse(root, tool): for e in root.findall('optionList/group/option/configItem/name') ] - for l in progress_bar(layouts, 'layout '): + combos = [] + for l in layouts: layout = l.find('configItem/name').text - tool(l=layout) + combos.append({'l': layout}) variants = l.findall('variantList/variant') - for v in progress_bar(variants, 'variant'): + for v in variants: variant = v.find('configItem/name').text - tool(l=layout, v=variant) - for option in progress_bar(options, 'option '): - tool(l=layout, v=variant, o=option) + combos.append({'l': layout, 'v': variant}) + for option in options: + combos.append({'l': layout, 'v': variant, 'o': option}) + + return combos + + +def run(combos, tool, njobs): + failed = False + with Pool(njobs) as p: + results = p.imap_unordered(tool, combos) + for r in progress_bar(results, 'testing'): + success, output = r + if not success: + failed = True + if output: + print(output) + + return failed def main(args): @@ -127,13 +167,16 @@ def main(args): parser.add_argument('--tool', choices=tools.keys(), type=str, default='libxkbcommon', help='parsing tool to use') + parser.add_argument('--jobs', '-j', type=int, + default=os.cpu_count() * 4, + help='number of processes to use') args = parser.parse_args() tool = tools[args.tool] - with open(args.path) as f: - root = ET.fromstring(f.read()) - parse(root, tool) + combos = parse(args.path) + failed = run(combos, tool, args.jobs) + sys.exit(failed) if __name__ == '__main__': -- 2.7.4