From: Sanjay Patel Date: Thu, 24 Mar 2016 23:19:26 +0000 (+0000) Subject: Add utility script to generate checks for opt or llc regression tests X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=fff7a3d0efde3f6f3d33c7b5e83d381101eea044;p=platform%2Fupstream%2Fllvm.git Add utility script to generate checks for opt or llc regression tests This is an enhancement of the existing update_llc_test_checks.py script. It adds some of the functionality from the script used in D17999 to make the IR checking more flexible. The bad news: This actually is 'My First Python Program'. Thus, it's likely that I have violated all best practices of Python programming if I've made a functional change from the original program. If you see anything that's obviously wrong, please let me know or feel free to fix it. I didn't even read any documentation... The good news: I tested this on ~10 existing opt/llc regression tests, and it does what I hoped for. It produces exact checking for IR regression tests and doesn't signficantly change the existing llc-with-x86-target asm checking. The opt tests that were modified in r263667, r263668, r263674, and r263679 are examples of the expected results, except that this version of the script puts the check lines ahead of the IR to follow the existing llc/asm behavior. If there are no complaints/fallout, we should be able to remove the original script. Extending this script to be used for non-x86 and clang regression tests would be the expected follow-up steps. llvm-svn: 264357 --- diff --git a/llvm/utils/update_test_checks.py b/llvm/utils/update_test_checks.py new file mode 100755 index 0000000..bf698c9 --- /dev/null +++ b/llvm/utils/update_test_checks.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python2.7 + +"""A test case update script. + +This script is a utility to update LLVM opt or llc test cases with new +FileCheck patterns. It can either update all of the tests in the file or +a single test function. +""" + +import argparse +import itertools +import os # Used to advertise this file's name ("autogenerated_note"). +import string +import subprocess +import sys +import tempfile +import re + + +# RegEx: this is where the magic happens. + +SCRUB_WHITESPACE_RE = re.compile(r'(?!^(| \w))[ \t]+', flags=re.M) +SCRUB_TRAILING_WHITESPACE_RE = re.compile(r'[ \t]+$', flags=re.M) +SCRUB_X86_SHUFFLES_RE = ( + re.compile( + r'^(\s*\w+) [^#\n]+#+ ((?:[xyz]mm\d+|mem) = .*)$', + flags=re.M)) +SCRUB_X86_SP_RE = re.compile(r'\d+$%(esp|rsp)$') +SCRUB_X86_RIP_RE = re.compile(r'[.\w]+$%rip$') +SCRUB_KILL_COMMENT_RE = re.compile(r'^ *#+ +kill:.*\n') +SCRUB_IR_COMMENT_RE = re.compile(r'\s*;.*') + +RUN_LINE_RE = re.compile('^\s*;\s*RUN:\s*(.*)$') +IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@(\w+)\s*\(') +LLC_FUNCTION_RE = re.compile( + r'^_?(?P[^:]+):[ \t]*#+[ \t]*@(?P=func)\n[^:]*?' + r'(?P^##?[ \t]+[^:]+:.*?)\s*' + r'^\s*(?:[^:\n]+?:\s*\n\s*\.size|\.cfi_endproc|\.globl|\.comm|\.(?:sub)?section)', + flags=(re.M | re.S)) +OPT_FUNCTION_RE = re.compile( + r'^\s*define\s+(?:internal\s+)?[^@]*@(?P\w+?)\s*\(' + r'(\s+)?[^{]*\{\n(?P.*?)\}', + flags=(re.M | re.S)) +CHECK_PREFIX_RE = re.compile('--check-prefix=(\S+)') +CHECK_RE = re.compile(r'^\s*;\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:') +IR_VALUE_DEF_RE = re.compile(r'\s+%(.*) =') + + +# Invoke the tool that is being tested. +def invoke_tool(args, cmd_args, ir): + with open(ir) as ir_file: + stdout = subprocess.check_output(args.tool_binary + ' ' + cmd_args, + shell=True, stdin=ir_file) + # Fix line endings to unix CR style. + stdout = stdout.replace('\r\n', '\n') + return stdout + + +# FIXME: Separate the x86-specific scrubbers, so this can be used for other targets. +def scrub_asm(asm): + # Detect shuffle asm comments and hide the operands in favor of the comments. + asm = SCRUB_X86_SHUFFLES_RE.sub(r'\1 {{.*#+}} \2', asm) + # Generically match the stack offset of a memory operand. + asm = SCRUB_X86_SP_RE.sub(r'{{[0-9]+}}(%\1)', asm) + # Generically match a RIP-relative memory operand. + asm = SCRUB_X86_RIP_RE.sub(r'{{.*}}(%rip)', asm) + # Strip kill operands inserted into the asm. + asm = SCRUB_KILL_COMMENT_RE.sub('', asm) + return asm + + +def scrub_body(body, tool_basename): + # Scrub runs of whitespace out of the assembly, but leave the leading + # whitespace in place. + body = SCRUB_WHITESPACE_RE.sub(r' ', body) + # Expand the tabs used for indentation. + body = string.expandtabs(body, 2) + # Strip trailing whitespace. + body = SCRUB_TRAILING_WHITESPACE_RE.sub(r'', body) + if tool_basename == "llc": + body = scrub_asm(body) + return body + + +# Build up a dictionary of all the function bodies. +def build_function_body_dictionary(raw_tool_output, prefixes, func_dict, verbose, tool_basename): + if tool_basename == "llc": + func_regex = LLC_FUNCTION_RE + else: + func_regex = OPT_FUNCTION_RE + for m in func_regex.finditer(raw_tool_output): + if not m: + continue + func = m.group('func') + scrubbed_body = scrub_body(m.group('body'), tool_basename) + if func.startswith('stress'): + # We only use the last line of the function body for stress tests. + scrubbed_body = '\n'.join(scrubbed_body.splitlines()[-1:]) + if verbose: + print >>sys.stderr, 'Processing function: ' + func + for l in scrubbed_body.splitlines(): + print >>sys.stderr, ' ' + l + for prefix in prefixes: + if func in func_dict[prefix] and func_dict[prefix][func] != scrubbed_body: + if prefix == prefixes[-1]: + print >>sys.stderr, ('WARNING: Found conflicting asm under the ' + 'same prefix: %r!' % (prefix,)) + else: + func_dict[prefix][func] = None + continue + + func_dict[prefix][func] = scrubbed_body + + +# Create a FileCheck variable name based on an IR name. +def get_value_name(var): + if var.isdigit(): + var = 'TMP' + var + var = var.replace('.', '_') + return var.upper() + + +# Create a FileCheck variable from regex. +def get_value_definition(var): + return '[[' + get_value_name(var) + ':%.*]]' + + +# Use a FileCheck variable. +def get_value_use(var): + return '[[' + get_value_name(var) + ']]' + + +# Replace IR value defs and uses with FileCheck variables. +def genericize_check_lines(lines): + lines_with_def = [] + vars_seen = [] + for line in lines: + m = IR_VALUE_DEF_RE.match(line) + if m: + vars_seen.append(m.group(1)) + line = line.replace('%' + m.group(1), get_value_definition(m.group(1))) + + lines_with_def.append(line) + + # A single def isn't worth replacing? + #if len(vars_seen) < 2: + # return lines + + output_lines = [] + vars_seen.sort(key=len, reverse=True) + for line in lines_with_def: + for var in vars_seen: + line = line.replace('%' + var, get_value_use(var)) + output_lines.append(line) + + return output_lines + + +def add_checks(output_lines, prefix_list, func_dict, func_name, tool_basename): + # Select a label format based on the whether we're checking asm or IR. + if tool_basename == "llc": + check_label_format = "; %s-LABEL: %s:" + else: + check_label_format = "; %s-LABEL: @%s(" + + printed_prefixes = [] + for checkprefixes, _ in prefix_list: + for checkprefix in checkprefixes: + if checkprefix in printed_prefixes: + break + if not func_dict[checkprefix][func_name]: + continue + # Add some space between different check prefixes, but not after the last + # check line (before the test code). + #if len(printed_prefixes) != 0: + # output_lines.append(';') + printed_prefixes.append(checkprefix) + output_lines.append(check_label_format % (checkprefix, func_name)) + func_body = func_dict[checkprefix][func_name].splitlines() + + # For IR output, change all defs to FileCheck variables, so we're immune + # to variable naming fashions. + if tool_basename == "opt": + func_body = genericize_check_lines(func_body) + + # Handle the first line of the function body as a special case because + # it's often just noise (a useless asm comment or entry label). + if func_body[0].startswith("#") or func_body[0].startswith("entry:"): + is_blank_line = True + else: + output_lines.append('; %s: %s' % (checkprefix, func_body[0])) + is_blank_line = False + + for func_line in func_body[1:]: + if func_line.strip() == '': + is_blank_line = True + continue + # Do not waste time checking IR comments. + if tool_basename == "opt": + func_line = SCRUB_IR_COMMENT_RE.sub(r'', func_line) + + # Skip blank lines instead of checking them. + if is_blank_line == True: + output_lines.append('; %s: %s' % (checkprefix, func_line)) + else: + output_lines.append('; %s-NEXT: %s' % (checkprefix, func_line)) + is_blank_line = False + + # Add space between different check prefixes and also before the first + # line of code in the test function. + output_lines.append(';') + break + return output_lines + + +def should_add_line_to_output(input_line, prefix_set): + # Skip any blank comment lines in the IR. + if input_line.strip() == ';': + return False + # Skip any blank lines in the IR. + #if input_line.strip() == '': + # return False + # And skip any CHECK lines. We're building our own. + m = CHECK_RE.match(input_line) + if m and m.group(1) in prefix_set: + return False + + return True + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('-v', '--verbose', action='store_true', + help='Show verbose output') + parser.add_argument('--tool-binary', default='llc', + help='The tool used to generate the test case') + parser.add_argument( + '--function', help='The function in the test file to update') + parser.add_argument('tests', nargs='+') + args = parser.parse_args() + + autogenerated_note = ('; NOTE: Assertions have been autogenerated by ' + + os.path.basename(__file__)) + + tool_basename = os.path.basename(args.tool_binary) + if (tool_basename != "llc" and tool_basename != "opt"): + print >>sys.stderr, 'ERROR: Unexpected tool name: ' + tool_basename + sys.exit(1) + + for test in args.tests: + if args.verbose: + print >>sys.stderr, 'Scanning for RUN lines in test file: %s' % (test,) + with open(test) as f: + input_lines = [l.rstrip() for l in f] + + run_lines = [m.group(1) + for m in [RUN_LINE_RE.match(l) for l in input_lines] if m] + if args.verbose: + print >>sys.stderr, 'Found %d RUN lines:' % (len(run_lines),) + for l in run_lines: + print >>sys.stderr, ' RUN: ' + l + + prefix_list = [] + for l in run_lines: + (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)]) + + if not tool_cmd.startswith(tool_basename + ' '): + print >>sys.stderr, 'WARNING: Skipping non-%s RUN line: %s' % (tool_basename, l) + continue + + if not filecheck_cmd.startswith('FileCheck '): + print >>sys.stderr, 'WARNING: Skipping non-FileChecked RUN line: ' + l + continue + + tool_cmd_args = tool_cmd[len(tool_basename):].strip() + tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip() + + check_prefixes = [m.group(1) + for m in CHECK_PREFIX_RE.finditer(filecheck_cmd)] + if not check_prefixes: + check_prefixes = ['CHECK'] + + # FIXME: We should use multiple check prefixes to common check lines. For + # now, we just ignore all but the last. + prefix_list.append((check_prefixes, tool_cmd_args)) + + func_dict = {} + for prefixes, _ in prefix_list: + for prefix in prefixes: + func_dict.update({prefix: dict()}) + for prefixes, tool_args in prefix_list: + if args.verbose: + print >>sys.stderr, 'Extracted tool cmd: ' + tool_basename + ' ' + tool_args + print >>sys.stderr, 'Extracted FileCheck prefixes: ' + str(prefixes) + + raw_tool_output = invoke_tool(args, tool_args, test) + build_function_body_dictionary(raw_tool_output, prefixes, func_dict, args.verbose, tool_basename) + + is_in_function = False + is_in_function_start = False + prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes]) + if args.verbose: + print >>sys.stderr, 'Rewriting FileCheck prefixes: %s' % (prefix_set,) + output_lines = [] + output_lines.append(autogenerated_note) + + for input_line in input_lines: + if is_in_function_start: + if input_line == '': + continue + if input_line.lstrip().startswith(';'): + m = CHECK_RE.match(input_line) + if not m or m.group(1) not in prefix_set: + output_lines.append(input_line) + continue + + # Print out the various check lines here. + output_lines = add_checks(output_lines, prefix_list, func_dict, name, tool_basename) + is_in_function_start = False + + if is_in_function: + if should_add_line_to_output(input_line, prefix_set) == True: + # This input line of the function body will go as-is into the output. + output_lines.append(input_line) + else: + continue + if input_line.strip() == '}': + is_in_function = False + continue + + if input_line == autogenerated_note: + continue + + # If it's outside a function, it just gets copied to the output. + output_lines.append(input_line) + + m = IR_FUNCTION_RE.match(input_line) + if not m: + continue + name = m.group(1) + if args.function is not None and name != args.function: + # When filtering on a specific function, skip all others. + continue + is_in_function = is_in_function_start = True + + if args.verbose: + print>>sys.stderr, 'Writing %d lines to %s...' % (len(output_lines), test) + + with open(test, 'wb') as f: + f.writelines([l + '\n' for l in output_lines]) + + +if __name__ == '__main__': + main() +