From: Vedant Kumar Date: Tue, 18 Sep 2018 19:31:47 +0000 (+0000) Subject: Allow use of self.filecheck in LLDB tests (c.f self.expect) X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=9b13bea61a576a41eda7c88d1628f9914bfc02b4;p=platform%2Fupstream%2Fllvm.git Allow use of self.filecheck in LLDB tests (c.f self.expect) Add a "filecheck" method to the LLDB test base. This allows test authors to pattern match command output using FileCheck, making it possible to write stricter tests than what `self.expect` allows. For context (motivation, examples of stricter checking, etc), see the lldb-dev thread: "Using FileCheck in lldb inline tests". Differential Revision: https://reviews.llvm.org/D50751 llvm-svn: 342508 --- diff --git a/lldb/CMakeLists.txt b/lldb/CMakeLists.txt index 65cc86b..e3163ad 100644 --- a/lldb/CMakeLists.txt +++ b/lldb/CMakeLists.txt @@ -93,11 +93,15 @@ option(LLDB_TEST_USE_CUSTOM_C_COMPILER "Use the C compiler provided via LLDB_TES option(LLDB_TEST_USE_CUSTOM_CXX_COMPILER "Use the C++ compiler provided via LLDB_TEST_CXX_COMPILER for building test inferiors (instead of the just-built compiler). Defaults to OFF." OFF) if(LLDB_INCLUDE_TESTS) - # The difference between the following two paths is significant. The path to - # LLDB will point to LLDB's binary directory, while the other will point to - # LLVM's binary directory in case the two differ. + # Set the path to the default lldb test executable. Make the path relative to + # LLVM_RUNTIME_OUTPUT_INTDIR: this will be correct even when LLVM and LLDB + # have separate binary directories. set(LLDB_DEFAULT_TEST_EXECUTABLE "${LLVM_RUNTIME_OUTPUT_INTDIR}/lldb${CMAKE_EXECUTABLE_SUFFIX}") + + # Set the paths to default llvm tools. Make these paths relative to the LLVM + # binary directory. set(LLDB_DEFAULT_TEST_DSYMUTIL "${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin/dsymutil${CMAKE_EXECUTABLE_SUFFIX}") + set(LLDB_DEFAULT_TEST_FILECHECK "${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin/FileCheck${CMAKE_EXECUTABLE_SUFFIX}") if (NOT LLDB_TEST_USE_CUSTOM_C_COMPILER AND TARGET clang) set(LLDB_DEFAULT_TEST_C_COMPILER "${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin/clang${CMAKE_EXECUTABLE_SUFFIX}") @@ -115,6 +119,7 @@ if(LLDB_INCLUDE_TESTS) set(LLDB_TEST_C_COMPILER "${LLDB_DEFAULT_TEST_C_COMPILER}" CACHE PATH "C Compiler to use for building LLDB test inferiors") set(LLDB_TEST_CXX_COMPILER "${LLDB_DEFAULT_TEST_CXX_COMPILER}" CACHE PATH "C++ Compiler to use for building LLDB test inferiors") set(LLDB_TEST_DSYMUTIL "${LLDB_DEFAULT_TEST_DSYMUTIL}" CACHE PATH "dsymutil used for generating dSYM bundles") + set(LLDB_TEST_FILECHECK "${LLDB_DEFAULT_TEST_FILECHECK}" CACHE PATH "FileCheck used for testing purposes") if (("${LLDB_TEST_C_COMPILER}" STREQUAL "") OR ("${LLDB_TEST_CXX_COMPILER}" STREQUAL "")) diff --git a/lldb/lldb.xcodeproj/project.pbxproj b/lldb/lldb.xcodeproj/project.pbxproj index 2e65bc4..3cbd4d9 100644 --- a/lldb/lldb.xcodeproj/project.pbxproj +++ b/lldb/lldb.xcodeproj/project.pbxproj @@ -7091,7 +7091,7 @@ /* Begin PBXLegacyTarget section */ 2387551E1C24974600CCE8C3 /* lldb-python-test-suite */ = { isa = PBXLegacyTarget; - buildArgumentsString = "-u $(SRCROOT)/test/dotest.py --apple-sdk $(PLATFORM_NAME) --executable=$(BUILD_DIR)/$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)/lldb -C $(LLDB_PYTHON_TESTSUITE_CC) --arch $(LLDB_PYTHON_TESTSUITE_ARCH) --session-file-format fm --results-formatter lldbsuite.test_event.formatter.xunit.XunitFormatter --build-dir $(BUILD_DIR)/lldb-test-build.noindex --results-file $(BUILD_DIR)/test-results-$(LLDB_PYTHON_TESTSUITE_ARCH).xml --rerun-all-issues --env TERM=vt100 -O--xpass=ignore"; + buildArgumentsString = "-u $(SRCROOT)/test/dotest.py --apple-sdk $(PLATFORM_NAME) --executable=$(BUILD_DIR)/$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)/lldb -C $(LLDB_PYTHON_TESTSUITE_CC) --arch $(LLDB_PYTHON_TESTSUITE_ARCH) --filecheck $(LLVM_BUILD_DIR)/x86_64/bin/FileCheck --session-file-format fm --results-formatter lldbsuite.test_event.formatter.xunit.XunitFormatter --build-dir $(BUILD_DIR)/lldb-test-build.noindex --results-file $(BUILD_DIR)/test-results-$(LLDB_PYTHON_TESTSUITE_ARCH).xml --rerun-all-issues --env TERM=vt100 -O--xpass=ignore"; buildConfigurationList = 238755241C24974600CCE8C3 /* Build configuration list for PBXLegacyTarget "lldb-python-test-suite" */; buildPhases = ( ); diff --git a/lldb/packages/Python/lldbsuite/test/configuration.py b/lldb/packages/Python/lldbsuite/test/configuration.py index a1a0ced..f92b898 100644 --- a/lldb/packages/Python/lldbsuite/test/configuration.py +++ b/lldb/packages/Python/lldbsuite/test/configuration.py @@ -46,6 +46,9 @@ count = 1 arch = None # Must be initialized after option parsing compiler = None # Must be initialized after option parsing +# Path to the FileCheck testing tool. Not optional. +filecheck = None + # The arch might dictate some specific CFLAGS to be passed to the toolchain to build # the inferior programs. The global variable cflags_extras provides a hook to do # just that. @@ -179,3 +182,11 @@ def get_absolute_path_to_root_test_dir(): return test_subdir return os.path.dirname(os.path.realpath(__file__)) + + +def get_filecheck_path(): + """ + Get the path to the FileCheck testing tool. + """ + assert os.path.lexists(filecheck) + return filecheck diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py index f0c5844..ab46628 100644 --- a/lldb/packages/Python/lldbsuite/test/dotest.py +++ b/lldb/packages/Python/lldbsuite/test/dotest.py @@ -307,6 +307,13 @@ def parseOptionsAndInitTestdirs(): os.environ['DSYMUTIL'] = seven.get_command_output( 'xcrun -find -toolchain default dsymutil') + if args.filecheck: + # The CMake build passes in a path to a working FileCheck binary. + configuration.filecheck = os.path.abspath(args.filecheck) + else: + logging.error('No valid FileCheck executable; aborting...') + sys.exit(-1) + if args.channels: lldbtest_config.channels = args.channels diff --git a/lldb/packages/Python/lldbsuite/test/dotest_args.py b/lldb/packages/Python/lldbsuite/test/dotest_args.py index 58190d8..225ccf1 100644 --- a/lldb/packages/Python/lldbsuite/test/dotest_args.py +++ b/lldb/packages/Python/lldbsuite/test/dotest_args.py @@ -85,6 +85,8 @@ def create_parser(): group.add_argument('--dsymutil', metavar='dsymutil', dest='dsymutil', help=textwrap.dedent('Specify which dsymutil to use.')) + group.add_argument('--filecheck', metavar='filecheck', dest='filecheck', help=textwrap.dedent('Specify which FileCheck binary to use.')) + # Test filtering options group = parser.add_argument_group('Test filtering options') group.add_argument( diff --git a/lldb/packages/Python/lldbsuite/test/expression_command/formatters/TestFormatters.py b/lldb/packages/Python/lldbsuite/test/expression_command/formatters/TestFormatters.py index f14b02b..ee5c4a0 100644 --- a/lldb/packages/Python/lldbsuite/test/expression_command/formatters/TestFormatters.py +++ b/lldb/packages/Python/lldbsuite/test/expression_command/formatters/TestFormatters.py @@ -57,14 +57,21 @@ class ExprFormattersTestCase(TestBase): self.runCmd("frame variable foo1.b --show-types") self.runCmd("frame variable foo1.b.b_ref --show-types") - self.expect( - "expression --show-types -- *(new foo(47))", - substrs=[ - '(int) a = 47', - '(bar) b = {', - '(int) i = 94', - '(baz) b = {', - '(int) k = 99']) + self.filecheck("expression --show-types -- *(new foo(47))", __file__, + '-check-prefix=EXPR-TYPES-NEW-FOO') + # EXPR-TYPES-NEW-FOO: (foo) ${{.*}} = { + # EXPR-TYPES-NEW-FOO-NEXT: (int) a = 47 + # EXPR-TYPES-NEW-FOO-NEXT: (int *) a_ptr = 0x + # EXPR-TYPES-NEW-FOO-NEXT: (bar) b = { + # EXPR-TYPES-NEW-FOO-NEXT: (int) i = 94 + # EXPR-TYPES-NEW-FOO-NEXT: (int *) i_ptr = 0x + # EXPR-TYPES-NEW-FOO-NEXT: (baz) b = { + # EXPR-TYPES-NEW-FOO-NEXT: (int) h = 97 + # EXPR-TYPES-NEW-FOO-NEXT: (int) k = 99 + # EXPR-TYPES-NEW-FOO-NEXT: } + # EXPR-TYPES-NEW-FOO-NEXT: (baz &) b_ref = 0x + # EXPR-TYPES-NEW-FOO-NEXT: } + # EXPR-TYPES-NEW-FOO-NEXT: } self.runCmd("type summary add -F formatters.foo_SummaryProvider foo") @@ -80,68 +87,49 @@ class ExprFormattersTestCase(TestBase): self.expect("expression foo1.a_ptr", substrs=['(int *) $', '= 0x', ' -> 13']) - self.expect( - "expression foo1", - substrs=[ - '(foo) $', - ' a = 12', - 'a_ptr = ', - ' -> 13', - 'i = 24', - 'i_ptr = ', - ' -> 25']) - - self.expect( - "expression --ptr-depth=1 -- new foo(47)", - substrs=[ - '(foo *) $', - 'a = 47', - 'a_ptr = ', - ' -> 48', - 'i = 94', - 'i_ptr = ', - ' -> 95']) - - self.expect( - "expression foo2", - substrs=[ - '(foo) $', - 'a = 121', - 'a_ptr = ', - ' -> 122', - 'i = 242', - 'i_ptr = ', - ' -> 243']) + self.filecheck("expression foo1", __file__, '-check-prefix=EXPR-FOO1') + # EXPR-FOO1: (foo) $ + # EXPR-FOO1-SAME: a = 12 + # EXPR-FOO1-SAME: a_ptr = {{[0-9]+}} -> 13 + # EXPR-FOO1-SAME: i = 24 + # EXPR-FOO1-SAME: i_ptr = {{[0-9]+}} -> 25 + # EXPR-FOO1-SAME: b_ref = {{[0-9]+}} + # EXPR-FOO1-SAME: h = 27 + # EXPR-FOO1-SAME: k = 29 + + self.filecheck("expression --ptr-depth=1 -- new foo(47)", __file__, + '-check-prefix=EXPR-PTR-DEPTH1') + # EXPR-PTR-DEPTH1: (foo *) $ + # EXPR-PTR-DEPTH1-SAME: a = 47 + # EXPR-PTR-DEPTH1-SAME: a_ptr = {{[0-9]+}} -> 48 + # EXPR-PTR-DEPTH1-SAME: i = 94 + # EXPR-PTR-DEPTH1-SAME: i_ptr = {{[0-9]+}} -> 95 + + self.filecheck("expression foo2", __file__, '-check-prefix=EXPR-FOO2') + # EXPR-FOO2: (foo) $ + # EXPR-FOO2-SAME: a = 121 + # EXPR-FOO2-SAME: a_ptr = {{[0-9]+}} -> 122 + # EXPR-FOO2-SAME: i = 242 + # EXPR-FOO2-SAME: i_ptr = {{[0-9]+}} -> 243 + # EXPR-FOO2-SAME: h = 245 + # EXPR-FOO2-SAME: k = 247 object_name = self.res.GetOutput() object_name = object_name[7:] object_name = object_name[0:object_name.find(' =')] - self.expect( - "frame variable foo2", - substrs=[ - '(foo)', - 'foo2', - 'a = 121', - 'a_ptr = ', - ' -> 122', - 'i = 242', - 'i_ptr = ', - ' -> 243']) - - self.expect( - "expression $" + - object_name, - substrs=[ - '(foo) $', - 'a = 121', - 'a_ptr = ', - ' -> 122', - 'i = 242', - 'i_ptr = ', - ' -> 243', - 'h = 245', - 'k = 247']) + self.filecheck("frame variable foo2", __file__, '-check-prefix=VAR-FOO2') + # VAR-FOO2: (foo) foo2 + # VAR-FOO2-SAME: a = 121 + # VAR-FOO2-SAME: a_ptr = {{[0-9]+}} -> 122 + # VAR-FOO2-SAME: i = 242 + # VAR-FOO2-SAME: i_ptr = {{[0-9]+}} -> 243 + # VAR-FOO2-SAME: h = 245 + # VAR-FOO2-SAME: k = 247 + + # The object is the same as foo2, so use the EXPR-FOO2 checks. + self.filecheck("expression $" + object_name, __file__, + '-check-prefix=EXPR-FOO2') self.runCmd("type summary delete foo") self.runCmd( diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/typedef_array/main.cpp b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/typedef_array/main.cpp index 649c1e0..5b43102 100644 --- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/typedef_array/main.cpp +++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/typedef_array/main.cpp @@ -9,6 +9,11 @@ typedef int Foo; int main() { + // CHECK: (Foo [3]) array = { + // CHECK-NEXT: (Foo) [0] = 1 + // CHECK-NEXT: (Foo) [1] = 2 + // CHECK-NEXT: (Foo) [2] = 3 + // CHECK-NEXT: } Foo array[3] = {1,2,3}; - return 0; //% self.expect("frame variable array --show-types --", substrs = ['(Foo [3]) array = {','(Foo) [0] = 1','(Foo) [1] = 2','(Foo) [2] = 3']) + return 0; //% self.filecheck("frame variable array --show-types --", 'main.cpp') } diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py index 7af9728..96ce825 100644 --- a/lldb/packages/Python/lldbsuite/test/lldbtest.py +++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py @@ -2214,6 +2214,54 @@ class TestBase(Base): compare_string, msg=COMPLETION_MSG( str_input, p, match_strings), exe=False, patterns=[p]) + def filecheck( + self, + command, + check_file, + filecheck_options = ''): + # Run the command. + self.runCmd( + command, + msg="FileCheck'ing result of `{0}`".format(command)) + + # Get the error text if there was an error, and the regular text if not. + output = self.res.GetOutput() if self.res.Succeeded() \ + else self.res.GetError() + + # Assemble the absolute path to the check file. As a convenience for + # LLDB inline tests, assume that the check file is a relative path to + # a file within the inline test directory. + if check_file.endswith('.pyc'): + check_file = check_file[:-1] + if hasattr(self, 'test_filename'): + check_file_abs = os.path.join(os.path.dirname(self.test_filename), + check_file) + else: + check_file_abs = os.path.abspath(check_file) + + # Run FileCheck. + filecheck_bin = configuration.get_filecheck_path() + filecheck_args = [filecheck_bin, check_file_abs] + if filecheck_options: + filecheck_args.append(filecheck_options) + subproc = Popen(filecheck_args, stdin=PIPE, stdout=PIPE, stderr=PIPE) + cmd_stdout, cmd_stderr = subproc.communicate(input=output) + cmd_status = subproc.returncode + + if cmd_status != 0: + filecheck_cmd = " ".join(filecheck_args) + self.assertTrue(cmd_status == 0, """ +--- FileCheck failed (code={0}) --- +{1} + +FileCheck input: +{2} + +FileCheck output: +{3} +{4} +""".format(cmd_status, filecheck_cmd, output, cmd_stdout, cmd_stderr)) + def expect( self, str, diff --git a/lldb/test/CMakeLists.txt b/lldb/test/CMakeLists.txt index 6f0a5f2..790ab3e 100644 --- a/lldb/test/CMakeLists.txt +++ b/lldb/test/CMakeLists.txt @@ -49,6 +49,7 @@ set(LLDB_TEST_COMMON_ARGS list(APPEND LLDB_TEST_COMMON_ARGS --executable ${LLDB_TEST_EXECUTABLE} --dsymutil ${LLDB_TEST_DSYMUTIL} + --filecheck ${LLDB_TEST_FILECHECK} -C ${LLDB_TEST_C_COMPILER} )