From 0faca0f09f43369ab7132f884d4f54873f54a168 Mon Sep 17 00:00:00 2001 From: Stella Stamenova Date: Thu, 2 Aug 2018 21:26:19 +0000 Subject: [PATCH] [lldbsuite, windows] Mark tests as XFAIL on Windows or skip them Summary: 1) Several tests that are flakey on windows fail the run even if they are marked as expected to be flakey. This is because they fail frequently enough that even a retry won't help 2) Skip several tests on Windows that will occasionally hang rather than failing or exiting. This is causing the entire test suite to hang Reviewers: asmith, labath, zturner Reviewed By: zturner Differential Revision: https://reviews.llvm.org/D50198 llvm-svn: 338769 --- .../test/functionalities/attach_resume/TestAttachResume.py | 2 +- .../breakpoint_ignore_count/TestBreakpointIgnoreCount.py | 2 ++ .../test/functionalities/memory/cache/TestMemoryCache.py | 4 +++- .../thread/exit_during_step/TestExitDuringStep.py | 9 +++++++++ .../test/functionalities/thread/step_out/TestThreadStepOut.py | 3 +++ .../test/functionalities/thread/thread_exit/TestThreadExit.py | 4 ++++ .../Python/lldbsuite/test/lang/cpp/virtual/TestVirtual.py | 1 + .../Python/lldbsuite/test/python_api/event/TestEvents.py | 1 + 8 files changed, 24 insertions(+), 2 deletions(-) diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/attach_resume/TestAttachResume.py b/lldb/packages/Python/lldbsuite/test/functionalities/attach_resume/TestAttachResume.py index ad87796..754acad 100644 --- a/lldb/packages/Python/lldbsuite/test/functionalities/attach_resume/TestAttachResume.py +++ b/lldb/packages/Python/lldbsuite/test/functionalities/attach_resume/TestAttachResume.py @@ -21,7 +21,7 @@ class AttachResumeTestCase(TestBase): @skipIfRemote @expectedFailureAll(oslist=['freebsd'], bugnumber='llvm.org/pr19310') - @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24778") + @skipIfWindows # llvm.org/pr24778, llvm.org/pr21753 def test_attach_continue_interrupt_detach(self): """Test attach/continue/interrupt/detach""" self.build() diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ignore_count/TestBreakpointIgnoreCount.py b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ignore_count/TestBreakpointIgnoreCount.py index eb4bac7..e3bf4c2 100644 --- a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ignore_count/TestBreakpointIgnoreCount.py +++ b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ignore_count/TestBreakpointIgnoreCount.py @@ -18,12 +18,14 @@ class BreakpointIgnoreCountTestCase(TestBase): mydir = TestBase.compute_mydir(__file__) + @skipIfWindows # This test will hang on windows llvm.org/pr21753 def test_with_run_command(self): """Exercise breakpoint ignore count with 'breakpoint set -i '.""" self.build() self.breakpoint_ignore_count() @add_test_categories(['pyapi']) + @skipIfWindows # This test will hang on windows llvm.org/pr21753 def test_with_python_api(self): """Use Python APIs to set breakpoint ignore count.""" self.build() diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/memory/cache/TestMemoryCache.py b/lldb/packages/Python/lldbsuite/test/functionalities/memory/cache/TestMemoryCache.py index f45479b..6775e16 100644 --- a/lldb/packages/Python/lldbsuite/test/functionalities/memory/cache/TestMemoryCache.py +++ b/lldb/packages/Python/lldbsuite/test/functionalities/memory/cache/TestMemoryCache.py @@ -24,7 +24,9 @@ class MemoryCacheTestCase(TestBase): # Find the line number to break inside main(). self.line = line_number('main.cpp', '// Set break point at this line.') - @expectedFlakeyOS(oslist=["windows"]) + # The test is actually flakey on Windows, failing every dozen or so runs, but even with the flakey + # decorator it still fails + @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr38373") def test_memory_cache(self): """Test the MemoryCache class with a sequence of 'memory read' and 'memory write' operations.""" self.build() diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_step/TestExitDuringStep.py b/lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_step/TestExitDuringStep.py index 0343a88..f83b467 100644 --- a/lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_step/TestExitDuringStep.py +++ b/lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_step/TestExitDuringStep.py @@ -18,6 +18,9 @@ class ExitDuringStepTestCase(TestBase): mydir = TestBase.compute_mydir(__file__) @skipIfFreeBSD # llvm.org/pr21411: test is hanging + # The test is actually flakey on Windows, failing every dozen or so runs, but even with the flakey + # decorator it still fails + @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr38373") def test(self): """Test thread exit during step handling.""" self.build(dictionary=self.getBuildFlags()) @@ -27,6 +30,9 @@ class ExitDuringStepTestCase(TestBase): True) @skipIfFreeBSD # llvm.org/pr21411: test is hanging + # The test is actually flakey on Windows, failing every dozen or so runs, but even with the flakey + # decorator it still fails + @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr38373") def test_step_over(self): """Test thread exit during step-over handling.""" self.build(dictionary=self.getBuildFlags()) @@ -36,6 +42,9 @@ class ExitDuringStepTestCase(TestBase): False) @skipIfFreeBSD # llvm.org/pr21411: test is hanging + # The test is actually flakey on Windows, failing every dozen or so runs, but even with the flakey + # decorator it still fails + @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr38373") def test_step_in(self): """Test thread exit during step-in handling.""" self.build(dictionary=self.getBuildFlags()) diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/step_out/TestThreadStepOut.py b/lldb/packages/Python/lldbsuite/test/functionalities/thread/step_out/TestThreadStepOut.py index 238b188..e786e8d 100644 --- a/lldb/packages/Python/lldbsuite/test/functionalities/thread/step_out/TestThreadStepOut.py +++ b/lldb/packages/Python/lldbsuite/test/functionalities/thread/step_out/TestThreadStepOut.py @@ -25,6 +25,7 @@ class ThreadStepOutTestCase(TestBase): @expectedFailureAll( oslist=["freebsd"], bugnumber="llvm.org/pr18066 inferior does not exit") + @skipIfWindows # This test will hang on windows llvm.org/pr21753 @expectedFailureAll(oslist=["windows"]) def test_step_single_thread(self): """Test thread step out on one thread via command interpreter. """ @@ -39,6 +40,7 @@ class ThreadStepOutTestCase(TestBase): @expectedFailureAll( oslist=["freebsd"], bugnumber="llvm.org/pr19347 2nd thread stops at breakpoint") + @skipIfWindows # This test will hang on windows llvm.org/pr21753 @expectedFailureAll(oslist=["windows"]) @expectedFailureAll(oslist=["watchos"], archs=['armv7k'], bugnumber="rdar://problem/34674488") # stop reason is trace when it should be step-out def test_step_all_threads(self): @@ -54,6 +56,7 @@ class ThreadStepOutTestCase(TestBase): @expectedFailureAll( oslist=["freebsd"], bugnumber="llvm.org/pr19347 2nd thread stops at breakpoint") + @skipIfWindows # This test will hang on windows llvm.org/pr21753 @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24681") def test_python(self): """Test thread step out on one thread via Python API (dwarf).""" diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_exit/TestThreadExit.py b/lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_exit/TestThreadExit.py index 07ceb3f..db1d31d 100644 --- a/lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_exit/TestThreadExit.py +++ b/lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_exit/TestThreadExit.py @@ -8,6 +8,7 @@ from __future__ import print_function import os import time import lldb +from lldbsuite.test.decorators import * from lldbsuite.test.lldbtest import * import lldbsuite.test.lldbutil as lldbutil @@ -25,6 +26,9 @@ class ThreadExitTestCase(TestBase): self.break_3 = line_number('main.cpp', '// Set third breakpoint here') self.break_4 = line_number('main.cpp', '// Set fourth breakpoint here') + # The test is actually flakey on Windows, failing every dozen or so runs, but even with the flakey + # decorator it still fails + @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr38373") def test(self): """Test thread exit handling.""" self.build(dictionary=self.getBuildFlags()) diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/virtual/TestVirtual.py b/lldb/packages/Python/lldbsuite/test/lang/cpp/virtual/TestVirtual.py index b3fe9c4..365eb82 100644 --- a/lldb/packages/Python/lldbsuite/test/lang/cpp/virtual/TestVirtual.py +++ b/lldb/packages/Python/lldbsuite/test/lang/cpp/virtual/TestVirtual.py @@ -36,6 +36,7 @@ class CppVirtualMadness(TestBase): @expectedFailureAll( compiler="icc", bugnumber="llvm.org/pr16808 lldb does not call the correct virtual function with icc.") + @skipIfWindows # This test will hang on windows llvm.org/pr21753 def test_virtual_madness(self): """Test that expression works correctly with virtual inheritance as well as virtual function.""" self.build() diff --git a/lldb/packages/Python/lldbsuite/test/python_api/event/TestEvents.py b/lldb/packages/Python/lldbsuite/test/python_api/event/TestEvents.py index 8a9e456..9c8fa978 100644 --- a/lldb/packages/Python/lldbsuite/test/python_api/event/TestEvents.py +++ b/lldb/packages/Python/lldbsuite/test/python_api/event/TestEvents.py @@ -31,6 +31,7 @@ class EventAPITestCase(TestBase): @expectedFailureAll( oslist=["linux"], bugnumber="llvm.org/pr23730 Flaky, fails ~1/10 cases") + @skipIfWindows # This test will hang on windows llvm.org/pr21753 def test_listen_for_and_print_event(self): """Exercise SBEvent API.""" self.build() -- 2.7.4