src/chrome/test/chromedriver/test/test_expectations

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Test expectation list for WebDriver Java acceptance tests.
   6
   7 It is evaluated through Python.
   8 """
   9
  10 import fnmatch
  11
  12
  13 _REVISION_NEGATIVE_FILTER = {}
  14 _REVISION_NEGATIVE_FILTER['HEAD'] = [
  15     'AlertsTest.testIncludesAlertTextInUnhandledAlertException',
  16     'AlertsTest.testSettingTheValueOfAnAlertThrows',
  17     'AlertsTest.testShouldGetTextOfAlertOpenedInSetTimeout',
  18     'AlertsTest.testShouldHandleAlertOnPageUnload',
  19     'AlertsTest.testShouldHandleAlertOnWindowClose',
  20     'AlertsTest.testShouldNotHandleAlertInAnotherWindow',
  21     'AlertsTest.testSwitchingToMissingAlertInAClosedWindowThrows',
  22     'AuthenticatedPageLoadingTest.*',
  23     'AvailableLogsTest.driverLogShouldBeEnabledByDefault',
  24     'BasicMouseInterfaceTest.testMousePositionIsNotPreservedInActionsChain',
  25     'BasicMouseInterfaceTest.testMovingIntoAnImageEnclosedInALink',
  26     'BasicMouseInterfaceTest.testMovingMouseBackAndForthPastViewPort',
  27     'BasicMouseInterfaceTest.testMovingMousePastViewPort',
  28     'ChromeOptionsFunctionalTest.canStartChromeWithCustomOptions',
  29     'ClickScrollingTest.testShouldBeAbleToClickOnAnElementHiddenByDoubleOverflow',
  30     'ClickScrollingTest.testShouldNotBeAbleToClickElementThatIsOutOfViewInANonScrollableFrame',
  31     'ClickScrollingTest.testShouldNotScrollIfAlreadyScrolledAndElementIsInView',
  32     'ClickTest.testShouldBeAbleToClickOnAnElementInFrameGreaterThanTwoViewports',
  33     'CombinedInputActionsTest.testClickingOnFormElements',
  34     'CoordinatesTest.testShouldScrollPageAndGetCoordinatesOfAnElementThatIsOutOfViewPort',
  35     'CorrectEventFiringTest.testShouldFireEventsInTheRightOrder',
  36     'CorrectEventFiringTest.testShouldFireFocusEventWhenClicking',
  37     'DragAndDropTest.testDragTooFar',
  38     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=374
  39     'ElementFindingTest.testShouldNotBeAbleToFindAnElementOnABlankPage',
  40     'ExecutingAsyncJavascriptTest.includesAlertTextInUnhandledAlertException',
  41     'ExecutingAsyncJavascriptTest.shouldCatchErrorsWithMessageAndStacktraceWhenExecutingInitialScript',
  42     'ExecutingAsyncJavascriptTest.throwsIfAlertHappensDuringScript',
  43     'ExecutingAsyncJavascriptTest.throwsIfAlertHappensDuringScriptWhichTimesOut',
  44     'ExecutingAsyncJavascriptTest.throwsIfScriptTriggersAlert',
  45     'ExecutingAsyncJavascriptTest.throwsIfScriptTriggersAlertWhichTimesOut',
  46     'ExecutingJavascriptTest.testShouldThrowExceptionIfExecutingOnNoPage',
  47     'ExecutingJavascriptTest.testShouldThrowAnExceptionWithMessageAndStacktraceWhenTheJavascriptIsBad',
  48     'FormHandlingTest.testShouldNotBeAbleToSubmitAFormThatDoesNotExist',
  49     'FrameSwitchingTest.testShouldNotBeAbleToDoAnythingTheFrameIsDeletedFromUnderUs',
  50     'I18nTest.testShouldBeAbleToActivateIMEEngine',
  51     # Broken because AddWebStorage.java is broken.
  52     'LocalStorageTest.*',
  53     'LocationContextTest.*',
  54     'MiscTest.testShouldBeAbleToGetTheSourceOfAnXmlDocument',
  55     'PageLoadingTest.shouldBeAbleToDisableAcceptOfInsecureSslCertsWithRequiredCapability',
  56     'PageLoadingTest.testShouldNotWaitIndefinitelyIfAnExternalResourceFailsToLoad',
  57     # PerformanceLoggingTest is for firefox only; "performance" log deprecated.
  58     'PerformanceLoggingTest.*',
  59     'ProxySettingTest.canConfigureProxyWithRequiredCapability',
  60     'ProxySettingTest.requiredProxyCapabilityShouldHavePriority',
  61     'RenderedWebElementTest.testShouldHandleNonIntegerPositionAndSize',
  62     # Broken because AddWebStorage.java is broken.
  63     'SessionStorageTest.*',
  64     'SqlDatabaseTest.*',
  65     'SvgDocumentTest.testClickOnSvgElement',
  66     'TakesScreenshotTest.testShouldCaptureScreenshot',
  67     'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePage',
  68     'TakesScreenshotTest.testShouldCaptureScreenshotWithLongX',
  69     'TakesScreenshotTest.testShouldCaptureScreenshotWithLongY',
  70     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLong',
  71     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLongX',
  72     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLongY',
  73     'TextHandlingTest.testShouldNotReturnLtrMarks',
  74     'TextPagesTest.testShouldBeAbleToLoadASimplePageOfText',
  75     'TextPagesTest.testShouldThrowExceptionWhenAddingCookieToAPageThatIsNotHtml',
  76     'TypingTest.testGenerateKeyPressEventEvenWhenElementPreventsDefault',
  77     'TypingTest.testNonPrintableCharactersShouldWorkWithContentEditableOrDesignModeSet',
  78     'TypingTest.testShouldBeAbleToTypeIntoContentEditableElementWithExistingValue',
  79     'TypingTest.testShouldNotTypeIntoElementsThatPreventKeyDownEvents',
  80     'TypingTest.testTypingIntoAnIFrameWithContentEditableOrDesignModeSet',
  81     'UnexpectedAlertBehaviorTest.*',
  82     'VisibilityTest.testElementHiddenByOverflowXIsNotVisible',
  83     'VisibilityTest.testElementHiddenByOverflowYIsNotVisible',
  84     'VisibilityTest.tooSmallAWindowWithOverflowHiddenIsNotAProblem',
  85     'WindowTest.*',
  86     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=528
  87     'PageLoadingTest.testShouldDoNothingIfThereIsNothingToGoBackTo',
  88     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=653
  89     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
  90     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistory',
  91 ]
  92 _REVISION_NEGATIVE_FILTER['31'] = _REVISION_NEGATIVE_FILTER['HEAD'] + [
  93     'I18nTest.*',  # This was fixed in a later version of 31 than we use.
  94 ]
  95
  96
  97 _OS_NEGATIVE_FILTER = {}
  98 _OS_NEGATIVE_FILTER['win'] = [
  99     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=282
 100     'PageLoadingTest.testShouldNotHangIfDocumentOpenCallIsNeverFollowedByDocumentCloseCall',
 101     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=373
 102     'RenderedWebElementTest.testHoverPersists',
 103     'RenderedWebElementTest.canClickOnASuckerFishStyleMenu',
 104     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=416
 105     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePageAfterSwitching',
 106     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
 107 ]
 108 _OS_NEGATIVE_FILTER['linux'] = []
 109 _OS_NEGATIVE_FILTER['mac'] = [
 110     # https://code.google.com/p/chromedriver/issues/detail?id=26
 111     'AlertsTest.testAlertShouldNotAllowAdditionalCommandsIfDismissed',
 112     'AlertsTest.testShouldAllowUsersToDismissAnAlertManually',
 113     'FormHandlingTest.handleFormWithJavascriptAction',
 114     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=354
 115     'AlertsTest.testShouldAllowUsersToAcceptAnAlertInAFrame',
 116     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=375
 117     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
 118 ]
 119 _OS_NEGATIVE_FILTER['android:chrome'] = [
 120     'ChromeOptionsFunctionalTest.canStartChromeWithCustomOptions',
 121     'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrame',
 122     'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrameThatIsOutOfView',
 123     'ClickTest.testShouldOnlyFollowHrefOnce',
 124     'CombinedInputActionsTest.testCombiningShiftAndClickResultsInANewWindow',
 125     'ElementSelectingTest.testShouldBeAbleToToggleEnabledMultiSelectOption',
 126     'FrameSwitchingTest.testShouldBeAbleToClickInAFrameThatRewritesTopWindowLocation',
 127     'JavascriptEnabledDriverTest.testShouldBeAbleToClickALinkThatClosesAWindow',
 128     'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
 129     'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
 130     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
 131     'PageLoadingTest.testShouldFollowMetaRedirects',
 132     'PageLoadingTest.testShouldWaitForDocumentToBeLoaded',
 133     'PerformanceLogTypeTest.pageLoadShouldProducePerformanceLogEntries',
 134     'PerformanceLogTypeTest.shouldBeAbleToEnablePerformanceLog',
 135     'SelectElementHandlingTest.testShouldBePossibleToDeselectASingleOptionFromASelectWhichAllowsMultipleChoices',
 136     'SelectElementTest.shouldAllowOptionsToBeDeselectedByIndex',
 137     'SelectElementTest.shouldAllowOptionsToBeDeselectedByReturnedValue',
 138     'SelectElementTest.shouldAllowUserToDeselectAllWhenSelectSupportsMultipleSelections',
 139     'SelectElementTest.shouldAllowUserToDeselectOptionsByVisibleText',
 140     'SessionHandlingTest.callingAnyOperationAfterClosingTheLastWindowShouldThrowAnException',
 141     'SessionHandlingTest.callingQuitAfterClosingTheLastWindowIsANoOp',
 142     'TakesScreenshotTest.testCaptureToBase64',
 143     'TakesScreenshotTest.testSaveScreenshotAsFile',
 144     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
 145     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePageAfterSwitching',
 146     'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePageAfterSwitching',
 147
 148     # Alerts are not yet supported on Android.
 149     'AlertsTest.*',
 150
 151     # http://crbug.com/156390
 152     'DragAndDropTest.*',
 153
 154     # Scrolling touch events are not supported.
 155     'TouchScrollTest.*',
 156
 157     # These tests start multiple sessions, which is not supported on a single
 158     # Android device.
 159     "AvailableLogsTest.shouldBeAbleToEnableProfilerLog",
 160     "GetLogsTest.turningOffLogShouldMeanNoLogMessages",
 161     "SessionHandlingTest.callingAnyOperationAfterQuitShouldThrowAnException",
 162     "SessionHandlingTest.callingQuitMoreThanOnceOnASessionIsANoOp",
 163     # Test is written using local files; doesn't work on Android.
 164     'UploadTest.testFileUploading',
 165     # Not applicable on Chromium Test Shell (doesn't support tabs).
 166     'WindowSwitchingTest.*',
 167
 168     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=441
 169     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistory',
 170     'PageLoadingTest.testShouldBeAbleToNavigateForwardsInTheBrowserHistory',
 171     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=528
 172     'PageLoadingTest.testShouldDoNothingIfThereIsNothingToGoBackTo',
 173
 174     # https://code.google.com/p/chromedriver/issues/detail?id=526
 175     'ElementFindingTest.testShouldNotBeAbleToLocateASingleElementOnABlankPage',
 176     'ElementFindingTest.testShouldBeAbleToFindAnElementByBooleanAttributeUsingShortCssSelector',
 177
 178     # https://code.google.com/p/chromedriver/issues/detail?id=527
 179     'ClickTest.testCanClickOnALinkThatContainsEmbeddedBlockElements',
 180
 181     # https://code.google.com/p/chromedriver/issues/detail?id=604
 182     'I18nTest.*',
 183 ]
 184 _OS_NEGATIVE_FILTER['android:chrome_stable'] = (
 185     _OS_NEGATIVE_FILTER['android:chrome'])
 186 _OS_NEGATIVE_FILTER['android:chrome_beta'] = (
 187     _OS_NEGATIVE_FILTER['android:chrome'])
 188 _OS_NEGATIVE_FILTER['android:chromium_test_shell'] = (
 189     _OS_NEGATIVE_FILTER['android:chrome'])
 190 _OS_NEGATIVE_FILTER['android:chromedriver_webview_shell'] = (
 191     _OS_NEGATIVE_FILTER['android:chrome'] + [
 192          # https://code.google.com/p/chromedriver/issues/detail?id=645
 193         'ClickScrollingTest.testShouldBeAbleToClickElementInAFrameThatIsOutOfView',
 194         'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInAFrameThatIsOutOfView',
 195         'FormHandlingTest.handleFormWithJavascriptAction',
 196         'TypingTest.testShouldReportKeyCodeOfArrowKeysUpDownEvents',
 197         'TypingTest.testShouldReportKeyCodeOfArrowKeys',
 198         'CombinedInputActionsTest.testCanClickOnLinks',
 199         'CombinedInputActionsTest.testCanClickOnLinksWithAnOffset',
 200         'CombinedInputActionsTest.testMouseMovementWorksWhenNavigatingToAnotherPage',
 201     ]
 202 )
 203
 204
 205 def _GetRevisionNegativeFilter(chrome_version):
 206   if chrome_version in _REVISION_NEGATIVE_FILTER:
 207     return _REVISION_NEGATIVE_FILTER[chrome_version]
 208   return _REVISION_NEGATIVE_FILTER['HEAD']
 209
 210
 211 def GetDisabledTestMatchers(operating_system, chrome_version):
 212   """Returns the list of disabled test matchers for the specific configuration.
 213
 214   Args:
 215     operating_system: The operating system, one of 'linux', 'mac', 'win', or
 216         'android'.
 217     chrome_version: Chrome version to test against, e.g., 'HEAD' or '26'.
 218
 219   Returns:
 220     List of disabled test matchers, which may contain '*' wildcards.
 221   """
 222   return (_OS_NEGATIVE_FILTER[operating_system] +
 223           _GetRevisionNegativeFilter(chrome_version))[:]
 224
 225
 226 def ApplyJavaTestFilter(operating_system, chrome_version, tests):
 227   """Applies the test filter to the given list of tests.
 228
 229   Args:
 230     operating_system: The operating system, one of 'linux', 'mac', 'win', or
 231         'android'.
 232     chrome_version: Chrome version to test against, e.g., 'HEAD' or '26'.
 233     test: list of test names to filter.
 234
 235   Returns:
 236     Set of passed test names.
 237   """
 238   filters = (_OS_NEGATIVE_FILTER[operating_system] +
 239              _GetRevisionNegativeFilter(chrome_version))
 240   passed = set(tests)
 241   for f in filters:
 242     passed.difference_update(set(t for t in tests if fnmatch.fnmatch(t, f)))
 243   return passed