src/chrome/test/chromedriver/test/test_expectations

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Test expectation list for WebDriver Java acceptance tests.
   6
   7 It is evaluated through Python.
   8 """
   9
  10 import fnmatch
  11
  12
  13 _REVISION_NEGATIVE_FILTER = {}
  14 _REVISION_NEGATIVE_FILTER['HEAD'] = [
  15     'AlertsTest.testIncludesAlertTextInUnhandledAlertException',
  16     'AlertsTest.testSettingTheValueOfAnAlertThrows',
  17     'AlertsTest.testShouldGetTextOfAlertOpenedInSetTimeout',
  18     'AlertsTest.testShouldHandleAlertOnPageUnload',
  19     'AlertsTest.testShouldHandleAlertOnWindowClose',
  20     'AlertsTest.testShouldNotHandleAlertInAnotherWindow',
  21     'AlertsTest.testSwitchingToMissingAlertInAClosedWindowThrows',
  22     'AuthenticatedPageLoadingTest.*',
  23     'AvailableLogsTest.driverLogShouldBeEnabledByDefault',
  24     'BasicMouseInterfaceTest.testMousePositionIsNotPreservedInActionsChain',
  25     'BasicMouseInterfaceTest.testMovingIntoAnImageEnclosedInALink',
  26     'BasicMouseInterfaceTest.testMovingMouseBackAndForthPastViewPort',
  27     'BasicMouseInterfaceTest.testMovingMousePastViewPort',
  28     'ChromeOptionsFunctionalTest.canStartChromeWithCustomOptions',
  29     'ClickScrollingTest.testShouldBeAbleToClickOnAnElementHiddenByDoubleOverflow',
  30     'ClickScrollingTest.testShouldNotBeAbleToClickElementThatIsOutOfViewInANonScrollableFrame',
  31     'ClickScrollingTest.testShouldNotScrollIfAlreadyScrolledAndElementIsInView',
  32     'ClickTest.testShouldBeAbleToClickOnAnElementInFrameGreaterThanTwoViewports',
  33     'CombinedInputActionsTest.testClickingOnFormElements',
  34     'CoordinatesTest.testShouldScrollPageAndGetCoordinatesOfAnElementThatIsOutOfViewPort',
  35     'CorrectEventFiringTest.testShouldFireEventsInTheRightOrder',
  36     'CorrectEventFiringTest.testShouldFireFocusEventWhenClicking',
  37     'DragAndDropTest.testDragTooFar',
  38     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=374
  39     'ElementFindingTest.testShouldNotBeAbleToFindAnElementOnABlankPage',
  40     'ExecutingAsyncJavascriptTest.includesAlertTextInUnhandledAlertException',
  41     'ExecutingAsyncJavascriptTest.shouldCatchErrorsWithMessageAndStacktraceWhenExecutingInitialScript',
  42     'ExecutingAsyncJavascriptTest.throwsIfAlertHappensDuringScript',
  43     'ExecutingAsyncJavascriptTest.throwsIfAlertHappensDuringScriptWhichTimesOut',
  44     'ExecutingAsyncJavascriptTest.throwsIfScriptTriggersAlert',
  45     'ExecutingAsyncJavascriptTest.throwsIfScriptTriggersAlertWhichTimesOut',
  46     'ExecutingJavascriptTest.testShouldThrowExceptionIfExecutingOnNoPage',
  47     'ExecutingJavascriptTest.testShouldThrowAnExceptionWithMessageAndStacktraceWhenTheJavascriptIsBad',
  48     'FormHandlingTest.testShouldNotBeAbleToSubmitAFormThatDoesNotExist',
  49     'FrameSwitchingTest.testShouldNotBeAbleToDoAnythingTheFrameIsDeletedFromUnderUs',
  50     'I18nTest.testShouldBeAbleToActivateIMEEngine',
  51     # Broken because AddWebStorage.java is broken.
  52     'LocalStorageTest.*',
  53     'LocationContextTest.*',
  54     'MiscTest.testShouldBeAbleToGetTheSourceOfAnXmlDocument',
  55     'PageLoadingTest.shouldBeAbleToDisableAcceptOfInsecureSslCertsWithRequiredCapability',
  56     'PageLoadingTest.testShouldNotWaitIndefinitelyIfAnExternalResourceFailsToLoad',
  57     # PerformanceLoggingTest is for firefox only; "performance" log deprecated.
  58     'PerformanceLoggingTest.*',
  59     'ProxySettingTest.canConfigureProxyWithRequiredCapability',
  60     'ProxySettingTest.requiredProxyCapabilityShouldHavePriority',
  61     'RenderedWebElementTest.testShouldHandleNonIntegerPositionAndSize',
  62     # Broken because AddWebStorage.java is broken.
  63     'SessionStorageTest.*',
  64     'SqlDatabaseTest.*',
  65     'SvgDocumentTest.testClickOnSvgElement',
  66     'TakesScreenshotTest.testShouldCaptureScreenshot',
  67     'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePage',
  68     'TakesScreenshotTest.testShouldCaptureScreenshotWithLongX',
  69     'TakesScreenshotTest.testShouldCaptureScreenshotWithLongY',
  70     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLong',
  71     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLongX',
  72     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLongY',
  73     'TextHandlingTest.testShouldNotReturnLtrMarks',
  74     'TextPagesTest.testShouldBeAbleToLoadASimplePageOfText',
  75     'TextPagesTest.testShouldThrowExceptionWhenAddingCookieToAPageThatIsNotHtml',
  76     'TypingTest.testGenerateKeyPressEventEvenWhenElementPreventsDefault',
  77     'TypingTest.testNonPrintableCharactersShouldWorkWithContentEditableOrDesignModeSet',
  78     'TypingTest.testShouldBeAbleToTypeIntoContentEditableElementWithExistingValue',
  79     'TypingTest.testShouldNotTypeIntoElementsThatPreventKeyDownEvents',
  80     'TypingTest.testTypingIntoAnIFrameWithContentEditableOrDesignModeSet',
  81     'UnexpectedAlertBehaviorTest.*',
  82     'VisibilityTest.testElementHiddenByOverflowXIsNotVisible',
  83     'VisibilityTest.testElementHiddenByOverflowYIsNotVisible',
  84     'VisibilityTest.tooSmallAWindowWithOverflowHiddenIsNotAProblem',
  85     'WindowTest.*',
  86     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=528
  87     'PageLoadingTest.testShouldDoNothingIfThereIsNothingToGoBackTo',
  88     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=653
  89     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
  90     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistory',
  91 ]
  92
  93
  94 _OS_NEGATIVE_FILTER = {}
  95 _OS_NEGATIVE_FILTER['win'] = [
  96     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=282
  97     'PageLoadingTest.testShouldNotHangIfDocumentOpenCallIsNeverFollowedByDocumentCloseCall',
  98     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=373
  99     'RenderedWebElementTest.testHoverPersists',
 100     'RenderedWebElementTest.canClickOnASuckerFishStyleMenu',
 101     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=416
 102     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePageAfterSwitching',
 103     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
 104 ]
 105 _OS_NEGATIVE_FILTER['linux'] = []
 106 _OS_NEGATIVE_FILTER['mac'] = [
 107     # https://code.google.com/p/chromedriver/issues/detail?id=26
 108     'AlertsTest.testAlertShouldNotAllowAdditionalCommandsIfDismissed',
 109     'AlertsTest.testShouldAllowUsersToDismissAnAlertManually',
 110     'FormHandlingTest.handleFormWithJavascriptAction',
 111     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=354
 112     'AlertsTest.testShouldAllowUsersToAcceptAnAlertInAFrame',
 113     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=375
 114     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
 115     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=416
 116     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
 117 ]
 118 _OS_NEGATIVE_FILTER['android:chrome'] = [
 119     'ChromeOptionsFunctionalTest.canStartChromeWithCustomOptions',
 120     'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrame',
 121     'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrameThatIsOutOfView',
 122     'ClickTest.testShouldOnlyFollowHrefOnce',
 123     'CombinedInputActionsTest.testCombiningShiftAndClickResultsInANewWindow',
 124     'ElementSelectingTest.testShouldBeAbleToToggleEnabledMultiSelectOption',
 125     'FrameSwitchingTest.testShouldBeAbleToClickInAFrameThatRewritesTopWindowLocation',
 126     'JavascriptEnabledDriverTest.testShouldBeAbleToClickALinkThatClosesAWindow',
 127     'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
 128     'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
 129     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
 130     'PageLoadingTest.testShouldFollowMetaRedirects',
 131     'PageLoadingTest.testShouldWaitForDocumentToBeLoaded',
 132     'PerformanceLogTypeTest.pageLoadShouldProducePerformanceLogEntries',
 133     'PerformanceLogTypeTest.shouldBeAbleToEnablePerformanceLog',
 134     'SelectElementHandlingTest.testShouldBePossibleToDeselectASingleOptionFromASelectWhichAllowsMultipleChoices',
 135     'SelectElementTest.shouldAllowOptionsToBeDeselectedByIndex',
 136     'SelectElementTest.shouldAllowOptionsToBeDeselectedByReturnedValue',
 137     'SelectElementTest.shouldAllowUserToDeselectAllWhenSelectSupportsMultipleSelections',
 138     'SelectElementTest.shouldAllowUserToDeselectOptionsByVisibleText',
 139     'SessionHandlingTest.callingAnyOperationAfterClosingTheLastWindowShouldThrowAnException',
 140     'SessionHandlingTest.callingQuitAfterClosingTheLastWindowIsANoOp',
 141     'TakesScreenshotTest.testCaptureToBase64',
 142     'TakesScreenshotTest.testSaveScreenshotAsFile',
 143     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
 144     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePageAfterSwitching',
 145     'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePageAfterSwitching',
 146
 147     # Alerts are not yet supported on Android.
 148     'AlertsTest.*',
 149
 150     # http://crbug.com/156390
 151     'DragAndDropTest.*',
 152
 153     # Scrolling touch events are not supported.
 154     'TouchScrollTest.*',
 155
 156     # These tests start multiple sessions, which is not supported on a single
 157     # Android device.
 158     "AvailableLogsTest.shouldBeAbleToEnableProfilerLog",
 159     "GetLogsTest.turningOffLogShouldMeanNoLogMessages",
 160     "SessionHandlingTest.callingAnyOperationAfterQuitShouldThrowAnException",
 161     "SessionHandlingTest.callingQuitMoreThanOnceOnASessionIsANoOp",
 162     # Test is written using local files; doesn't work on Android.
 163     'UploadTest.testFileUploading',
 164     # Not applicable on ChromeShell (doesn't support tabs).
 165     'WindowSwitchingTest.*',
 166
 167     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=441
 168     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistory',
 169     'PageLoadingTest.testShouldBeAbleToNavigateForwardsInTheBrowserHistory',
 170     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=528
 171     'PageLoadingTest.testShouldDoNothingIfThereIsNothingToGoBackTo',
 172
 173     # https://code.google.com/p/chromedriver/issues/detail?id=526
 174     'ElementFindingTest.testShouldNotBeAbleToLocateASingleElementOnABlankPage',
 175     'ElementFindingTest.testShouldBeAbleToFindAnElementByBooleanAttributeUsingShortCssSelector',
 176
 177     # https://code.google.com/p/chromedriver/issues/detail?id=527
 178     'ClickTest.testCanClickOnALinkThatContainsEmbeddedBlockElements',
 179
 180     # https://code.google.com/p/chromedriver/issues/detail?id=604
 181     'I18nTest.*',
 182 ]
 183 _OS_NEGATIVE_FILTER['android:chrome_stable'] = (
 184     _OS_NEGATIVE_FILTER['android:chrome'])
 185 _OS_NEGATIVE_FILTER['android:chrome_beta'] = (
 186     _OS_NEGATIVE_FILTER['android:chrome'])
 187 _OS_NEGATIVE_FILTER['android:chrome_shell'] = (
 188     _OS_NEGATIVE_FILTER['android:chrome'])
 189 _OS_NEGATIVE_FILTER['android:chromedriver_webview_shell'] = (
 190     _OS_NEGATIVE_FILTER['android:chrome'] + [
 191          # https://code.google.com/p/chromedriver/issues/detail?id=645
 192         'ClickScrollingTest.testShouldBeAbleToClickElementInAFrameThatIsOutOfView',
 193         'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInAFrameThatIsOutOfView',
 194         'FormHandlingTest.handleFormWithJavascriptAction',
 195         'TypingTest.testShouldReportKeyCodeOfArrowKeysUpDownEvents',
 196         'TypingTest.testShouldReportKeyCodeOfArrowKeys',
 197         'CombinedInputActionsTest.testCanClickOnLinks',
 198         'CombinedInputActionsTest.testCanClickOnLinksWithAnOffset',
 199         'CombinedInputActionsTest.testMouseMovementWorksWhenNavigatingToAnotherPage',
 200     ]
 201 )
 202
 203
 204 def _GetRevisionNegativeFilter(chrome_version):
 205   if chrome_version in _REVISION_NEGATIVE_FILTER:
 206     return _REVISION_NEGATIVE_FILTER[chrome_version]
 207   return _REVISION_NEGATIVE_FILTER['HEAD']
 208
 209
 210 def GetDisabledTestMatchers(operating_system, chrome_version):
 211   """Returns the list of disabled test matchers for the specific configuration.
 212
 213   Args:
 214     operating_system: The operating system, one of 'linux', 'mac', 'win', or
 215         'android'.
 216     chrome_version: Chrome version to test against, e.g., 'HEAD' or '26'.
 217
 218   Returns:
 219     List of disabled test matchers, which may contain '*' wildcards.
 220   """
 221   return (_OS_NEGATIVE_FILTER[operating_system] +
 222           _GetRevisionNegativeFilter(chrome_version))[:]
 223
 224
 225 def ApplyJavaTestFilter(operating_system, chrome_version, tests):
 226   """Applies the test filter to the given list of tests.
 227
 228   Args:
 229     operating_system: The operating system, one of 'linux', 'mac', 'win', or
 230         'android'.
 231     chrome_version: Chrome version to test against, e.g., 'HEAD' or '26'.
 232     test: list of test names to filter.
 233
 234   Returns:
 235     Set of passed test names.
 236   """
 237   filters = (_OS_NEGATIVE_FILTER[operating_system] +
 238              _GetRevisionNegativeFilter(chrome_version))
 239   passed = set(tests)
 240   for f in filters:
 241     passed.difference_update(set(t for t in tests if fnmatch.fnmatch(t, f)))
 242   return passed