src/chrome/test/chromedriver/test/test_expectations

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Test expectation list for WebDriver Java acceptance tests.
   6
   7 It is evaluated through Python.
   8 """
   9
  10 import fnmatch
  11
  12
  13 _REVISION_NEGATIVE_FILTER = {}
  14 _REVISION_NEGATIVE_FILTER['HEAD'] = [
  15     'AlertsTest.testIncludesAlertTextInUnhandledAlertException',
  16     'AlertsTest.testSettingTheValueOfAnAlertThrows',
  17     'AlertsTest.testShouldGetTextOfAlertOpenedInSetTimeout',
  18     'AlertsTest.testShouldHandleAlertOnPageUnload',
  19     'AlertsTest.testShouldHandleAlertOnWindowClose',
  20     'AlertsTest.testShouldNotHandleAlertInAnotherWindow',
  21     'AlertsTest.testSwitchingToMissingAlertInAClosedWindowThrows',
  22     'AuthenticatedPageLoadingTest.*',
  23     'AvailableLogsTest.driverLogShouldBeEnabledByDefault',
  24     'BasicMouseInterfaceTest.testMousePositionIsNotPreservedInActionsChain',
  25     'BasicMouseInterfaceTest.testMovingIntoAnImageEnclosedInALink',
  26     'BasicMouseInterfaceTest.testMovingMouseBackAndForthPastViewPort',
  27     'BasicMouseInterfaceTest.testMovingMousePastViewPort',
  28     'ChromeOptionsFunctionalTest.canStartChromeWithCustomOptions',
  29     'ClickScrollingTest.testShouldBeAbleToClickOnAnElementHiddenByDoubleOverflow',
  30     'ClickScrollingTest.testShouldNotBeAbleToClickElementThatIsOutOfViewInANonScrollableFrame',
  31     'ClickScrollingTest.testShouldNotScrollIfAlreadyScrolledAndElementIsInView',
  32     'ClickTest.testCanClickAnImageMapArea',
  33     'ClickTest.testShouldBeAbleToClickOnAnElementInFrameGreaterThanTwoViewports',
  34     'CombinedInputActionsTest.testClickingOnFormElements',
  35     'CoordinatesTest.testShouldScrollPageAndGetCoordinatesOfAnElementThatIsOutOfViewPort',
  36     'CorrectEventFiringTest.testShouldFireEventsInTheRightOrder',
  37     'CorrectEventFiringTest.testShouldFireFocusEventWhenClicking',
  38     'DragAndDropTest.testDragTooFar',
  39     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=374
  40     'ElementFindingTest.testShouldNotBeAbleToFindAnElementOnABlankPage',
  41     'ExecutingAsyncJavascriptTest.includesAlertTextInUnhandledAlertException',
  42     'ExecutingAsyncJavascriptTest.shouldCatchErrorsWithMessageAndStacktraceWhenExecutingInitialScript',
  43     'ExecutingAsyncJavascriptTest.throwsIfAlertHappensDuringScript',
  44     'ExecutingAsyncJavascriptTest.throwsIfAlertHappensDuringScriptWhichTimesOut',
  45     'ExecutingAsyncJavascriptTest.throwsIfScriptTriggersAlert',
  46     'ExecutingAsyncJavascriptTest.throwsIfScriptTriggersAlertWhichTimesOut',
  47     'ExecutingJavascriptTest.testShouldThrowExceptionIfExecutingOnNoPage',
  48     'ExecutingJavascriptTest.testShouldThrowAnExceptionWithMessageAndStacktraceWhenTheJavascriptIsBad',
  49     'FormHandlingTest.testShouldNotBeAbleToSubmitAFormThatDoesNotExist',
  50     'FrameSwitchingTest.testShouldNotBeAbleToDoAnythingTheFrameIsDeletedFromUnderUs',
  51     'I18nTest.testShouldBeAbleToActivateIMEEngine',
  52     # Broken because AddWebStorage.java is broken.
  53     'LocalStorageTest.*',
  54     'LocationContextTest.*',
  55     'MiscTest.testShouldBeAbleToGetTheSourceOfAnXmlDocument',
  56     'PageLoadingTest.shouldBeAbleToDisableAcceptOfInsecureSslCertsWithRequiredCapability',
  57     'PageLoadingTest.testShouldNotWaitIndefinitelyIfAnExternalResourceFailsToLoad',
  58     # PerformanceLoggingTest is for firefox only; "performance" log deprecated.
  59     'PerformanceLoggingTest.*',
  60     'ProxySettingTest.canConfigureProxyWithRequiredCapability',
  61     'ProxySettingTest.requiredProxyCapabilityShouldHavePriority',
  62     'RenderedWebElementTest.testShouldHandleNonIntegerPositionAndSize',
  63     # Broken because AddWebStorage.java is broken.
  64     'SessionStorageTest.*',
  65     'SqlDatabaseTest.*',
  66     'SvgDocumentTest.testClickOnSvgElement',
  67     'TakesScreenshotTest.testShouldCaptureScreenshot',
  68     'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePage',
  69     'TakesScreenshotTest.testShouldCaptureScreenshotWithLongX',
  70     'TakesScreenshotTest.testShouldCaptureScreenshotWithLongY',
  71     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLong',
  72     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLongX',
  73     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLongY',
  74     'TextHandlingTest.testShouldNotReturnLtrMarks',
  75     'TextPagesTest.testShouldBeAbleToLoadASimplePageOfText',
  76     'TextPagesTest.testShouldThrowExceptionWhenAddingCookieToAPageThatIsNotHtml',
  77     'TypingTest.testGenerateKeyPressEventEvenWhenElementPreventsDefault',
  78     'TypingTest.testNonPrintableCharactersShouldWorkWithContentEditableOrDesignModeSet',
  79     'TypingTest.testShouldBeAbleToTypeIntoContentEditableElementWithExistingValue',
  80     'TypingTest.testShouldNotTypeIntoElementsThatPreventKeyDownEvents',
  81     'TypingTest.testTypingIntoAnIFrameWithContentEditableOrDesignModeSet',
  82     'UnexpectedAlertBehaviorTest.*',
  83     'VisibilityTest.testElementHiddenByOverflowXIsNotVisible',
  84     'VisibilityTest.testElementHiddenByOverflowYIsNotVisible',
  85     'VisibilityTest.tooSmallAWindowWithOverflowHiddenIsNotAProblem',
  86     'WindowTest.*',
  87     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=528
  88     'PageLoadingTest.testShouldDoNothingIfThereIsNothingToGoBackTo',
  89 ]
  90 _REVISION_NEGATIVE_FILTER['31'] = _REVISION_NEGATIVE_FILTER['HEAD'] + [
  91     'I18nTest.*',  # This was fixed in a later version of 31 than we use.
  92 ]
  93 _REVISION_NEGATIVE_FILTER['30'] = _REVISION_NEGATIVE_FILTER['HEAD'] + [
  94     'I18nTest.*',
  95 ]
  96
  97
  98 _OS_NEGATIVE_FILTER = {}
  99 _OS_NEGATIVE_FILTER['win'] = [
 100     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=282
 101     'PageLoadingTest.testShouldNotHangIfDocumentOpenCallIsNeverFollowedByDocumentCloseCall',
 102     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=373
 103     'RenderedWebElementTest.testHoverPersists',
 104     'RenderedWebElementTest.canClickOnASuckerFishStyleMenu',
 105     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=416
 106     'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePageAfterSwitching',
 107     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
 108 ]
 109 _OS_NEGATIVE_FILTER['linux'] = []
 110 _OS_NEGATIVE_FILTER['mac'] = [
 111     # https://code.google.com/p/chromedriver/issues/detail?id=26
 112     'AlertsTest.testAlertShouldNotAllowAdditionalCommandsIfDismissed',
 113     'AlertsTest.testShouldAllowUsersToDismissAnAlertManually',
 114     'FormHandlingTest.handleFormWithJavascriptAction',
 115     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=354
 116     'AlertsTest.testShouldAllowUsersToAcceptAnAlertInAFrame',
 117     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=375
 118     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
 119 ]
 120 _OS_NEGATIVE_FILTER['android'] = [
 121     'ChromeOptionsFunctionalTest.canStartChromeWithCustomOptions',
 122     'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrame',
 123     'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrameThatIsOutOfView',
 124     'ClickTest.testShouldOnlyFollowHrefOnce',
 125     'CombinedInputActionsTest.testCombiningShiftAndClickResultsInANewWindow',
 126     'ElementSelectingTest.testShouldBeAbleToToggleEnabledMultiSelectOption',
 127     'FrameSwitchingTest.testShouldBeAbleToClickInAFrameThatRewritesTopWindowLocation',
 128     'JavascriptEnabledDriverTest.testShouldBeAbleToClickALinkThatClosesAWindow',
 129     'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
 130     'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
 131     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
 132     'PageLoadingTest.testShouldFollowMetaRedirects',
 133     'PageLoadingTest.testShouldWaitForDocumentToBeLoaded',
 134     'PerformanceLogTypeTest.pageLoadShouldProducePerformanceLogEntries',
 135     'PerformanceLogTypeTest.shouldBeAbleToEnablePerformanceLog',
 136     'SelectElementHandlingTest.testShouldBePossibleToDeselectASingleOptionFromASelectWhichAllowsMultipleChoices',
 137     'SelectElementTest.shouldAllowOptionsToBeDeselectedByIndex',
 138     'SelectElementTest.shouldAllowOptionsToBeDeselectedByReturnedValue',
 139     'SelectElementTest.shouldAllowUserToDeselectAllWhenSelectSupportsMultipleSelections',
 140     'SelectElementTest.shouldAllowUserToDeselectOptionsByVisibleText',
 141     'SessionHandlingTest.callingAnyOperationAfterClosingTheLastWindowShouldThrowAnException',
 142     'SessionHandlingTest.callingQuitAfterClosingTheLastWindowIsANoOp',
 143     'TakesScreenshotTest.testCaptureToBase64',
 144     'TakesScreenshotTest.testSaveScreenshotAsFile',
 145     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
 146     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePageAfterSwitching',
 147     'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePageAfterSwitching',
 148
 149     # Alerts are not yet supported on Android.
 150     'AlertsTest.*',
 151
 152     # http://crbug.com/156390
 153     'DragAndDropTest.*',
 154
 155     # Flick touch events are not yet implemented.
 156     'TouchFlickTest.*',
 157
 158     # Scrolling touch events are not supported.
 159     'TouchScrollTest.*',
 160
 161     # These tests start multiple sessions, which is not supported on a single
 162     # Android device.
 163     "AvailableLogsTest.shouldBeAbleToEnableProfilerLog",
 164     "GetLogsTest.turningOffLogShouldMeanNoLogMessages",
 165     "SessionHandlingTest.callingAnyOperationAfterQuitShouldThrowAnException",
 166     "SessionHandlingTest.callingQuitMoreThanOnceOnASessionIsANoOp",
 167     # Test is written using local files; doesn't work on Android.
 168     'UploadTest.testFileUploading',
 169     # Not applicable on Chromium Test Shell (doesn't support tabs).
 170     'WindowSwitchingTest.*',
 171
 172     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=441
 173     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistory',
 174     'PageLoadingTest.testShouldBeAbleToNavigateForwardsInTheBrowserHistory',
 175     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=528
 176     'PageLoadingTest.testShouldDoNothingIfThereIsNothingToGoBackTo',
 177
 178     # https://code.google.com/p/chromedriver/issues/detail?id=526
 179     'ElementFindingTest.testShouldNotBeAbleToLocateASingleElementOnABlankPage',
 180     'ElementFindingTest.testShouldBeAbleToFindAnElementByBooleanAttributeUsingShortCssSelector',
 181
 182     # https://code.google.com/p/chromedriver/issues/detail?id=527
 183     'ClickTest.testCanClickOnALinkThatContainsEmbeddedBlockElements',
 184
 185     # https://code.google.com/p/chromedriver/issues/detail?id=604
 186     'I18nTest.*',
 187 ]
 188
 189
 190 def _GetRevisionNegativeFilter(chrome_version):
 191   if chrome_version in _REVISION_NEGATIVE_FILTER:
 192     return _REVISION_NEGATIVE_FILTER[chrome_version]
 193   return _REVISION_NEGATIVE_FILTER['HEAD']
 194
 195
 196 def GetDisabledTestMatchers(operating_system, chrome_version):
 197   """Returns the list of disabled test matchers for the specific configuration.
 198
 199   Args:
 200     operating_system: The operating system, one of 'linux', 'mac', 'win', or
 201         'android'.
 202     chrome_version: Chrome version to test against, e.g., 'HEAD' or '26'.
 203
 204   Returns:
 205     List of disabled test matchers, which may contain '*' wildcards.
 206   """
 207   return (_OS_NEGATIVE_FILTER[operating_system] +
 208           _GetRevisionNegativeFilter(chrome_version))[:]
 209
 210
 211 def ApplyJavaTestFilter(operating_system, chrome_version, tests):
 212   """Applies the test filter to the given list of tests.
 213
 214   Args:
 215     operating_system: The operating system, one of 'linux', 'mac', 'win', or
 216         'android'.
 217     chrome_version: Chrome version to test against, e.g., 'HEAD' or '26'.
 218     test: list of test names to filter.
 219
 220   Returns:
 221     Set of passed test names.
 222   """
 223   filters = (_OS_NEGATIVE_FILTER[operating_system] +
 224              _GetRevisionNegativeFilter(chrome_version))
 225   passed = set(tests)
 226   for f in filters:
 227     passed.difference_update(set(t for t in tests if fnmatch.fnmatch(t, f)))
 228   return passed