• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Test expectation list for WebDriver Java acceptance tests.
6
7It is evaluated through Python.
8"""
9
10import fnmatch
11
12
13_REVISION_NEGATIVE_FILTER = {}
14_REVISION_NEGATIVE_FILTER['HEAD'] = [
15    'AlertsTest.testIncludesAlertTextInUnhandledAlertException',
16    'AlertsTest.testSettingTheValueOfAnAlertThrows',
17    'AlertsTest.testShouldGetTextOfAlertOpenedInSetTimeout',
18    'AlertsTest.testShouldHandleAlertOnPageUnload',
19    'AlertsTest.testShouldHandleAlertOnWindowClose',
20    'AlertsTest.testShouldNotHandleAlertInAnotherWindow',
21    'AlertsTest.testSwitchingToMissingAlertInAClosedWindowThrows',
22    'AuthenticatedPageLoadingTest.*',
23    'AvailableLogsTest.driverLogShouldBeEnabledByDefault',
24    'BasicMouseInterfaceTest.testMousePositionIsNotPreservedInActionsChain',
25    'BasicMouseInterfaceTest.testMovingIntoAnImageEnclosedInALink',
26    'BasicMouseInterfaceTest.testMovingMouseBackAndForthPastViewPort',
27    'BasicMouseInterfaceTest.testMovingMousePastViewPort',
28    'ChromeOptionsFunctionalTest.canStartChromeWithCustomOptions',
29    'ClickScrollingTest.testShouldBeAbleToClickOnAnElementHiddenByDoubleOverflow',
30    'ClickScrollingTest.testShouldNotBeAbleToClickElementThatIsOutOfViewInANonScrollableFrame',
31    'ClickScrollingTest.testShouldNotScrollIfAlreadyScrolledAndElementIsInView',
32    'ClickTest.testCanClickAnImageMapArea',
33    'ClickTest.testShouldBeAbleToClickOnAnElementInFrameGreaterThanTwoViewports',
34    'CombinedInputActionsTest.testClickingOnFormElements',
35    'CoordinatesTest.testShouldScrollPageAndGetCoordinatesOfAnElementThatIsOutOfViewPort',
36    'CorrectEventFiringTest.testShouldFireEventsInTheRightOrder',
37    'CorrectEventFiringTest.testShouldFireFocusEventWhenClicking',
38    'DragAndDropTest.testDragTooFar',
39    # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=374
40    'ElementFindingTest.testShouldNotBeAbleToFindAnElementOnABlankPage',
41    'ExecutingAsyncJavascriptTest.includesAlertTextInUnhandledAlertException',
42    'ExecutingAsyncJavascriptTest.shouldCatchErrorsWithMessageAndStacktraceWhenExecutingInitialScript',
43    'ExecutingAsyncJavascriptTest.throwsIfAlertHappensDuringScript',
44    'ExecutingAsyncJavascriptTest.throwsIfAlertHappensDuringScriptWhichTimesOut',
45    'ExecutingAsyncJavascriptTest.throwsIfScriptTriggersAlert',
46    'ExecutingAsyncJavascriptTest.throwsIfScriptTriggersAlertWhichTimesOut',
47    'ExecutingJavascriptTest.testShouldThrowExceptionIfExecutingOnNoPage',
48    'ExecutingJavascriptTest.testShouldThrowAnExceptionWithMessageAndStacktraceWhenTheJavascriptIsBad',
49    'FormHandlingTest.testShouldNotBeAbleToSubmitAFormThatDoesNotExist',
50    'FrameSwitchingTest.testShouldNotBeAbleToDoAnythingTheFrameIsDeletedFromUnderUs',
51    'I18nTest.testShouldBeAbleToActivateIMEEngine',
52    # Broken because AddWebStorage.java is broken.
53    'LocalStorageTest.*',
54    'LocationContextTest.*',
55    'MiscTest.testShouldBeAbleToGetTheSourceOfAnXmlDocument',
56    'PageLoadingTest.shouldBeAbleToDisableAcceptOfInsecureSslCertsWithRequiredCapability',
57    'PageLoadingTest.testShouldNotWaitIndefinitelyIfAnExternalResourceFailsToLoad',
58    # PerformanceLoggingTest is for firefox only; "performance" log deprecated.
59    'PerformanceLoggingTest.*',
60    'ProxySettingTest.canConfigureProxyWithRequiredCapability',
61    'ProxySettingTest.requiredProxyCapabilityShouldHavePriority',
62    'RenderedWebElementTest.testShouldHandleNonIntegerPositionAndSize',
63    # Broken because AddWebStorage.java is broken.
64    'SessionStorageTest.*',
65    'SqlDatabaseTest.*',
66    'SvgDocumentTest.testClickOnSvgElement',
67    'TakesScreenshotTest.testShouldCaptureScreenshot',
68    'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePage',
69    'TakesScreenshotTest.testShouldCaptureScreenshotWithLongX',
70    'TakesScreenshotTest.testShouldCaptureScreenshotWithLongY',
71    'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLong',
72    'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLongX',
73    'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLongY',
74    'TextHandlingTest.testShouldNotReturnLtrMarks',
75    'TextPagesTest.testShouldBeAbleToLoadASimplePageOfText',
76    'TextPagesTest.testShouldThrowExceptionWhenAddingCookieToAPageThatIsNotHtml',
77    'TypingTest.testGenerateKeyPressEventEvenWhenElementPreventsDefault',
78    'TypingTest.testNonPrintableCharactersShouldWorkWithContentEditableOrDesignModeSet',
79    'TypingTest.testShouldBeAbleToTypeIntoContentEditableElementWithExistingValue',
80    'TypingTest.testShouldNotTypeIntoElementsThatPreventKeyDownEvents',
81    'TypingTest.testTypingIntoAnIFrameWithContentEditableOrDesignModeSet',
82    'UnexpectedAlertBehaviorTest.*',
83    'VisibilityTest.testElementHiddenByOverflowXIsNotVisible',
84    'VisibilityTest.testElementHiddenByOverflowYIsNotVisible',
85    'VisibilityTest.tooSmallAWindowWithOverflowHiddenIsNotAProblem',
86    'WindowTest.*',
87    # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=528
88    'PageLoadingTest.testShouldDoNothingIfThereIsNothingToGoBackTo',
89    # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=653
90    'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
91    'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistory',
92]
93_REVISION_NEGATIVE_FILTER['31'] = _REVISION_NEGATIVE_FILTER['HEAD'] + [
94    'I18nTest.*',  # This was fixed in a later version of 31 than we use.
95]
96_REVISION_NEGATIVE_FILTER['30'] = _REVISION_NEGATIVE_FILTER['HEAD'] + [
97    'I18nTest.*',
98]
99
100
101_OS_NEGATIVE_FILTER = {}
102_OS_NEGATIVE_FILTER['win'] = [
103    # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=282
104    'PageLoadingTest.testShouldNotHangIfDocumentOpenCallIsNeverFollowedByDocumentCloseCall',
105    # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=373
106    'RenderedWebElementTest.testHoverPersists',
107    'RenderedWebElementTest.canClickOnASuckerFishStyleMenu',
108    # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=416
109    'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePageAfterSwitching',
110    'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
111]
112_OS_NEGATIVE_FILTER['linux'] = []
113_OS_NEGATIVE_FILTER['mac'] = [
114    # https://code.google.com/p/chromedriver/issues/detail?id=26
115    'AlertsTest.testAlertShouldNotAllowAdditionalCommandsIfDismissed',
116    'AlertsTest.testShouldAllowUsersToDismissAnAlertManually',
117    'FormHandlingTest.handleFormWithJavascriptAction',
118    # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=354
119    'AlertsTest.testShouldAllowUsersToAcceptAnAlertInAFrame',
120    # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=375
121    'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
122]
123_OS_NEGATIVE_FILTER['android:chrome'] = [
124    'ChromeOptionsFunctionalTest.canStartChromeWithCustomOptions',
125    'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrame',
126    'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrameThatIsOutOfView',
127    'ClickTest.testShouldOnlyFollowHrefOnce',
128    'CombinedInputActionsTest.testCombiningShiftAndClickResultsInANewWindow',
129    'ElementSelectingTest.testShouldBeAbleToToggleEnabledMultiSelectOption',
130    'FrameSwitchingTest.testShouldBeAbleToClickInAFrameThatRewritesTopWindowLocation',
131    'JavascriptEnabledDriverTest.testShouldBeAbleToClickALinkThatClosesAWindow',
132    'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
133    'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
134    'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
135    'PageLoadingTest.testShouldFollowMetaRedirects',
136    'PageLoadingTest.testShouldWaitForDocumentToBeLoaded',
137    'PerformanceLogTypeTest.pageLoadShouldProducePerformanceLogEntries',
138    'PerformanceLogTypeTest.shouldBeAbleToEnablePerformanceLog',
139    'SelectElementHandlingTest.testShouldBePossibleToDeselectASingleOptionFromASelectWhichAllowsMultipleChoices',
140    'SelectElementTest.shouldAllowOptionsToBeDeselectedByIndex',
141    'SelectElementTest.shouldAllowOptionsToBeDeselectedByReturnedValue',
142    'SelectElementTest.shouldAllowUserToDeselectAllWhenSelectSupportsMultipleSelections',
143    'SelectElementTest.shouldAllowUserToDeselectOptionsByVisibleText',
144    'SessionHandlingTest.callingAnyOperationAfterClosingTheLastWindowShouldThrowAnException',
145    'SessionHandlingTest.callingQuitAfterClosingTheLastWindowIsANoOp',
146    'TakesScreenshotTest.testCaptureToBase64',
147    'TakesScreenshotTest.testSaveScreenshotAsFile',
148    'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
149    'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePageAfterSwitching',
150    'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePageAfterSwitching',
151
152    # Alerts are not yet supported on Android.
153    'AlertsTest.*',
154
155    # http://crbug.com/156390
156    'DragAndDropTest.*',
157
158    # Flick touch events are not yet implemented.
159    'TouchFlickTest.*',
160
161    # Scrolling touch events are not supported.
162    'TouchScrollTest.*',
163
164    # These tests start multiple sessions, which is not supported on a single
165    # Android device.
166    "AvailableLogsTest.shouldBeAbleToEnableProfilerLog",
167    "GetLogsTest.turningOffLogShouldMeanNoLogMessages",
168    "SessionHandlingTest.callingAnyOperationAfterQuitShouldThrowAnException",
169    "SessionHandlingTest.callingQuitMoreThanOnceOnASessionIsANoOp",
170    # Test is written using local files; doesn't work on Android.
171    'UploadTest.testFileUploading',
172    # Not applicable on Chromium Test Shell (doesn't support tabs).
173    'WindowSwitchingTest.*',
174
175    # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=441
176    'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistory',
177    'PageLoadingTest.testShouldBeAbleToNavigateForwardsInTheBrowserHistory',
178    # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=528
179    'PageLoadingTest.testShouldDoNothingIfThereIsNothingToGoBackTo',
180
181    # https://code.google.com/p/chromedriver/issues/detail?id=526
182    'ElementFindingTest.testShouldNotBeAbleToLocateASingleElementOnABlankPage',
183    'ElementFindingTest.testShouldBeAbleToFindAnElementByBooleanAttributeUsingShortCssSelector',
184
185    # https://code.google.com/p/chromedriver/issues/detail?id=527
186    'ClickTest.testCanClickOnALinkThatContainsEmbeddedBlockElements',
187
188    # https://code.google.com/p/chromedriver/issues/detail?id=604
189    'I18nTest.*',
190]
191_OS_NEGATIVE_FILTER['android:chrome_stable'] = (
192    _OS_NEGATIVE_FILTER['android:chrome'])
193_OS_NEGATIVE_FILTER['android:chrome_beta'] = (
194    _OS_NEGATIVE_FILTER['android:chrome'])
195_OS_NEGATIVE_FILTER['android:chromium_test_shell'] = (
196    _OS_NEGATIVE_FILTER['android:chrome'])
197_OS_NEGATIVE_FILTER['android:chromedriver_webview_shell'] = (
198    _OS_NEGATIVE_FILTER['android:chrome'] + [
199         # https://code.google.com/p/chromedriver/issues/detail?id=645
200        'ClickScrollingTest.testShouldBeAbleToClickElementInAFrameThatIsOutOfView',
201        'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInAFrameThatIsOutOfView',
202        'FormHandlingTest.handleFormWithJavascriptAction',
203        'TypingTest.testShouldReportKeyCodeOfArrowKeysUpDownEvents',
204        'TypingTest.testShouldReportKeyCodeOfArrowKeys',
205        'CombinedInputActionsTest.testCanClickOnLinks',
206        'CombinedInputActionsTest.testCanClickOnLinksWithAnOffset',
207        'CombinedInputActionsTest.testMouseMovementWorksWhenNavigatingToAnotherPage',
208    ]
209)
210
211
212def _GetRevisionNegativeFilter(chrome_version):
213  if chrome_version in _REVISION_NEGATIVE_FILTER:
214    return _REVISION_NEGATIVE_FILTER[chrome_version]
215  return _REVISION_NEGATIVE_FILTER['HEAD']
216
217
218def GetDisabledTestMatchers(operating_system, chrome_version):
219  """Returns the list of disabled test matchers for the specific configuration.
220
221  Args:
222    operating_system: The operating system, one of 'linux', 'mac', 'win', or
223        'android'.
224    chrome_version: Chrome version to test against, e.g., 'HEAD' or '26'.
225
226  Returns:
227    List of disabled test matchers, which may contain '*' wildcards.
228  """
229  return (_OS_NEGATIVE_FILTER[operating_system] +
230          _GetRevisionNegativeFilter(chrome_version))[:]
231
232
233def ApplyJavaTestFilter(operating_system, chrome_version, tests):
234  """Applies the test filter to the given list of tests.
235
236  Args:
237    operating_system: The operating system, one of 'linux', 'mac', 'win', or
238        'android'.
239    chrome_version: Chrome version to test against, e.g., 'HEAD' or '26'.
240    test: list of test names to filter.
241
242  Returns:
243    Set of passed test names.
244  """
245  filters = (_OS_NEGATIVE_FILTER[operating_system] +
246             _GetRevisionNegativeFilter(chrome_version))
247  passed = set(tests)
248  for f in filters:
249    passed.difference_update(set(t for t in tests if fnmatch.fnmatch(t, f)))
250  return passed
251