1# Copyright 2013 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4# 5# This file lists the pyauto tests that run as a part of the functional test 6# suite. 7# 8# Tests can be enabled on a per-platform basis. Tests not listed here will 9# not be run. 10# 11# Within each platform, tests are specified in up to 3 groups: (1) enabled 12# tests; (2) tests that are permanently disabled because they do not apply 13# to the given platform and hence should not be run; and (3) disabled tests 14# that need to be investigated/fixed. Tests are listed alphabetically within 15# each group. 16# 17# Test names can be specified in any of the following ways: 18# 1. as a module, in which case all tests in that module will be run 19# example: 'test_basic' 20# 2. or as a test case, in which case all tests in that test case will be run 21# example: 'test_basic.SimpleTest' 22# 3. or as an individual test 23# example: 'test_basic.SimpleTest.testCanOpenGoogle' 24# 25# Tests beginning with '-' will be excluded. This can be used to enforce 26# exclusions for a particular platform. 27# Names beginning with '@' will be treated as suite names and will be 28# recursively expanded. 29 30{ 31 # This suite gets run on 'Google Chrome' builds. 32 'FULL': { 33 'all': [ 34 'about_plugins_ui.AboutPluginsUITest', 35 'about_plugins_ui.ChromeAboutPluginsUITest', 36 'apptest', 37 'autofill', 38 'codesign', 39 'crash_reporter', 40 'execute_javascript', 41 'extensions', 42 'fullscreen_mouselock', 43 'gtalk.test_basic', 44 'infobars', 45 'media.media_basic_playback', 46 'multiprofile', 47 'nacl_sdk', 48 'ntp', 49 'omnibox', 50 'passwords', 51 'prefs', 52 'prefs_ui', 53 'pyauto_webdriver', 54 'search_engines', 55 'special_tabs', 56 'test_pyauto', 57 58 # =========================== 59 # Permanently-disabled tests. 60 # =========================== 61 62 # ================================================== 63 # Disabled tests that need to be investigated/fixed. 64 # ================================================== 65 # crbug.com/145006 66 '-about_plugins_ui.ChromeAboutPluginsUITest.testEnableAndDisableFlashPlugin', 67 # crbug.com/100365 68 '-autofill.AutofillTest.testDisplayLineItemForEntriesWithNoCCNum', 69 # crbug.com/171828 70 '-autofill.AutofillTest.testNoDuplicatePhoneNumsInPrefs', 71 # The source is behind. Waiting for dev to automate the update. 72 # crbug.com/109160 73 '-execute_javascript.ExecuteJavascriptTest.testExecuteJavascriptInExtension', 74 # crbug.com/123396 75 '-fullscreen_mouselock.FullscreenMouselockTest.testPatternsForFSAndML', 76 # crbug.com/132665 77 '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSExitWhenNavBackToPrevPage', 78 '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSExitWhenNavToNewPage', 79 '-fullscreen_mouselock.FullscreenMouselockTest.testMLExitWhenNavBackToPrevPage', 80 '-fullscreen_mouselock.FullscreenMouselockTest.testMLExitWhenNavToNewPage', 81 # crbug.com/140460 82 '-fullscreen_mouselock.FullscreenMouselockTest.testNoTabFSExitWhenJSExitMouseLock', 83 # crosbug.com/136875 84 '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSDoesNotExitForAnchorLinks', 85 # crbug.com/179263 86 '-gtalk.test_basic.BasicTest.testCurrentVersion', 87 '-gtalk.test_basic.BasicTest.testRCVersion', 88 # crbug.com/165796 89 '-infobars.OneClickInfobarTest', 90 # crbug.com/131874 91 '-infobars.OneClickInfobarTest.testNoOneClickInfobarAfterCancel', 92 # crbug.com/133315 93 '-infobars.OneClickInfobarTest.testDisplayOneClickInfobarAfterDismiss', 94 # Mysteriously broken? 95 # crbug.com/138857 96 '-multiprofile.MultiprofileTest.testMakeSearchEngineDefaultInMultiprofile', 97 # crbug.com/179268 98 '-ntp.NTPTest.testCannotUninstallWebStore', 99 '-ntp.NTPTest.testGetAppsInNewProfile', 100 '-ntp.NTPTest.testGetAppsWhenInstallApp', 101 '-ntp.NTPTest.testGetAppsWhenInstallNonApps', 102 '-ntp.NTPTest.testUninstallApp', 103 # crbug.com/143308 104 '-omnibox.OmniboxLiveTest.testGoogleSearch', 105 # crbug.com/71715 106 '-omnibox.OmniboxTest.testHistoryResult', 107 # crbug.com/123019 108 '-omnibox.OmniboxTest.testAutoCompleteForNonAsciiSearch', 109 # crbug.com/137041 110 '-omnibox.OmniboxTest.testDifferentTypesOfResults', 111 # crbug.com/162341 112 '-prefs.PrefsTest.testAllowSelectedGeoTracking', 113 '-prefs.PrefsTest.testDismissedInfobarSavesNoEntry', 114 '-prefs.PrefsTest.testGeolocationBlockedWhenTrackingDenied', 115 '-prefs.PrefsTest.testGeolocationPref', 116 # crbug.com/85600 117 '-prefs.PrefsTest.testNavigationStateOnSessionRestore', 118 # crbug.com/157271 119 '-prefs_ui.BasicSettingsUITest.testCancelStartupURLSetting', 120 '-prefs_ui.BasicSettingsUITest.testSetStartupPages', 121 '-prefs_ui.BasicSettingsUITest.testUseCurrentPagesForStartup', 122 # crbug.com/162341 123 '-prefs_ui.PrefsUITest.testBehaviorValueCorrectlyDisplayed', 124 # crbug.com/112051 125 '-prefs_ui.PrefsUITest.testChangeExceptionBehaviorUI', 126 # crbug.com/157271 127 '-prefs_ui.PrefsUITest.testDeleteExceptionUI', 128 # crbug.com/151973 129 '-prefs_ui.PrefsUITest.testInitialLineEntryInIncognitoUI', 130 # crbug.com/162341 131 '-prefs_ui.PrefsUITest.testLocationSettingOptionsUI', 132 '-prefs_ui.PrefsUITest.testNoInitialLineEntryInUI', 133 # crbug.com/132285 134 '-special_tabs.SpecialTabsTest.testSpecialURLTabs', 135 # crbug.com/168081 136 '-ntp.NTPTest.testCloseOneTab', 137 ], 138 139 'win': [ 140 'gpu', 141 # =========================== 142 # Permanently-disabled tests. 143 # =========================== 144 145 # ================================================== 146 # Disabled tests that need to be investigated/fixed. 147 # ================================================== 148 # crbug.com/105948 149 '-autofill.AutofillTest.testPostalCodeAndStateLabelsBasedOnCountry', 150 # crbug.com/111289 151 '-extensions.ExtensionsTest.testAllowAccessFileURLs', 152 # crbug.com/113090 153 '-extensions.ExtensionsTest.testAllowIncognitoExtension', 154 # crbug.com/171490 155 '-passwords.PasswordTest.testPasswdInfoNotStoredWhenAutocompleteOff', 156 # crbug.com/117569 157 '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows', 158 # crbug.com/98526 159 '-pyauto_webdriver.PyAutoWebDriverTest.testCanConnectToRestartedBrowser', 160 '-pyauto_webdriver.PyAutoWebDriverTest.testTypeIntoTextBox', 161 ], 162 163 'mac': [ 164 # =========================== 165 # Permanently-disabled tests. 166 # =========================== 167 168 # ================================================== 169 # Disabled tests that need to be investigated/fixed. 170 # ================================================== 171 # Keychain popups make autofill/password tests difficult: crbug.com/49378 172 '-prefs_ui.PrefsUITest.testSetPasswordAndDelete', 173 # codesign tests should run *after* signing. crbug.com/50481 174 '-codesign', 175 # crbug.com/124922 176 '-fullscreen_mouselock.FullscreenMouselockTest.testMouseLockExitWhenBrowserLoseFocus', 177 # crbug.com/125989 178 '-fullscreen_mouselock.FullscreenMouselockTest.testMouseLockExitWhenAlertDialogShow', 179 # Fails on chrome-mac-10_7-qa only: crbug.com/124886 180 '-fullscreen_mouselock.FullscreenMouselockTest.testPrefsForFullscreenExit', 181 '-fullscreen_mouselock.FullscreenMouselockTest.testNoMouseLockWhenCancelFS', 182 # crbug.com/121484 183 '-multiprofile.MultiprofileTest.test20NewProfiles', 184 '-ntp.NTPTest.testLaunchAppNewWindow', # crbug.com/79812 185 # crbug.com/70437 186 '-omnibox.OmniboxTest.testHistoryResult', 187 # crbug.com/91617 188 '-omnibox.OmniboxTest.testContentHistory', 189 # Keychain popups make autofill/password tests difficult: crbug.com/49378 190 '-passwords', 191 # crbug.com/69619 192 '-search_engines.SearchEnginesTest.testDiscoverSearchEngine', 193 # crbug.com/98526 194 '-pyauto_webdriver.PyAutoWebDriverTest.testCanConnectToRestartedBrowser', 195 '-pyauto_webdriver.PyAutoWebDriverTest.testTypeIntoTextBox', 196 ], 197 198 'linux': [ 199 'test_clean_exit', 200 201 # =========================== 202 # Permanently-disabled tests. 203 # =========================== 204 # System password manager obstructs password automation. 205 '-passwords', 206 207 # ================================================== 208 # Disabled tests that need to be investigated/fixed. 209 # ================================================== 210 # crbug.com/111289 211 '-extensions.ExtensionsTest.testAllowAccessFileURLs', 212 # crbug.com/91033 213 '-omnibox.OmniboxTest.testOmniboxSearchHistory', 214 ], 215 216 'chromeos': [ 217 'chromeos_basic', 218 'chromeos_browser', 219 'chromeos_crosh', 220 'chromeos_power', 221 'chromeos_prefs', 222 'chromeos_security', 223 'chromeos_time', 224 'secure_shell', 225 'youtube', 226 227 # =========================== 228 # Permanently-disabled tests. 229 # =========================== 230 # No codesign verification on ChromeOS. 231 '-codesign', 232 # Sync is already signed in with the login account. 233 # So one-click infobar tests do tno apply to chromeos. 234 '-infobars.OneClickInfobarTest', 235 # Multi-profile doesn't apply to chromeos yet. 236 '-multiprofile', 237 '-ntp.NTPTest.testDifferentProfileNotAppearInMostVisited', 238 # No NaCl support on ChromeOS. 239 '-nacl_sdk', 240 241 # ================================================== 242 # Disabled tests that need to be investigated/fixed. 243 # ================================================== 244 # crbug.com/132337 245 '-autofill.AutofillTest.testTabOrderForEditAddress', 246 # crosbug.com/19556 247 '-extensions.ExtensionsTest.testAllowAccessFileURLs', 248 '-extensions.ExtensionsTest.testAllowIncognitoExtension', 249 '-extensions.ExtensionsTest.testDisableEnableExtension', 250 # crbug.com/134593 251 '-gtalk.test_basic.BasicTest.testCurrentVersion', 252 '-gtalk.test_basic.BasicTest.testRCVersion', 253 # crosbug.com/24496 254 '-infobars.InfobarTest.testPluginCrashForMultiTabs', 255 # crbug.com/109035 256 '-infobars.InfobarTest.testPluginCrashInfobar', 257 # crosbug.com/14256 258 '-ntp.NTPTest.testLaunchAppFullScreen', 259 # Content history broken in omnibox. crosbug.com/14416 260 '-omnibox.OmniboxTest.testContentHistory', 261 # crbug.com/91033 262 '-omnibox.OmniboxTest.testOmniboxSearchHistory', 263 # crosbug.com/19760 264 '-passwords.PasswordTest.testClearFetchedCredForNewUserName', 265 '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows', 266 # onunload popups get created in the same window on chromeos 267 # Session restore not working with PyAuto. crosbug.com/12648 268 '-prefs.PrefsTest.testNavigationStateOnSessionRestore', 269 '-prefs.PrefsTest.testSessionRestoreURLs', 270 '-prefs.PrefsTest.testSessionRestore', 271 # Deal with i18n chars. crosbug.com/12639 272 '-omnibox.OmniboxTest.testCrazyFilenames', 273 # crosbug.com/20025 274 '-chromeos_browser.ChromeosBrowserTest.testFullScreen', 275 # Chrome driver does not work in Chrome OS. 276 # crosbug.com/19556 277 '-prefs_ui', 278 '-prefs.PrefsTest.testGeolocationBlockedWhenTrackingDenied', 279 '-prefs.PrefsTest.testGeolocationPref', 280 '-pyauto_webdriver', 281 ], 282 }, 283 284 # Performance tests. 285 'PERFORMANCE': { 286 'all': [ 287 'perf', 288 289 # ================================================== 290 # Disabled tests that need to be investigated/fixed. 291 # ================================================== 292 '-perf.HTML5BenchmarkTest', # crbug.com/134476 293 '-perf.LiveWebappLoadTest.testNewTabGmail', # crbug.com/136554 294 '-perf.ScrollTest.testGmailScroll', # crbug.com/136554 295 '-perf.WebGLTest.testWebGLField', # crbug.com/132797 296 297 # =========================== 298 # Permanently-disabled tests. 299 # =========================== 300 # Invoked outside of the pyauto_tests framework. 301 '-perf.BenchmarkPerfTest.testSpaceport', 302 '-perf.PopularSitesScrollTest.test2012Q3', 303 ], 304 'win': [ 305 # ================================================== 306 # Disabled tests that need to be investigated/fixed. 307 # ================================================== 308 '-perf.GPUPerfTest', # Fails. Discuss with prachij@. 309 '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 310 '-perf.LiveGamePerfTest', # Requires linux /proc/stat. 311 '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. 312 313 # =========================== 314 # Permanently-disabled tests. 315 # =========================== 316 '-perf.MemoryTest', # Designed only for ChromeOS. 317 '-perf.NetflixPerfTest', # Designed only for ChromeOS. 318 ], 319 'mac': [ 320 # ================================================== 321 # Disabled tests that need to be investigated/fixed. 322 # ================================================== 323 '-perf.GPUPerfTest', # Fails. Discuss with prachij@. 324 '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 325 '-perf.LiveGamePerfTest', # Requires linux /proc/stat. 326 '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. 327 328 # =========================== 329 # Permanently-disabled tests. 330 # =========================== 331 '-perf.MemoryTest', # Designed only for ChromeOS. 332 '-perf.NetflixPerfTest', # Designed only for ChromeOS. 333 ], 334 'linux': [ 335 # ================================================== 336 # Disabled tests that need to be investigated/fixed. 337 # ================================================== 338 '-perf.GPUPerfTest', # Fails. Discuss with prachij@. 339 '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 340 '-perf.WebGLTest', # May not render WebGL; need to verify on QA bot. 341 '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. 342 343 # =========================== 344 # Permanently-disabled tests. 345 # =========================== 346 '-perf.MemoryTest', # Designed only for ChromeOS. 347 '-perf.NetflixPerfTest', # Designed only for ChromeOS. 348 ], 349 'chromeos': [ 350 # ================================================== 351 # Disabled tests that need to be investigated/fixed. 352 # ================================================== 353 '-perf.NetflixPerfTest', # crosbug.com/32320. 354 '-perf.PageCyclerNetSimTest', # Investigate Web Page Replay on ChromeOS. 355 '-perf.YoutubePerfTest', # crbug.com/233706 356 357 # =========================== 358 # Permanently-disabled tests. 359 # =========================== 360 '-perf.GPUPerfTest', # Designed only for Chrome desktop. 361 ], 362 }, 363 364 'PERFORMANCE_LAB': { 365 'all': [ 366 'perf.BenchmarkPerfTest.testV8BenchmarkSuite', 367 'perf.FlashTest.testFlashGaming', 368 'perf.ScrollTest.testGooglePlusScroll', 369 'perf.TabPerfTest.test20Tabs', 370 'perf.WebGLTest.testWebGLSpaceRocks', 371 ], 372 }, 373 374 'EMPTY': { 375 }, 376 377 # ChromeOS flash tests. 378 'CHROMEOS_FLASH': { 379 'chromeos': [ 380 'flash', 381 ], 382 }, 383 384 # ChromeOS volume tests. 385 'CHROMEOS_VOLUME': { 386 'chromeos': [ 387 'chromeos_volume', 388 ], 389 }, 390 391 # ChromeOS Accessibility tests. 392 'CHROMEOS_ACCESSIBILITY': { 393 'chromeos': [ 394 'chromeos_accessibility', 395 ], 396 }, 397 398 # PGO performance suite. 399 'PGO': { 400 'chromeos': [ 401 'perf.BenchmarkPerfTest.testV8BenchmarkSuite', 402 'perf.PageCyclerTest', 403 'perf.ScrollTest.testGooglePlusScroll', 404 ], 405 }, 406 407 # Subset of ChromeOS performance tests for seaboard. 408 'CHROMEOS_PERF_SEABOARD': { 409 'chromeos': [ 410 'perf.ScrollTest.testBlankPageScroll', 411 'perf.ScrollTest.testGooglePlusScroll', 412 'perf.ScrollTest.testTextScroll', 413 ], 414 }, 415 416 # HTML5 media performance tests. 417 'AV_PERF': { 418 'linux': [ 419 'media.audio_latency_perf', 420 'media.audio_playback_perf', 421 'media.media_constrained_network_perf', 422 'media.media_scrub_perf', 423 'media.media_seek_perf', 424 'media.media_stat_perf', 425 'media.mixed_audio_latency_perf', 426 ], 427 'win': [ 428 'media.audio_latency_perf', 429 'media.audio_playback_perf', 430 'media.media_constrained_network_perf', 431 'media.media_scrub_perf', 432 'media.media_seek_perf', 433 'media.media_stat_perf', 434 'media.mixed_audio_latency_perf', 435 ], 436 }, 437 438 # Trace event tests. 439 'TRACING': { 440 'all': [ 441 'tracing.tracing_smoke_test', 442 ], 443 }, 444 445 # Chromoting tests. 446 'CHROMOTING': { 447 'all': [ 448 'chromoting.auth', 449 'chromoting.it2me_basic', 450 'chromoting.me2me_connect', 451 'chromoting.me2me_enable', 452 ], 453 'linux': [ 454 # TODO(yihongg): Me2me test against Linux is not working yet. 455 '-chromoting.me2me_connect', 456 '-chromoting.me2me_enable', 457 ], 458 'chromeos': [ 459 # ChromeOS doesn't yet support the chromoting host. 460 '-chromoting.auth', 461 '-chromoting.it2me_basic', 462 '-chromoting.me2me_connect', 463 '-chromoting.me2me_enable', 464 ], 465 }, 466 467 # Pyauto functional tests running on coverage bots. 468 'CODE_COVERAGE': { 469 'all': [ 470 '@FULL', 471 ], 472 'linux': [ 473 # These tests fail on coverage bots. Disabling for now. 474 '-test_clean_exit', 475 '-about_plugins_ui', 476 '-autofill', 477 '-pyauto_webdriver', 478 '-extensions', 479 '-flash', 480 '-fullscreen_mouselock', 481 '-gtalk.test_basic', 482 '-infobars', 483 '-multiprofile', 484 '-prefs', 485 '-prefs_ui', 486 '-apptest', 487 '-plugins', 488 '-omnibox', 489 '-special_tabs', 490 '-ntp.NTPTest.testLaunchAppFullScreen', 491 '-media.media_basic_playback.MediaBasicPlaybackTest.testBasicPlaybackMatrix', 492 '-search_engines.SearchEnginesTest.testDiscoverSearchEngine', 493 '-ntp.NTPTest.testUninstallApp', 494 '-ntp.NTPTest.testLaunchAppWithDefaultSettings', 495 '-ntp.NTPTest.testLaunchAppRegularTab', 496 '-ntp.NTPTest.testLaunchAppPinnedTab', 497 '-ntp.NTPTest.testGetAppsWhenInstallApp', 498 '-ntp.NTPTest.testLaunchAppNewWindow', 499 ], 500 }, 501} 502