1# Copyright 2024 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import argparse 6import json 7import pathlib 8from typing import Dict, List, Type 9from unittest import mock 10 11import hjson 12from tests import test_helper 13from tests.crossbench import mock_browser 14from tests.crossbench.base import BaseCliTestCase, SysExitTestException 15 16from crossbench import __version__ 17from crossbench.browsers.settings import Settings 18from crossbench.cli.cli import CrossBenchCLI 19from crossbench.cli.config.browser import BrowserConfig 20from crossbench.cli.config.browser_variants import BrowserVariantsConfig 21from crossbench.cli.config.driver import BrowserDriverType 22from crossbench.network.local_file_server import LocalFileNetwork 23from crossbench.probes import internal 24 25 26class CliSlowTestCase(BaseCliTestCase): 27 """Collection of slower tests that are not worth running 28 as part of the presubmit""" 29 30 def test_subcommand_help(self): 31 for benchmark_cls in CrossBenchCLI.BENCHMARKS: 32 subcommands = (benchmark_cls.NAME,) + benchmark_cls.aliases() 33 for subcommand in subcommands: 34 with self.assertRaises(SysExitTestException) as cm: 35 self.run_cli(subcommand, "--help") 36 self.assertEqual(cm.exception.exit_code, 0) 37 _, stdout, stderr = self.run_cli_output( 38 subcommand, "--help", raises=SysExitTestException) 39 self.assertFalse(stderr) 40 self.assertIn("--env-validation ENV_VALIDATION", stdout) 41 42 def test_subcommand_help_subcommand(self): 43 for benchmark_cls in CrossBenchCLI.BENCHMARKS: 44 subcommands = (benchmark_cls.NAME,) + benchmark_cls.aliases() 45 for subcommand in subcommands: 46 with self.assertRaises(SysExitTestException) as cm: 47 self.run_cli(subcommand, "help") 48 self.assertEqual(cm.exception.exit_code, 0) 49 _, stdout, stderr = self.run_cli_output( 50 subcommand, "help", raises=SysExitTestException) 51 self.assertFalse(stderr) 52 self.assertIn("--env-validation ENV_VALIDATION", stdout) 53 54 def test_subcommand_describe_subcommand(self): 55 for benchmark_cls in CrossBenchCLI.BENCHMARKS: 56 subcommands = (benchmark_cls.NAME,) + benchmark_cls.aliases() 57 for subcommand in subcommands: 58 with self.assertRaises(SysExitTestException) as cm: 59 self.run_cli(subcommand, "describe") 60 self.assertEqual(cm.exception.exit_code, 0) 61 _, stdout, stderr = self.run_cli_output( 62 subcommand, "describe", raises=SysExitTestException) 63 output = stderr + stdout 64 self.assertIn("See `describe benchmark ", output) 65 66 def test_browser_identifiers(self): 67 browsers: Dict[str, Type[mock_browser.MockBrowser]] = { 68 "chrome": mock_browser.MockChromeStable, 69 "chrome-stable": mock_browser.MockChromeStable, 70 "chr-stable": mock_browser.MockChromeStable, 71 "chrome-beta": mock_browser.MockChromeBeta, 72 "chr-beta": mock_browser.MockChromeBeta, 73 "chrome-dev": mock_browser.MockChromeDev, 74 "edge": mock_browser.MockEdgeStable, 75 "edge-stable": mock_browser.MockEdgeStable, 76 "edge-beta": mock_browser.MockEdgeBeta, 77 "edge-dev": mock_browser.MockEdgeDev, 78 "ff": mock_browser.MockFirefox, 79 "firefox": mock_browser.MockFirefox, 80 "firefox-dev": mock_browser.MockFirefoxDeveloperEdition, 81 "firefox-developer-edition": mock_browser.MockFirefoxDeveloperEdition, 82 "ff-dev": mock_browser.MockFirefoxDeveloperEdition, 83 "firefox-nightly": mock_browser.MockFirefoxNightly, 84 "ff-nightly": mock_browser.MockFirefoxNightly, 85 "ff-trunk": mock_browser.MockFirefoxNightly, 86 } 87 if not self.platform.is_linux: 88 browsers["chr-canary"] = mock_browser.MockChromeCanary 89 browsers["chrome-canary"] = mock_browser.MockChromeCanary 90 browsers["edge-canary"] = mock_browser.MockEdgeCanary 91 if self.platform.is_macos: 92 browsers.update({ 93 "safari": mock_browser.MockSafari, 94 "sf": mock_browser.MockSafari, 95 "safari-technology-preview": mock_browser.MockSafariTechnologyPreview, 96 "sf-tp": mock_browser.MockSafariTechnologyPreview, 97 "tp": mock_browser.MockSafariTechnologyPreview, 98 }) 99 100 for identifier, browser_cls in browsers.items(): 101 out_dir = self.out_dir / identifier 102 self.assertFalse(out_dir.exists()) 103 with mock.patch.object( 104 BrowserVariantsConfig, "get_browser_cls", 105 return_value=browser_cls) as get_browser_cls: 106 url = "http://test.com" 107 self.run_cli("loading", f"--browser={identifier}", f"--urls={url}", 108 "--env-validation=skip", f"--out-dir={out_dir}") 109 self.assertTrue(out_dir.exists()) 110 get_browser_cls.assert_called_once() 111 result_files = list( 112 out_dir.glob(f"**/{internal.ResultsSummaryProbe.NAME}.json")) 113 result_file = result_files[1] 114 with result_file.open(encoding="utf-8") as f: 115 results = json.load(f) 116 self.assertEqual(results["browser"]["version"], browser_cls.VERSION) 117 self.assertIn("test.com", results["stories"]) 118 119 def test_config_file_with_network(self): 120 local_server_path = pathlib.Path("custom/server") 121 local_server_path.mkdir(parents=True) 122 self.fs.create_file(local_server_path / "index.html", st_size=100) 123 config_file = pathlib.Path("/config.hjson") 124 config_data = { 125 "probes": {}, 126 "env": {}, 127 "browsers": {}, 128 "network": str(local_server_path), 129 } 130 with config_file.open("w", encoding="utf-8") as f: 131 hjson.dump(config_data, f) 132 133 browsers = [] 134 135 def get_browser(self, args: argparse.Namespace): 136 session = Settings( 137 platform=self.platform, network=args.network.create(self.platform)) 138 browsers = [ 139 mock_browser.MockChromeDev("dev", settings=session), 140 ] 141 return browsers 142 143 with mock.patch.object(CrossBenchCLI, "_get_browsers", get_browser): 144 url = "http://test.com" 145 self.run_cli("loading", f"--config={config_file}", f"--urls={url}", 146 "--env-validation=skip") 147 for browser in browsers: 148 self.assertListEqual([url], browser.url_list[self.SPLASH_URLS_LEN:]) 149 assert isinstance(browser.network, LocalFileNetwork) 150 network: LocalFileNetwork = browser.network 151 self.assertFalse(network.is_live) 152 self.assertEqual(network.path, local_server_path) 153 154 def test_multiple_browser_compatible_flags(self): 155 mock_browsers: List[Type[mock_browser.MockBrowser]] = [ 156 mock_browser.MockChromeStable, 157 mock_browser.MockFirefox, 158 mock_browser.MockChromeDev, 159 ] 160 161 def mock_get_browser_cls(browser_config: BrowserConfig): 162 self.assertEqual(browser_config.driver.type, BrowserDriverType.WEB_DRIVER) 163 for mock_browser_cls in mock_browsers: 164 if mock_browser_cls.mock_app_path() == browser_config.path: 165 return mock_browser_cls 166 raise ValueError("Unknown browser path") 167 168 for chrome_flag in ("--js-flags=--no-opt", "--enable-features=Foo", 169 "--disable-features=bar"): 170 # Fail for chrome flags for non-chrome browser 171 with self.assertRaises(argparse.ArgumentTypeError), mock.patch.object( 172 BrowserVariantsConfig, 173 "get_browser_cls", 174 side_effect=mock_get_browser_cls): 175 self.run_cli("loading", "--urls=http://test.com", 176 "--env-validation=skip", "--throw", "--browser=firefox", 177 chrome_flag) 178 # Fail for mixed browsers and chrome flags 179 with self.assertRaises(argparse.ArgumentTypeError), mock.patch.object( 180 BrowserVariantsConfig, 181 "get_browser_cls", 182 side_effect=mock_get_browser_cls): 183 self.run_cli("loading", "--urls=http://test.com", 184 "--env-validation=skip", "--throw", "--browser=chrome", 185 "--browser=firefox", chrome_flag) 186 with self.assertRaises(argparse.ArgumentTypeError), mock.patch.object( 187 BrowserVariantsConfig, 188 "get_browser_cls", 189 side_effect=mock_get_browser_cls): 190 self.run_cli("loading", "--urls=http://test.com", 191 "--env-validation=skip", "--throw", "--browser=chrome", 192 "--browser=firefox", "--", chrome_flag) 193 # Flags for the same type are allowed. 194 with self.patch_get_browser(): 195 self.run_cli("loading", "--urls=http://test.com", "--env-validation=skip", 196 "--throw", "--browser=chrome", "--browser=chrome-dev", "--", 197 "--js-flags=--no-opt") 198 199if __name__ == "__main__": 200 test_helper.run_pytest(__file__) 201