• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2024 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import argparse
6import json
7import pathlib
8import unittest
9from unittest import mock
10
11import hjson
12from tests import test_helper
13from tests.crossbench import mock_browser
14from tests.crossbench.base import BaseCliTestCase, SysExitTestException
15
16from crossbench import __version__, plt
17from crossbench.cli.config.browser import BrowserConfig
18from crossbench.cli.config.browser_variants import BrowserVariantsConfig
19from crossbench.env import HostEnvironmentConfig
20
21
22class FastCliTestCasePartA(BaseCliTestCase):
23  """These tests are run as part of the presubmit and should be
24  reasonably fast.
25  Slow tests run on the CQ are in CliSlowTestCase.
26
27  Keep FastCliTestCasePartA and FastCliTestCasePartB balanced for faster local
28  presubmit checks.
29  """
30
31  def test_invalid(self):
32    with self.assertRaises(SysExitTestException):
33      self.run_cli("unknown subcommand", "--invalid flag")
34
35  def test_describe_invalid_empty(self):
36    with self.assertRaises(SysExitTestException) as cm:
37      self.run_cli("describe", "")
38    self.assertEqual(cm.exception.exit_code, 0)
39    with self.assertRaises(SysExitTestException) as cm:
40      self.run_cli("describe", "", "--json")
41    self.assertEqual(cm.exception.exit_code, 0)
42
43    with self.assertRaises(SysExitTestException) as cm:
44      self.run_cli("describe", "--unknown")
45    self.assertEqual(cm.exception.exit_code, 0)
46    with self.assertRaises(SysExitTestException) as cm:
47      self.run_cli("describe", "--unknown", "--json")
48    self.assertEqual(cm.exception.exit_code, 0)
49
50  def test_describe_invalid_probe(self):
51    with self.assertRaises(SysExitTestException) as cm:
52      self.run_cli("describe", "probe", "unknown probe")
53    self.assertEqual(cm.exception.exit_code, 0)
54    with self.assertRaises(SysExitTestException) as cm:
55      self.run_cli("describe", "probe", "unknown probe", "--json")
56    self.assertEqual(cm.exception.exit_code, 0)
57
58  def test_describe_invalid_benchmark(self):
59    with self.assertRaises(SysExitTestException) as cm:
60      self.run_cli("describe", "benchmark", "unknown benchmark")
61    self.assertEqual(cm.exception.exit_code, 0)
62    with self.assertRaises(SysExitTestException) as cm:
63      self.run_cli("describe", "benchmark", "unknown benchmark", "--json")
64    self.assertEqual(cm.exception.exit_code, 0)
65
66  def test_describe_invalid_all(self):
67    with self.assertRaises(SysExitTestException) as cm:
68      self.run_cli("describe", "all", "unknown probe or benchmark")
69    self.assertEqual(cm.exception.exit_code, 0)
70    with self.assertRaises(SysExitTestException) as cm:
71      self.run_cli("describe", "--json", "all", "unknown probe or benchmark")
72    self.assertEqual(cm.exception.exit_code, 0)
73
74  def test_describe(self):
75    # Non-json output shouldn't fail
76    self.run_cli("describe")
77    self.run_cli("describe", "all")
78    _, stdout, stderr = self.run_cli_output("describe", "--json")
79    self.assertFalse(stderr)
80    data = json.loads(stdout)
81    self.assertIn("benchmarks", data)
82    self.assertIn("probes", data)
83    self.assertIsInstance(data["benchmarks"], dict)
84    self.assertIsInstance(data["probes"], dict)
85
86  def test_describe_benchmarks(self):
87    # Non-json output shouldn't fail
88    self.run_cli("describe", "benchmarks")
89    _, stdout, stderr = self.run_cli_output("describe", "--json", "benchmarks")
90    self.assertFalse(stderr)
91    data = json.loads(stdout)
92    self.assertNotIn("benchmarks", data)
93    self.assertNotIn("probes", data)
94    self.assertIsInstance(data, dict)
95    self.assertIn("loading", data)
96
97  def test_describe_probes(self):
98    # Non-json output shouldn't fail
99    self.run_cli("describe", "probes")
100    _, stdout, stderr = self.run_cli_output("describe", "--json", "probes")
101    self.assertFalse(stderr)
102    data = json.loads(stdout)
103    self.assertNotIn("benchmarks", data)
104    self.assertNotIn("probes", data)
105    self.assertIsInstance(data, dict)
106    self.assertIn("v8.log", data)
107
108  def test_describe_all(self):
109    self.run_cli("describe", "probes")
110    _, stdout, stderr = self.run_cli_output("describe", "all")
111    self.assertFalse(stderr)
112    self.assertIn("benchmarks", stdout)
113    self.assertIn("v8.log", stdout)
114    self.assertIn("speedometer", stdout)
115
116  def test_describe_all_filtered(self):
117    self.run_cli("describe", "probes")
118    _, stdout, stderr = self.run_cli_output("describe", "all", "v8.log")
119    self.assertFalse(stderr)
120    self.assertNotIn("benchmarks", stdout)
121    self.assertIn("v8.log", stdout)
122    self.assertNotIn("speedometer", stdout)
123
124  def test_describe_all_json(self):
125    self.run_cli("describe", "probes")
126    _, stdout, stderr = self.run_cli_output("describe", "--json", "all")
127    self.assertFalse(stderr)
128    data = json.loads(stdout)
129    self.assertIsInstance(data, dict)
130    self.assertIn("benchmarks", data)
131    self.assertIn("v8.log", data["probes"])
132
133  def test_describe_all_json_filtered(self):
134    self.run_cli("describe", "probes")
135    _, stdout, stderr = self.run_cli_output("describe", "--json", "all",
136                                            "v8.log")
137    self.assertFalse(stderr)
138    data = json.loads(stdout)
139    self.assertIsInstance(data, dict)
140    self.assertEqual(data["benchmarks"], {})
141    self.assertEqual(len(data["probes"]), 1)
142    self.assertIn("v8.log", data["probes"])
143
144  def test_help(self):
145    with self.assertRaises(SysExitTestException) as cm:
146      self.run_cli("--help")
147    self.assertEqual(cm.exception.exit_code, 0)
148    _, stdout, stderr = self.run_cli_output(
149        "--help", raises=SysExitTestException)
150    self.assertFalse(stderr)
151    self.assertIn("usage:", stdout)
152    self.assertIn("Subcommands:", stdout)
153    # Check for top-level option:
154    self.assertIn("--no-color", stdout)
155    self.assertIn("Disable colored output", stdout)
156    self.assertIn("Available Probes for all Benchmarks:", stdout)
157
158  def test_help_subcommand(self):
159    with self.assertRaises(SysExitTestException) as cm:
160      self.run_cli("help")
161    self.assertEqual(cm.exception.exit_code, 0)
162    _, stdout, stderr = self.run_cli_output("help", raises=SysExitTestException)
163    self.assertFalse(stderr)
164    self.assertIn("usage:", stdout)
165    self.assertIn("Subcommands:", stdout)
166    # Check for top-level option:
167    self.assertIn("--no-color", stdout)
168    self.assertIn("Disable colored output", stdout)
169    self.assertIn("Available Probes for all Benchmarks:", stdout)
170
171  def test_version(self):
172    with self.assertRaises(SysExitTestException) as cm:
173      self.run_cli("--version")
174    self.assertEqual(cm.exception.exit_code, 0)
175    _, stdout, stderr = self.run_cli_output(
176        "--version", raises=SysExitTestException)
177    self.assertFalse(stderr)
178    self.assertIn(__version__, stdout)
179
180  def test_version_subcommand(self):
181    with self.assertRaises(SysExitTestException) as cm:
182      self.run_cli("version")
183    self.assertEqual(cm.exception.exit_code, 0)
184    _, stdout, stderr = self.run_cli_output(
185        "version", raises=SysExitTestException)
186    self.assertFalse(stderr)
187    self.assertIn(__version__, stdout)
188
189  def test_subcommand_run_subcommand(self):
190    with self.patch_get_browser():
191      url = "http://test.com"
192      self.run_cli("loading", "run", f"--urls={url}", "--env-validation=skip",
193                   "--throw")
194      for browser in self.browsers:
195        self.assertListEqual([url], browser.url_list[self.SPLASH_URLS_LEN:])
196
197  def test_invalid_probe(self):
198    with self.assertRaises(argparse.ArgumentError), self.patch_get_browser():
199      self.run_cli("loading", "--probe=invalid_probe_name", "--throw")
200
201  def test_basic_probe_setting(self):
202    with self.patch_get_browser():
203      url = "http://test.com"
204      self.run_cli("loading", "--probe=v8.log", f"--urls={url}",
205                   "--env-validation=skip", "--throw")
206      for browser in self.browsers:
207        self.assertListEqual([url], browser.url_list[self.SPLASH_URLS_LEN:])
208        self.assertIn("--log-all", browser.js_flags)
209
210  def test_invalid_empty_probe_config_file(self):
211    config_file = pathlib.Path("/config.hjson")
212    config_file.touch()
213    with self.patch_get_browser():
214      url = "http://test.com"
215      with self.assertRaises(argparse.ArgumentError) as cm:
216        self.run_cli("loading", f"--probe-config={config_file}",
217                     f"--urls={url}", "--env-validation=skip", "--throw")
218      message = str(cm.exception)
219      self.assertIn("--probe-config", message)
220      self.assertIn("empty", message)
221      for browser in self.browsers:
222        self.assertListEqual([], browser.url_list[self.SPLASH_URLS_LEN:])
223        self.assertNotIn("--log", browser.js_flags)
224
225  def test_empty_probe_config_file(self):
226    config_file = pathlib.Path("/config.hjson")
227    config_data = {"probes": {}}
228    with config_file.open("w", encoding="utf-8") as f:
229      hjson.dump(config_data, f)
230
231    with self.patch_get_browser():
232      url = "http://test.com"
233      self.run_cli("loading", f"--probe-config={config_file}", f"--urls={url}",
234                   "--env-validation=skip")
235      for browser in self.browsers:
236        self.assertListEqual([url], browser.url_list[self.SPLASH_URLS_LEN:])
237        self.assertNotIn("--log", browser.js_flags)
238
239  def test_invalid_probe_config_file(self):
240    config_file = pathlib.Path("/config.hjson")
241    config_data = {"probes": {"invalid probe name": {}}}
242    with config_file.open("w", encoding="utf-8") as f:
243      hjson.dump(config_data, f)
244    with self.patch_get_browser():
245      url = "http://test.com"
246      with self.assertRaises(argparse.ArgumentTypeError):
247        self.run_cli("loading", f"--probe-config={config_file}",
248                     f"--urls={url}", "--env-validation=skip", "--throw")
249      for browser in self.browsers:
250        self.assertListEqual([], browser.url_list)
251        self.assertEqual(len(browser.js_flags), 0)
252
253  def test_probe_config_file(self):
254    config_file = pathlib.Path("/config.hjson")
255    js_flags = ["--log-foo", "--log-bar"]
256    config_data = {"probes": {"v8.log": {"js_flags": js_flags}}}
257    with config_file.open("w", encoding="utf-8") as f:
258      hjson.dump(config_data, f)
259
260    with self.patch_get_browser():
261      url = "http://test.com"
262      self.run_cli("loading", f"--probe-config={config_file}", f"--urls={url}",
263                   "--env-validation=skip")
264      for browser in self.browsers:
265        self.assertListEqual([url], browser.url_list[self.SPLASH_URLS_LEN:])
266        for flag in js_flags:
267          self.assertIn(flag, browser.js_flags)
268
269  def test_probe_config_file_invalid_probe(self):
270    config_file = pathlib.Path("/config.hjson")
271    config_data = {"probes": {"invalid probe name": {}}}
272    with config_file.open("w", encoding="utf-8") as f:
273      hjson.dump(config_data, f)
274    with self.assertRaises(
275        argparse.ArgumentTypeError) as cm, self.patch_get_browser():
276      self.run_cli("loading", f"--probe-config={config_file}",
277                   "--urls=http://test.com", "--env-validation=skip", "--throw")
278    self.assertIn("invalid probe name", str(cm.exception))
279
280  def test_empty_config_file_properties(self):
281    config_file = pathlib.Path("/config.hjson")
282    config_data = {"probes": {}, "env": {}, "browsers": {}, "network": {}}
283    with config_file.open("w", encoding="utf-8") as f:
284      hjson.dump(config_data, f)
285    with self.assertRaises(
286        argparse.ArgumentTypeError) as cm, self.patch_get_browser():
287      url = "http://test.com"
288      self.run_cli("loading", f"--config={config_file}", f"--urls={url}",
289                   "--env-validation=skip", "--throw")
290    self.assertIn("no config properties", str(cm.exception))
291
292  def test_empty_config_files(self):
293    config_file = pathlib.Path("/config.hjson")
294    config_data = {}
295    with config_file.open("w", encoding="utf-8") as f:
296      hjson.dump(config_data, f)
297    with self.assertRaises(
298        argparse.ArgumentTypeError) as cm, self.patch_get_browser():
299      url = "http://test.com"
300      self.run_cli("loading", f"--config={config_file}", f"--urls={url}",
301                   "--env-validation=skip", "--throw")
302    self.assertIn("no config properties", str(cm.exception))
303
304  def test_conflicting_config_flags(self):
305    config_file = pathlib.Path("/config.hjson")
306    config_data = {"probes": {}, "env": {}, "browsers": {}, "network": {}}
307    for config_flag in ("--probe-config", "--env-config", "--browser-config",
308                        "--network-config"):
309      with config_file.open("w", encoding="utf-8") as f:
310        hjson.dump(config_data, f)
311      with self.assertRaises(argparse.ArgumentTypeError) as cm:
312        self.run_cli("sp2", f"--config={config_file}",
313                     f"{config_flag}={config_file}", "--env-validation=skip",
314                     "--throw")
315      message = str(cm.exception)
316      self.assertIn("--config", message)
317      self.assertIn(config_flag, message)
318
319  def test_config_file_with_probe(self):
320    config_file = pathlib.Path("/config.hjson")
321    js_flags = ["--log-foo", "--log-bar"]
322    config_data = {
323        "probes": {
324            "v8.log": {
325                "js_flags": js_flags
326            }
327        },
328        "env": {},
329        "browsers": {},
330        "network": {},
331    }
332    with config_file.open("w", encoding="utf-8") as f:
333      hjson.dump(config_data, f)
334
335    with self.patch_get_browser():
336      url = "http://test.com"
337      self.run_cli("loading", f"--config={config_file}", f"--urls={url}",
338                   "--env-validation=skip")
339      for browser in self.browsers:
340        self.assertListEqual([url], browser.url_list[self.SPLASH_URLS_LEN:])
341        for flag in js_flags:
342          self.assertIn(flag, browser.js_flags)
343
344  def test_config_file_with_env(self):
345    config_file = pathlib.Path("/config.hjson")
346    config_data = {
347        "probes": {},
348        "env": {
349            "screen_brightness_percent": 66,
350            "cpu_max_usage_percent": 77,
351        },
352        "browsers": {},
353        "network": {},
354    }
355    with config_file.open("w", encoding="utf-8") as f:
356      hjson.dump(config_data, f)
357
358    with self.patch_get_browser():
359      url = "http://test.com"
360      cli = self.run_cli("loading", f"--config={config_file}", f"--urls={url}",
361                         "--env-validation=skip")
362      for browser in self.browsers:
363        self.assertListEqual([url], browser.url_list[self.SPLASH_URLS_LEN:])
364        self.assertFalse(browser.js_flags)
365      config = cli.runner.env.config
366      self.assertEqual(config.disk_min_free_space_gib,
367                       HostEnvironmentConfig.IGNORE)
368      self.assertEqual(config.screen_brightness_percent, 66)
369      self.assertEqual(config.cpu_max_usage_percent, 77)
370
371  def test_config_file_with_browser(self):
372    config_file = pathlib.Path("/config.hjson")
373    config_data = {
374        "probes": {},
375        "env": {},
376        "browsers": {
377            "browser_1": {
378                "path": "chrome-dev",
379            },
380            "browser_2": {
381                "path": "chrome-stable"
382            }
383        },
384        "network": {},
385    }
386    with config_file.open("w", encoding="utf-8") as f:
387      hjson.dump(config_data, f)
388
389    def mock_get_browser_cls(browser_config: BrowserConfig):
390      path_str = str(browser_config.path).lower()
391      if "dev" in path_str:
392        return mock_browser.MockChromeDev
393      return mock_browser.MockChromeStable
394
395    with mock.patch.object(
396        BrowserVariantsConfig,
397        "get_browser_cls",
398        side_effect=mock_get_browser_cls):
399      url = "http://test.com"
400      cli = self.run_cli("loading", f"--config={config_file}", f"--urls={url}",
401                         "--env-validation=skip")
402      browsers = cli.runner.browsers
403      self.assertEqual(len(browsers), 2)
404      self.assertEqual(browsers[0].label, "browser_1")
405      self.assertEqual(browsers[1].label, "browser_2")
406      for browser in browsers:
407        self.assertFalse(browser.js_flags)
408
409  def test_invalid_browser_identifier(self):
410    with self.assertRaises(argparse.ArgumentError) as cm:
411      self.run_cli("loading", "--browser=unknown_browser_identifier",
412                   "--urls=http://test.com", "--env-validation=skip", "--throw")
413    self.assertIn("--browser", str(cm.exception))
414    self.assertIn("unknown_browser_identifier", str(cm.exception))
415
416  def test_unknown_browser_binary(self):
417    browser_bin = pathlib.Path("/foo/custom/browser.bin")
418    browser_bin.parent.mkdir(parents=True)
419    browser_bin.touch()
420    with self.assertRaises(argparse.ArgumentError) as cm:
421      self.run_cli("loading", f"--browser={browser_bin}",
422                   "--urls=http://test.com", "--env-validation=skip", "--throw")
423    self.assertIn("--browser", str(cm.exception))
424    self.assertIn(str(browser_bin), str(cm.exception))
425
426  @unittest.skipUnless(plt.PLATFORM.is_win, "Can only run on windows")
427  def test_unknown_browser_binary_win(self):
428    browser_bin = pathlib.Path("C:\\foo\\custom\\browser.bin")
429    browser_bin.parent.mkdir(parents=True)
430    browser_bin.touch()
431    with self.assertRaises(argparse.ArgumentError) as cm:
432      self.run_cli("loading", f"--browser={browser_bin}",
433                   "--urls=http://test.com", "--env-validation=skip", "--throw")
434    self.assertIn("--browser", str(cm.exception))
435    self.assertIn(str(browser_bin), str(cm.exception))
436
437
438if __name__ == "__main__":
439  test_helper.run_pytest(__file__)
440