1# Copyright 2024 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5from __future__ import annotations 6 7import argparse 8import copy 9import json 10import unittest 11from typing import Dict, Tuple, Type 12from unittest import mock 13 14import hjson 15from tests import test_helper 16from tests.crossbench import mock_browser 17from tests.crossbench.cli.config.base import (ADB_DEVICES_SINGLE_OUTPUT, 18 BaseConfigTestCase) 19from tests.crossbench.mock_helper import AndroidAdbMockPlatform, MockAdb 20 21from crossbench import path as pth 22from crossbench import plt 23from crossbench.browsers.chrome.applescript import ChromeAppleScript 24from crossbench.browsers.chrome.chrome import Chrome 25from crossbench.browsers.chrome.webdriver import (ChromeWebDriver, 26 ChromeWebDriverAndroid, 27 ChromeWebDriverChromeOsSsh, 28 ChromeWebDriverSsh, 29 LocalChromeWebDriverAndroid) 30from crossbench.browsers.chromium.applescript import ChromiumAppleScript 31from crossbench.browsers.chromium.webdriver import (ChromiumWebDriver, 32 ChromiumWebDriverAndroid, 33 ChromiumWebDriverSsh) 34from crossbench.browsers.safari.safari import Safari 35from crossbench.cli.config.browser import BrowserConfig 36from crossbench.cli.config.browser_variants import BrowserVariantsConfig 37from crossbench.cli.config.driver import BrowserDriverType, DriverConfig 38from crossbench.cli.config.network import NetworkConfig 39from crossbench.config import ConfigError 40 41 42class TestBrowserVariantsConfig(BaseConfigTestCase): 43 # pylint: disable=expression-not-assigned 44 45 EXAMPLE_CONFIG_PATH = test_helper.config_dir() / "doc/browser.config.hjson" 46 47 EXAMPLE_REMOTE_CONFIG_PATH = ( 48 test_helper.config_dir() / "doc/remote_browser.config.hjson") 49 50 def setUp(self): 51 super().setUp() 52 self.browser_lookup: Dict[str, Tuple[ 53 Type[mock_browser.MockBrowser], BrowserConfig]] = { 54 "chr-stable": 55 (mock_browser.MockChromeStable, 56 BrowserConfig(mock_browser.MockChromeStable.mock_app_path())), 57 "chr-dev": 58 (mock_browser.MockChromeDev, 59 BrowserConfig(mock_browser.MockChromeDev.mock_app_path())), 60 "chrome-stable": 61 (mock_browser.MockChromeStable, 62 BrowserConfig(mock_browser.MockChromeStable.mock_app_path())), 63 "chrome-dev": 64 (mock_browser.MockChromeDev, 65 BrowserConfig(mock_browser.MockChromeDev.mock_app_path())), 66 } 67 for _, (_, browser_config) in self.browser_lookup.items(): 68 self.assertTrue(browser_config.path.exists()) 69 70 def _expect_linux_ssh(self, cmd, **kwargs): 71 return self.platform.expect_sh("ssh", "-p", "22", "user@my-linux-machine", 72 cmd, **kwargs) 73 74 def _expect_chromeos_ssh(self, cmd, **kwargs): 75 return self.platform.expect_sh("ssh", "-p", "22", 76 "root@my-chromeos-machine", cmd, **kwargs) 77 78 def test_parse_browser_config_template(self): 79 if not self.EXAMPLE_CONFIG_PATH.exists(): 80 raise unittest.SkipTest( 81 f"Test file {self.EXAMPLE_CONFIG_PATH} does not exist") 82 self.fs.add_real_file(self.EXAMPLE_CONFIG_PATH) 83 with self.EXAMPLE_CONFIG_PATH.open(encoding="utf-8") as f: 84 config = BrowserVariantsConfig( 85 browser_lookup_override=self.browser_lookup) 86 config.parse_text_io(f, args=self.mock_args) 87 self.assertIn("flag-group-1", config.flags_config) 88 self.assertGreaterEqual(len(config.flags_config), 1) 89 self.assertGreaterEqual(len(config.variants), 1) 90 91 def test_parse_remote_browser_config_template(self): 92 if not self.EXAMPLE_REMOTE_CONFIG_PATH.exists(): 93 raise unittest.SkipTest( 94 f"Test file {self.EXAMPLE_REMOTE_CONFIG_PATH} does not exist") 95 self.fs.add_real_file(self.EXAMPLE_REMOTE_CONFIG_PATH) 96 97 self._expect_linux_ssh("uname -m", result="arm64") 98 self._expect_linux_ssh("'[' -e /path/to/google/chrome ']'") 99 self._expect_linux_ssh("'[' -f /path/to/google/chrome ']'") 100 self._expect_linux_ssh("'[' -e /path/to/google/chrome ']'") 101 self._expect_linux_ssh( 102 "/path/to/google/chrome --version", result="102.22.33.44") 103 self._expect_linux_ssh("env") 104 self._expect_linux_ssh("'[' -d /tmp ']'") 105 self._expect_linux_ssh("mktemp -d /tmp/chrome.XXXXXXXXXXX") 106 107 self._expect_chromeos_ssh("'[' -e /usr/local/autotest/bin/autologin.py ']'") 108 self._expect_chromeos_ssh("uname -m", result="arm64") 109 self._expect_chromeos_ssh("'[' -e /opt/google/chrome/chrome ']'") 110 self._expect_chromeos_ssh("'[' -f /opt/google/chrome/chrome ']'") 111 self._expect_chromeos_ssh("'[' -e /opt/google/chrome/chrome ']'") 112 self._expect_chromeos_ssh( 113 "/opt/google/chrome/chrome --version", result="125.0.6422.60") 114 self._expect_chromeos_ssh("env") 115 self._expect_chromeos_ssh("'[' -d /tmp ']'") 116 self._expect_chromeos_ssh("mktemp -d /tmp/chrome.XXXXXXXXXXX") 117 118 with self.EXAMPLE_REMOTE_CONFIG_PATH.open(encoding="utf-8") as f: 119 config = BrowserVariantsConfig() 120 config.parse_text_io(f, args=self.mock_args) 121 self.assertEqual(len(config.variants), 2) 122 for variant in config.variants: 123 self.assertTrue(variant.platform.is_remote) 124 self.assertTrue(variant.platform.is_linux) 125 self.assertEqual(config.variants[0].platform.name, "linux_ssh") 126 self.assertEqual(config.variants[1].platform.name, "chromeos_ssh") 127 self.assertEqual(config.variants[0].version, "102.22.33.44") 128 self.assertEqual(config.variants[1].version, "125.0.6422.60") 129 130 def test_browser_labels_attributes(self): 131 browsers = BrowserVariantsConfig( 132 { 133 "browsers": { 134 "chrome-stable-default": { 135 "path": "chrome-stable", 136 }, 137 "chrome-stable-noopt": { 138 "path": "chrome-stable", 139 "flags": ["--js-flags=--max-opt=0",] 140 }, 141 "chrome-stable-custom": { 142 "label": "custom-label-property", 143 "path": "chrome-stable", 144 "flags": ["--js-flags=--max-opt=0",] 145 } 146 } 147 }, 148 browser_lookup_override=self.browser_lookup, 149 args=self.mock_args).variants 150 self.assertEqual(len(browsers), 3) 151 self.assertEqual(browsers[0].label, "chrome-stable-default") 152 self.assertEqual(browsers[1].label, "chrome-stable-noopt") 153 self.assertEqual(browsers[2].label, "custom-label-property") 154 155 def test_browser_label_args(self): 156 self.platform.sh_results = [ADB_DEVICES_SINGLE_OUTPUT] 157 args = self.mock_args 158 adb_config = BrowserConfig.parse("adb:chrome") 159 desktop_config = BrowserConfig.parse("chrome") 160 args.browser = [ 161 adb_config, 162 desktop_config, 163 ] 164 self.assertFalse(self.platform.sh_results) 165 self.platform.sh_results = [ 166 ADB_DEVICES_SINGLE_OUTPUT, 167 ADB_DEVICES_SINGLE_OUTPUT, 168 ] 169 170 def mock_get_browser_cls(browser_config: BrowserConfig): 171 if browser_config is adb_config: 172 return mock_browser.MockChromeAndroidStable 173 if browser_config is desktop_config: 174 return mock_browser.MockChromeStable 175 raise ValueError("Unknown browser_config") 176 177 with mock.patch.object( 178 BrowserVariantsConfig, 179 "get_browser_cls", 180 side_effect=mock_get_browser_cls), mock.patch( 181 "crossbench.plt.android_adb.AndroidAdbPlatform.machine", 182 new_callable=mock.PropertyMock, 183 return_value=plt.MachineArch.ARM_64): 184 browsers = BrowserVariantsConfig.from_cli_args(args).variants 185 self.assertEqual(len(browsers), 2) 186 self.assertEqual(browsers[0].label, "android.arm64.remote_0") 187 self.assertEqual(browsers[1].label, f"{self.platform}_1") 188 189 with self.assertRaises(ConfigError) as cm: 190 BrowserVariantsConfig( 191 { 192 "browsers": { 193 "chrome-stable-label": { 194 "path": "chrome-stable", 195 }, 196 "chrome-stable-custom": { 197 "label": "chrome-stable-label", 198 "path": "chrome-stable", 199 } 200 } 201 }, 202 browser_lookup_override=self.browser_lookup, 203 args=self.mock_args).variants 204 message = str(cm.exception) 205 self.assertIn("chrome-stable-label", message) 206 self.assertIn("chrome-stable-custom", message) 207 208 def test_parse_invalid_browser_type(self): 209 for invalid in (None, 1, []): 210 with self.assertRaises(ConfigError) as cm: 211 _ = BrowserVariantsConfig( 212 { 213 "browsers": { 214 "chrome-stable-default": invalid 215 } 216 }, 217 args=self.mock_args).variants 218 self.assertIn("Expected str or dict", str(cm.exception)) 219 220 def test_browser_custom_driver_variants(self): 221 self.platform.sh_results = [ 222 ADB_DEVICES_SINGLE_OUTPUT, ADB_DEVICES_SINGLE_OUTPUT, 223 ADB_DEVICES_SINGLE_OUTPUT, ADB_DEVICES_SINGLE_OUTPUT 224 ] 225 226 def mock_get_browser_platform( 227 browser_config: BrowserConfig) -> plt.Platform: 228 if browser_config.driver.type == BrowserDriverType.ANDROID: 229 return AndroidAdbMockPlatform(self.platform, adb=MockAdb(self.platform)) 230 return self.platform 231 232 with self.mock_chrome_stable( 233 mock_browser.MockChromeAndroidStable), mock.patch.object( 234 BrowserVariantsConfig, 235 "_get_browser_platform", 236 side_effect=mock_get_browser_platform): 237 browsers = BrowserVariantsConfig( 238 { 239 "browsers": { 240 "chrome-stable-default": "chrome-stable", 241 "chrome-stable-adb": "adb:chrome", 242 "chrome-stable-adb2": { 243 "path": "chrome", 244 "driver": "adb" 245 } 246 } 247 }, 248 browser_lookup_override=self.browser_lookup, 249 args=self.mock_args).variants 250 self.assertEqual(len(browsers), 3) 251 self.assertEqual(browsers[0].label, "chrome-stable-default") 252 self.assertEqual(browsers[1].label, "chrome-stable-adb") 253 self.assertEqual(browsers[2].label, "chrome-stable-adb2") 254 self.assertIsInstance(browsers[0], mock_browser.MockChromeStable) 255 self.assertIsInstance(browsers[1], mock_browser.MockChromeAndroidStable) 256 self.assertIsInstance(browsers[2], mock_browser.MockChromeAndroidStable) 257 258 def test_flag_combination_invalid(self): 259 with self.assertRaises(ConfigError) as cm: 260 BrowserVariantsConfig( 261 { 262 "flags": { 263 "group1": { 264 "invalid-flag-name": [None, "", "v1"], 265 }, 266 }, 267 "browsers": { 268 "chrome-stable": { 269 "path": "chrome-stable", 270 "flags": ["group1",] 271 } 272 } 273 }, 274 browser_lookup_override=self.browser_lookup, 275 args=self.mock_args).variants 276 message = str(cm.exception) 277 self.assertIn("group1", message) 278 self.assertIn("invalid-flag-name", message) 279 280 def test_flag_combination_none(self): 281 with self.assertRaises(ConfigError) as cm: 282 BrowserVariantsConfig( 283 { 284 "flags": { 285 "group1": { 286 "--foo": ["None,", "", "v1"], 287 }, 288 }, 289 "browsers": { 290 "chrome-stable": { 291 "path": "chrome-stable", 292 "flags": ["group1"] 293 } 294 } 295 }, 296 browser_lookup_override=self.browser_lookup, 297 args=self.mock_args).variants 298 self.assertIn("None", str(cm.exception)) 299 300 def test_flag_combination_duplicate(self): 301 with self.assertRaises(ConfigError) as cm: 302 BrowserVariantsConfig( 303 { 304 "flags": { 305 "group1": { 306 "--duplicate-flag": [None, "", "v1"], 307 }, 308 "group2": { 309 "--duplicate-flag": [None, "", "v1"], 310 } 311 }, 312 "browsers": { 313 "chrome-stable": { 314 "path": "chrome-stable", 315 "flags": ["group1", "group2"] 316 } 317 } 318 }, 319 browser_lookup_override=self.browser_lookup, 320 args=self.mock_args).variants 321 self.assertIn("--duplicate-flag", str(cm.exception)) 322 323 def test_empty(self): 324 with self.assertRaises(ConfigError): 325 BrowserVariantsConfig({"other": {}}, args=self.mock_args).variants 326 with self.assertRaises(ConfigError): 327 BrowserVariantsConfig({"browsers": {}}, args=self.mock_args).variants 328 329 def test_unknown_group(self): 330 with self.assertRaises(ConfigError) as cm: 331 BrowserVariantsConfig( 332 { 333 "browsers": { 334 "chrome-stable": { 335 "path": "chrome-stable", 336 "flags": ["unknown-flag-group"] 337 } 338 } 339 }, 340 args=self.mock_args).variants 341 self.assertIn("unknown-flag-group", str(cm.exception)) 342 343 def test_duplicate_group(self): 344 with self.assertRaises(ConfigError): 345 BrowserVariantsConfig( 346 { 347 "flags": { 348 "group1": {} 349 }, 350 "browsers": { 351 "chrome-stable": { 352 "path": "chrome-stable", 353 "flags": ["group1", "group1"] 354 } 355 } 356 }, 357 args=self.mock_args).variants 358 359 def test_non_list_group(self): 360 BrowserVariantsConfig( 361 { 362 "flags": { 363 "group1": {} 364 }, 365 "browsers": { 366 "chrome-stable": { 367 "path": "chrome-stable", 368 "flags": "group1" 369 } 370 } 371 }, 372 browser_lookup_override=self.browser_lookup, 373 args=self.mock_args).variants 374 with self.assertRaises(ConfigError) as cm: 375 BrowserVariantsConfig( 376 { 377 "flags": { 378 "group1": {} 379 }, 380 "browsers": { 381 "chrome-stable": { 382 "path": "chrome-stable", 383 "flags": 1 384 } 385 } 386 }, 387 browser_lookup_override=self.browser_lookup, 388 args=self.mock_args).variants 389 self.assertIn("chrome-stable", str(cm.exception)) 390 self.assertIn("flags", str(cm.exception)) 391 392 with self.assertRaises(ConfigError) as cm: 393 BrowserVariantsConfig( 394 { 395 "flags": { 396 "group1": {} 397 }, 398 "browsers": { 399 "chrome-stable": { 400 "path": "chrome-stable", 401 "flags": { 402 "group1": True 403 } 404 } 405 } 406 }, 407 browser_lookup_override=self.browser_lookup, 408 args=self.mock_args).variants 409 self.assertIn("chrome-stable", str(cm.exception)) 410 self.assertIn("flags", str(cm.exception)) 411 412 def test_duplicate_flag_variant_value(self): 413 with self.assertRaises(ConfigError) as cm: 414 BrowserVariantsConfig( 415 { 416 "flags": { 417 "group1": { 418 "--flag": ["repeated", "repeated"] 419 } 420 }, 421 "browsers": { 422 "chrome-stable": { 423 "path": "chrome-stable", 424 "flags": "group1", 425 } 426 } 427 }, 428 args=self.mock_args).variants 429 self.assertIn("group1", str(cm.exception)) 430 self.assertIn("--flag", str(cm.exception)) 431 432 def test_unknown_path(self): 433 with self.assertRaises(Exception): 434 BrowserVariantsConfig( 435 { 436 "browsers": { 437 "chrome-stable": { 438 "path": "path/does/not/exist", 439 } 440 } 441 }, 442 args=self.mock_args).variants 443 with self.assertRaises(Exception): 444 BrowserVariantsConfig( 445 { 446 "browsers": { 447 "chrome-stable": { 448 "path": "chrome-unknown", 449 } 450 } 451 }, 452 args=self.mock_args).variants 453 454 def test_flag_combination_simple(self): 455 config = BrowserVariantsConfig( 456 { 457 "flags": { 458 "group1": { 459 "--foo": [None, "", "v1"], 460 } 461 }, 462 "browsers": { 463 "chrome-stable": { 464 "path": "chrome-stable", 465 "flags": ["group1"] 466 } 467 } 468 }, 469 browser_lookup_override=self.browser_lookup, 470 args=self.mock_args) 471 browsers = config.variants 472 self.assertEqual(len(browsers), 3) 473 for browser in browsers: 474 assert isinstance(browser, mock_browser.MockChromeStable) 475 self.assertDictEqual(browser.js_flags.to_dict(), {}) 476 self.assertDictEqual(browsers[0].flags.to_dict(), {}) 477 self.assertDictEqual(browsers[1].flags.to_dict(), {"--foo": None}) 478 self.assertDictEqual(browsers[2].flags.to_dict(), {"--foo": "v1"}) 479 480 def test_flag_list(self): 481 config = BrowserVariantsConfig( 482 { 483 "flags": { 484 "group1": [ 485 "", 486 "--foo", 487 "-foo=v1", 488 ] 489 }, 490 "browsers": { 491 "chrome-stable": { 492 "path": "chrome-stable", 493 "flags": ["group1"] 494 } 495 } 496 }, 497 browser_lookup_override=self.browser_lookup, 498 args=self.mock_args) 499 browsers = config.variants 500 self.assertEqual(len(browsers), 3) 501 for browser in browsers: 502 assert isinstance(browser, mock_browser.MockChromeStable) 503 self.assertDictEqual(browser.js_flags.to_dict(), {}) 504 self.assertDictEqual(browsers[0].flags.to_dict(), {}) 505 self.assertDictEqual(browsers[1].flags.to_dict(), {"--foo": None}) 506 self.assertDictEqual(browsers[2].flags.to_dict(), {"-foo": "v1"}) 507 508 def test_flag_combination(self): 509 config = BrowserVariantsConfig( 510 { 511 "flags": { 512 "group1": { 513 "--foo": [None, "", "v1"], 514 "--bar": [None, "", "v1"], 515 } 516 }, 517 "browsers": { 518 "chrome-stable": { 519 "path": "chrome-stable", 520 "flags": ["group1"] 521 } 522 } 523 }, 524 browser_lookup_override=self.browser_lookup, 525 args=self.mock_args) 526 self.assertEqual(len(config.variants), 3 * 3) 527 528 def test_flag_combination_mixed_inline(self): 529 config = BrowserVariantsConfig( 530 { 531 "flags": { 532 "compile-hints-experiment": { 533 "--enable-features": [None, "ConsumeCompileHints"] 534 } 535 }, 536 "browsers": { 537 "chrome-release": { 538 "path": "chrome-stable", 539 "flags": ["--no-sandbox", "compile-hints-experiment"] 540 } 541 } 542 }, 543 browser_lookup_override=self.browser_lookup, 544 args=self.mock_args) 545 browsers = config.variants 546 self.assertEqual(len(browsers), 2) 547 self.assertListEqual(["--no-sandbox"], list(browsers[0].flags)) 548 self.assertListEqual( 549 ["--no-sandbox", "--enable-features=ConsumeCompileHints"], 550 list(browsers[1].flags)) 551 552 def test_flag_single_inline(self): 553 config = BrowserVariantsConfig( 554 { 555 "browsers": { 556 "chrome-release": { 557 "path": "chrome-stable", 558 "flags": "--no-sandbox", 559 } 560 } 561 }, 562 browser_lookup_override=self.browser_lookup, 563 args=self.mock_args) 564 browsers = config.variants 565 self.assertEqual(len(browsers), 1) 566 self.assertListEqual(["--no-sandbox"], list(browsers[0].flags)) 567 568 def test_flag_combination_mixed_fixed(self): 569 config = BrowserVariantsConfig( 570 { 571 "flags": { 572 "compile-hints-experiment": { 573 "--no-sandbox": "", 574 "--enable-features": [None, "ConsumeCompileHints"] 575 } 576 }, 577 "browsers": { 578 "chrome-release": { 579 "path": "chrome-stable", 580 "flags": "compile-hints-experiment" 581 } 582 } 583 }, 584 browser_lookup_override=self.browser_lookup, 585 args=self.mock_args) 586 browsers = config.variants 587 self.assertEqual(len(browsers), 2) 588 self.assertListEqual(["--no-sandbox"], list(browsers[0].flags)) 589 self.assertListEqual( 590 ["--no-sandbox", "--enable-features=ConsumeCompileHints"], 591 list(browsers[1].flags)) 592 593 def test_conflicting_chrome_features(self): 594 with self.assertRaises(ConfigError) as cm: 595 _ = BrowserVariantsConfig( 596 { 597 "flags": { 598 "compile-hints-experiment": { 599 "--enable-features": [None, "ConsumeCompileHints"] 600 } 601 }, 602 "browsers": { 603 "chrome-release": { 604 "path": 605 "chrome-stable", 606 "flags": [ 607 "--disable-features=ConsumeCompileHints", 608 "compile-hints-experiment" 609 ] 610 } 611 } 612 }, 613 browser_lookup_override=self.browser_lookup, 614 args=self.mock_args) 615 msg = str(cm.exception) 616 self.assertIn("ConsumeCompileHints", msg) 617 618 def test_no_flags(self): 619 config = BrowserVariantsConfig( 620 { 621 "browsers": { 622 "chrome-stable": { 623 "path": "chrome-stable", 624 }, 625 "chrome-dev": { 626 "path": "chrome-dev", 627 } 628 } 629 }, 630 browser_lookup_override=self.browser_lookup, 631 args=self.mock_args) 632 self.assertEqual(len(config.variants), 2) 633 browser_0 = config.variants[0] 634 assert isinstance(browser_0, mock_browser.MockChromeStable) 635 self.assertEqual(browser_0.app_path, 636 mock_browser.MockChromeStable.mock_app_path()) 637 browser_1 = config.variants[1] 638 assert isinstance(browser_1, mock_browser.MockChromeDev) 639 self.assertEqual(browser_1.app_path, 640 mock_browser.MockChromeDev.mock_app_path()) 641 642 def test_custom_driver(self): 643 chromedriver = pth.LocalPath("path/to/chromedriver") 644 variants_config = { 645 "browsers": { 646 "chrome-stable": { 647 "browser": "chrome-stable", 648 "driver": str(chromedriver), 649 } 650 } 651 } 652 with self.assertRaises(argparse.ArgumentTypeError) as cm: 653 BrowserVariantsConfig( 654 copy.deepcopy(variants_config), 655 browser_lookup_override=self.browser_lookup, 656 args=self.mock_args) 657 self.assertIn(str(chromedriver), str(cm.exception)) 658 659 self.fs.create_file(chromedriver, st_size=100) 660 with mock.patch.object( 661 BrowserVariantsConfig, 662 "get_browser_cls", 663 return_value=mock_browser.MockChromeStable): 664 config = BrowserVariantsConfig( 665 variants_config, 666 browser_lookup_override=self.browser_lookup, 667 args=self.mock_args) 668 self.assertTrue(variants_config["browsers"]["chrome-stable"]) 669 self.assertEqual(len(config.variants), 1) 670 browser_0 = config.variants[0] 671 assert isinstance(browser_0, mock_browser.MockChromeStable) 672 self.assertEqual(browser_0.app_path, 673 mock_browser.MockChromeStable.mock_app_path()) 674 675 def test_inline_flags(self): 676 with mock.patch.object( 677 ChromeWebDriver, "_extract_version", 678 return_value="101.22.333.44"), mock.patch.object( 679 Chrome, 680 "stable_path", 681 return_value=mock_browser.MockChromeStable.mock_app_path()): 682 683 config = BrowserVariantsConfig( 684 { 685 "browsers": { 686 "stable": { 687 "path": "chrome-stable", 688 "flags": ["--foo=bar"] 689 } 690 } 691 }, 692 args=self.mock_args) 693 self.assertEqual(len(config.variants), 1) 694 browser = config.variants[0] 695 # TODO: Fix once app lookup is cleaned up 696 self.assertEqual(browser.app_path, 697 mock_browser.MockChromeStable.mock_app_path()) 698 self.assertEqual(browser.version, "101.22.333.44") 699 self.assertEqual(browser.flags["--foo"], "bar") 700 701 def test_inline_load_safari(self): 702 if not plt.PLATFORM.is_macos: 703 return 704 with mock.patch.object(Safari, "_extract_version", return_value="16.0"): 705 config = BrowserVariantsConfig( 706 {"browsers": { 707 "safari": { 708 "path": "safari", 709 } 710 }}, args=self.mock_args) 711 self.assertEqual(len(config.variants), 1) 712 713 def test_flag_combination_with_fixed(self): 714 config = BrowserVariantsConfig( 715 { 716 "flags": { 717 "group1": { 718 "--foo": [None, "", "v1"], 719 "--bar": [None, "", "w1"], 720 "--always_1": "true", 721 "--always_2": "true", 722 "--always_3": "true", 723 } 724 }, 725 "browsers": { 726 "chrome-stable": { 727 "path": "chrome-stable", 728 "flags": ["group1"] 729 } 730 } 731 }, 732 browser_lookup_override=self.browser_lookup, 733 args=self.mock_args) 734 self.assertEqual(len(config.variants), 3 * 3) 735 for browser in config.variants: 736 assert isinstance(browser, mock_browser.MockChromeStable) 737 self.assertEqual(browser.app_path, 738 mock_browser.MockChromeStable.mock_app_path()) 739 expected_flags = ( 740 "--always_1=true --always_2=true --always_3=true", 741 "--bar --always_1=true --always_2=true --always_3=true", 742 "--bar=w1 --always_1=true --always_2=true --always_3=true", 743 "--foo --always_1=true --always_2=true --always_3=true", 744 "--foo --bar --always_1=true --always_2=true --always_3=true", 745 "--foo --bar=w1 --always_1=true --always_2=true --always_3=true", 746 "--foo=v1 --always_1=true --always_2=true --always_3=true", 747 "--foo=v1 --bar --always_1=true --always_2=true --always_3=true", 748 "--foo=v1 --bar=w1 --always_1=true --always_2=true --always_3=true", 749 ) 750 self.verify_variant_flags(config.variants, expected_flags) 751 752 def verify_variant_flags(self, variants, expected_flags): 753 self.assertEqual(len(variants), len(expected_flags)) 754 for index, browser in enumerate(variants): 755 self.assertEqual( 756 str(browser.flags), expected_flags[index], 757 f"Unexpected flags for variant[{index}]") 758 759 def test_flag_combination_js_flags_with_fixed(self): 760 config = BrowserVariantsConfig( 761 { 762 "flags": { 763 "group1": { 764 "--js-flags": [ 765 None, "--max-opt=1,--trace-ic", "--max-opt=2 --log-all" 766 ], 767 }, 768 "group2": { 769 "default": "--bar=v1 --foo=w2" 770 } 771 }, 772 "browsers": { 773 "chrome-stable": { 774 "path": "chrome-stable", 775 "flags": ["group1", "group2"] 776 } 777 } 778 }, 779 browser_lookup_override=self.browser_lookup, 780 args=self.mock_args) 781 self.assertEqual(len(config.variants), 3) 782 for browser in config.variants: 783 assert isinstance(browser, mock_browser.MockChromeStable) 784 self.assertEqual(browser.app_path, 785 mock_browser.MockChromeStable.mock_app_path()) 786 expected_flags = ( 787 "--bar=v1 --foo=w2", 788 "--bar=v1 --foo=w2 --js-flags=--max-opt=1,--trace-ic", 789 "--bar=v1 --foo=w2 --js-flags=--max-opt=2,--log-all", 790 ) 791 self.verify_variant_flags(config.variants, expected_flags) 792 793 def test_flag_combination_js_flags_combinations_invalid(self): 794 with self.assertRaises(ConfigError) as cm: 795 _ = BrowserVariantsConfig( 796 { 797 "flags": { 798 "group1": { 799 "--js-flags": [ 800 None, "--max-opt=2,--trace-ic", 801 "--max-opt=3 --log-all" 802 ], 803 }, 804 "group2": { 805 "default": "--js-flags=--no-sparkplug" 806 } 807 }, 808 "browsers": { 809 "chrome-stable": { 810 "path": "chrome-stable", 811 "flags": ["group1", "group2"] 812 } 813 } 814 }, 815 browser_lookup_override=self.browser_lookup, 816 args=self.mock_args) 817 self.assertIn("--js-flags", str(cm.exception)) 818 819 def test_flag_group_combination(self): 820 config = BrowserVariantsConfig( 821 { 822 "flags": { 823 "group1": { 824 "--foo": [None, "", "v1"], 825 }, 826 "group2": { 827 "--bar": [None, "", "w1"], 828 }, 829 "group3": { 830 "--other": ["x1", "x2"], 831 } 832 }, 833 "browsers": { 834 "chrome-stable": { 835 "path": "chrome-stable", 836 "flags": ["group1", "group2", "group3"] 837 } 838 } 839 }, 840 browser_lookup_override=self.browser_lookup, 841 args=self.mock_args) 842 self.assertEqual(len(config.variants), 3 * 3 * 2) 843 expected_flags = ( 844 "--other=x1", 845 "--other=x2", 846 "--bar --other=x1", 847 "--bar --other=x2", 848 "--bar=w1 --other=x1", 849 "--bar=w1 --other=x2", 850 "--foo --other=x1", 851 "--foo --other=x2", 852 "--foo --bar --other=x1", 853 "--foo --bar --other=x2", 854 "--foo --bar=w1 --other=x1", 855 "--foo --bar=w1 --other=x2", 856 "--foo=v1 --other=x1", 857 "--foo=v1 --other=x2", 858 "--foo=v1 --bar --other=x1", 859 "--foo=v1 --bar --other=x2", 860 "--foo=v1 --bar=w1 --other=x1", 861 "--foo=v1 --bar=w1 --other=x2", 862 ) 863 self.verify_variant_flags(config.variants, expected_flags) 864 865 def test_from_cli_args_browser_config(self): 866 if self.platform.is_win: 867 self.skipTest("No auto-download available on windows") 868 browser_cls = mock_browser.MockChromeStable 869 # TODO: migrate to with_stem once python 3.9 is available everywhere 870 suffix = browser_cls.mock_app_path().suffix 871 browser_bin = browser_cls.mock_app_path().with_name( 872 f"Custom Google Chrome{suffix}") 873 browser_cls.setup_bin(self.fs, browser_bin, "Chrome") 874 config_data = {"browsers": {"chrome-stable": {"path": str(browser_bin),}}} 875 config_file = pth.LocalPath("config.hjson") 876 with config_file.open("w", encoding="utf-8") as f: 877 hjson.dump(config_data, f) 878 879 args = mock.Mock( 880 network=NetworkConfig.default(), 881 browser=None, 882 browser_config=config_file, 883 driver_path=None) 884 with mock.patch.object( 885 BrowserVariantsConfig, "get_browser_cls", return_value=browser_cls): 886 config = BrowserVariantsConfig.from_cli_args(args) 887 self.assertEqual(len(config.variants), 1) 888 browser = config.variants[0] 889 self.assertIsInstance(browser, browser_cls) 890 self.assertEqual(browser.app_path, browser_bin) 891 892 def test_from_cli_args_browser(self): 893 if self.platform.is_win: 894 self.skipTest("No auto-download available on windows") 895 browser_cls = mock_browser.MockChromeStable 896 # TODO: migrate to with_stem once python 3.9 is available everywhere 897 suffix = browser_cls.mock_app_path().suffix 898 browser_bin = browser_cls.mock_app_path().with_name( 899 f"Custom Google Chrome{suffix}") 900 browser_cls.setup_bin(self.fs, browser_bin, "Chrome") 901 args = mock.Mock( 902 network=NetworkConfig.default(), 903 browser=[ 904 BrowserConfig(browser_bin), 905 ], 906 browser_config=None, 907 enable_features=None, 908 disable_features=None, 909 driver_path=None, 910 js_flags=None, 911 other_browser_args=[]) 912 with mock.patch.object( 913 BrowserVariantsConfig, "get_browser_cls", return_value=browser_cls): 914 config = BrowserVariantsConfig.from_cli_args(args) 915 self.assertEqual(len(config.variants), 1) 916 browser = config.variants[0] 917 self.assertIsInstance(browser, browser_cls) 918 self.assertEqual(browser.app_path, browser_bin) 919 920 def test_from_cli_args_browser_additional_flags(self): 921 browser_cls = mock_browser.MockChromeStable 922 args = mock.Mock( 923 network=NetworkConfig.default(), 924 browser=[ 925 BrowserConfig.parse_str("chrome"), 926 ], 927 browser_config=None, 928 driver_path=None, 929 enable_features="feature_on", 930 disable_features="feature_off", 931 js_flags=None, 932 other_browser_args=["--no-sandbox", "--enable-logging=stderr"]) 933 with mock.patch.object( 934 BrowserVariantsConfig, "get_browser_cls", return_value=browser_cls): 935 config = BrowserVariantsConfig.from_cli_args(args) 936 self.assertEqual(len(config.variants), 1) 937 browser = config.variants[0] 938 self.assertIsInstance(browser, browser_cls) 939 self.assertFalse(browser.js_flags) 940 self.assertEqual(browser.flags["--enable-features"], "feature_on") 941 self.assertEqual(browser.flags["--disable-features"], "feature_off") 942 self.assertIn("--no-sandbox", browser.flags) 943 self.assertEqual(browser.flags["--enable-logging"], "stderr") 944 945 def test_from_cli_args_browser_js_flags(self): 946 browser_cls = mock_browser.MockChromeStable 947 args = mock.Mock( 948 network=NetworkConfig.default(), 949 browser=[ 950 BrowserConfig.parse_str("chrome"), 951 ], 952 browser_config=None, 953 driver_path=None, 954 enable_features=None, 955 disable_features=None, 956 js_flags=["--max-opt=1"], 957 other_browser_args=[]) 958 with mock.patch.object( 959 BrowserVariantsConfig, "get_browser_cls", return_value=browser_cls): 960 config = BrowserVariantsConfig.from_cli_args(args) 961 self.assertEqual(len(config.variants), 1) 962 browser = config.variants[0] 963 self.assertIsInstance(browser, browser_cls) 964 self.assertEqual(browser.js_flags.to_dict(), {"--max-opt": "1"}) 965 966 def test_from_cli_args_browser_extra_browser_js_flags(self): 967 browser_cls = mock_browser.MockChromeStable 968 args = mock.Mock( 969 network=NetworkConfig.default(), 970 browser=[ 971 BrowserConfig.parse_str("chrome"), 972 ], 973 browser_config=None, 974 driver_path=None, 975 enable_features=None, 976 disable_features=None, 977 js_flags=[], 978 other_browser_args=["--js-flags=--max-opt=1,--log-all"]) 979 with mock.patch.object( 980 BrowserVariantsConfig, "get_browser_cls", return_value=browser_cls): 981 config = BrowserVariantsConfig.from_cli_args(args) 982 self.assertEqual(len(config.variants), 1) 983 browser = config.variants[0] 984 self.assertIsInstance(browser, browser_cls) 985 self.assertEqual(browser.js_flags.to_dict(), { 986 "--max-opt": "1", 987 "--log-all": None 988 }) 989 990 def test_from_cli_args_browser_multiple_js_flags(self): 991 browser_cls = mock_browser.MockChromeStable 992 args = mock.Mock( 993 network=NetworkConfig.default(), 994 browser=[ 995 BrowserConfig.parse_str("chrome"), 996 ], 997 browser_config=None, 998 driver_path=None, 999 enable_features="feature_on", 1000 disable_features="feature_off", 1001 js_flags=["--max-opt=1", "--max-opt=2,--log-all"], 1002 other_browser_args=["--no-sandbox", "--enable-logging=stderr"]) 1003 with mock.patch.object( 1004 BrowserVariantsConfig, "get_browser_cls", return_value=browser_cls): 1005 config = BrowserVariantsConfig.from_cli_args(args) 1006 self.assertEqual(len(config.variants), 2) 1007 browser_0 = config.variants[0] 1008 self.assertIsInstance(browser_0, browser_cls) 1009 self.assertEqual(browser_0.js_flags.to_dict(), {"--max-opt": "1"}) 1010 browser_1 = config.variants[1] 1011 self.assertIsInstance(browser_1, browser_cls) 1012 self.assertEqual(browser_1.js_flags.to_dict(), { 1013 "--max-opt": "2", 1014 "--log-all": None 1015 }) 1016 1017 for browser in config.variants: 1018 self.assertEqual(browser.flags["--enable-features"], "feature_on") 1019 self.assertEqual(browser.flags["--disable-features"], "feature_off") 1020 self.assertIn("--no-sandbox", browser.flags) 1021 self.assertEqual(browser.flags["--enable-logging"], "stderr") 1022 1023 def test_from_cli_args_browser_config_network_override(self): 1024 ts_proxy_path = pth.LocalPath("/tsproxy/tsproxy.py") 1025 self.fs.create_file(ts_proxy_path, st_size=100) 1026 browser_config_dict = { 1027 "browsers": { 1028 "default-network": { 1029 "path": "chrome-stable", 1030 "network": "default" 1031 }, 1032 "default": "chrome-stable", 1033 "custom-network": { 1034 "path": "chrome-stable", 1035 "network": "4G" 1036 } 1037 } 1038 } 1039 config_file = pth.LocalPath("browsers.config.json") 1040 with config_file.open("w", encoding="utf-8") as f: 1041 json.dump(browser_config_dict, f) 1042 network_3g = NetworkConfig.parse("3G-slow") 1043 network_4g = NetworkConfig.parse("4G") 1044 self.assertNotEqual(network_3g.speed.in_kbps, network_4g.speed.in_kbps) 1045 args = mock.Mock( 1046 browser=None, 1047 browser_config=config_file, 1048 network=network_3g, 1049 enable_features=None, 1050 disable_features=None, 1051 driver_path=None, 1052 js_flags=None, 1053 other_browser_args=[]) 1054 1055 with mock.patch.object( 1056 BrowserVariantsConfig, 1057 "get_browser_cls", 1058 return_value=mock_browser.MockChromeStable 1059 ), mock.patch( 1060 "crossbench.network.traffic_shaping.ts_proxy.TsProxyFinder") as finder: 1061 finder.return_value = mock.Mock(path=ts_proxy_path) 1062 config = BrowserVariantsConfig.from_cli_args(args,) 1063 self.assertEqual(len(config.variants), 3) 1064 browser_1, browser_2, browser_3 = config.variants # pylint: disable=unbalanced-tuple-unpacking 1065 # Browser 1 provides an explicit default override: 1066 self.assertTrue(browser_1.network.is_live) 1067 self.assertTrue(browser_1.network.traffic_shaper.is_live) 1068 # Browser 2: uses the default --network: 1069 self.assertTrue(browser_2.network.is_live) 1070 self.assertFalse(browser_2.network.traffic_shaper.is_live) 1071 self.assertEqual(browser_2.network.traffic_shaper.ts_proxy.in_kbps, 1072 network_3g.speed.in_kbps) 1073 # Browser 3; Uses an explicit 4G override: 1074 self.assertTrue(browser_3.network.is_live) 1075 self.assertFalse(browser_3.network.traffic_shaper.is_live) 1076 self.assertEqual(browser_3.network.traffic_shaper.ts_proxy.in_kbps, 1077 network_4g.speed.in_kbps) 1078 1079 def test_get_browser_cls_unsupported(self): 1080 variants = BrowserVariantsConfig() 1081 with self.assertRaisesRegex(argparse.ArgumentTypeError, 1082 "Unsupported browser"): 1083 config = BrowserConfig(browser=pth.AnyPath("your/custom/browser.exe")) 1084 variants.get_browser_cls(config) 1085 1086 def test_get_browser_cls_chrome_default(self): 1087 variants = BrowserVariantsConfig() 1088 config = BrowserConfig(browser=pth.AnyPath("Chrome.app")) 1089 self.assertIs(variants.get_browser_cls(config), ChromeWebDriver) 1090 config = BrowserConfig(browser=pth.AnyPath("Chrome.exe")) 1091 self.assertIs(variants.get_browser_cls(config), ChromeWebDriver) 1092 1093 def test_get_browser_cls_chromium_default(self): 1094 variants = BrowserVariantsConfig() 1095 config = BrowserConfig(browser=pth.AnyPath("Chromium.app")) 1096 self.assertIs(variants.get_browser_cls(config), ChromiumWebDriver) 1097 config = BrowserConfig(browser=pth.AnyPath("Chromium.exe")) 1098 self.assertIs(variants.get_browser_cls(config), ChromiumWebDriver) 1099 1100 def test_get_browser_cls_chrome_driver_types(self): 1101 variants = BrowserVariantsConfig() 1102 expected_classes = ( 1103 (BrowserDriverType.APPLE_SCRIPT, ChromeAppleScript), 1104 (BrowserDriverType.WEB_DRIVER, ChromeWebDriver), 1105 (BrowserDriverType.LINUX_SSH, ChromeWebDriverSsh), 1106 ) 1107 for driver_type, browser_cls in expected_classes: 1108 config = BrowserConfig( 1109 browser=pth.AnyPath("Chrome.bin"), 1110 driver=DriverConfig(type=driver_type)) 1111 self.assertIs(variants.get_browser_cls(config), browser_cls) 1112 1113 def test_get_browser_cls_chromium_driver_types(self): 1114 variants = BrowserVariantsConfig() 1115 expected_classes = ( 1116 (BrowserDriverType.APPLE_SCRIPT, ChromiumAppleScript), 1117 (BrowserDriverType.WEB_DRIVER, ChromiumWebDriver), 1118 (BrowserDriverType.LINUX_SSH, ChromiumWebDriverSsh), 1119 ) 1120 for driver_type, browser_cls in expected_classes: 1121 config = BrowserConfig( 1122 browser=pth.AnyPath("Chromium.bin"), 1123 driver=DriverConfig(type=driver_type)) 1124 self.assertIs(variants.get_browser_cls(config), browser_cls) 1125 1126 def test_get_browser_cls_chromium_android_default(self): 1127 self.platform.sh_results = [ 1128 ADB_DEVICES_SINGLE_OUTPUT, 1129 ] 1130 variants = BrowserVariantsConfig() 1131 config = BrowserConfig( 1132 browser=pth.AnyPath("chromium.apk"), 1133 driver=DriverConfig(type=BrowserDriverType.ANDROID)) 1134 self.assertIs(variants.get_browser_cls(config), ChromiumWebDriverAndroid) 1135 1136 def test_get_browser_cls_chrome_android_default(self): 1137 self.platform.sh_results = [ 1138 ADB_DEVICES_SINGLE_OUTPUT, 1139 ] 1140 variants = BrowserVariantsConfig() 1141 config = BrowserConfig( 1142 browser=pth.AnyPath("chrome.apk"), 1143 driver=DriverConfig(type=BrowserDriverType.ANDROID)) 1144 self.assertIs(variants.get_browser_cls(config), ChromeWebDriverAndroid) 1145 1146 def test_get_browser_cls_chrome_android_local_helper(self): 1147 self.platform.sh_results = [ 1148 ADB_DEVICES_SINGLE_OUTPUT, 1149 ] 1150 variants = BrowserVariantsConfig() 1151 apk_helper = pth.AnyPath("/home/user/Documents/chrome/src/" 1152 "out/arm64.apk/bin/chrome_public_apk") 1153 config = BrowserConfig( 1154 browser=apk_helper, driver=DriverConfig(type=BrowserDriverType.ANDROID)) 1155 self.assertIs(variants.get_browser_cls(config), LocalChromeWebDriverAndroid) 1156 1157 def test_get_browser_cls_chromium_android_local_helper(self): 1158 """Currently there is no nice way to distinguish a local build between 1159 chrome/chromium.""" 1160 1161 def test_get_browser_cls_chromeos_ssh_default(self): 1162 self.platform.sh_results = [] 1163 variants = BrowserVariantsConfig() 1164 with mock.patch.object( 1165 DriverConfig, "validate_chromeos", return_value=None) as mock_method: 1166 driver = DriverConfig(type=BrowserDriverType.CHROMEOS_SSH) 1167 mock_method.assert_called_once() 1168 config = BrowserConfig(browser=pth.AnyPath("chrome"), driver=driver) 1169 self.assertIs(variants.get_browser_cls(config), ChromeWebDriverChromeOsSsh) 1170 1171 1172if __name__ == "__main__": 1173 test_helper.run_pytest(__file__) 1174