1# Copyright 2024 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5from __future__ import annotations 6 7import argparse 8import datetime as dt 9import json 10import pathlib 11import unittest 12from typing import Sequence 13 14import hjson 15 16from crossbench.action_runner.action.action_type import ActionType 17from crossbench.action_runner.action.click import ClickAction 18from crossbench.benchmarks.loading.config.login.google import GoogleLogin 19from crossbench.benchmarks.loading.config.page import PageConfig 20from crossbench.benchmarks.loading.config.pages import ( 21 DevToolsRecorderPagesConfig, ListPagesConfig, PagesConfig) 22from crossbench.cli.config.secret_type import SecretType 23from crossbench.cli.config.secrets import Secret, SecretsConfig 24from tests import test_helper 25from tests.crossbench.base import CrossbenchFakeFsTestCase 26 27 28class PagesConfigTestCase(CrossbenchFakeFsTestCase): 29 30 def test_parse_unknown_type(self): 31 with self.assertRaises(argparse.ArgumentTypeError) as cm: 32 PagesConfig.parse(self) 33 self.assertIn("type", str(cm.exception)) 34 35 def test_parse_invalid(self): 36 with self.assertRaises(argparse.ArgumentTypeError) as cm: 37 PagesConfig.parse("123s,") 38 self.assertIn("Duration", str(cm.exception)) 39 with self.assertRaises(argparse.ArgumentTypeError) as cm: 40 PagesConfig.parse(",") 41 self.assertIn("empty", str(cm.exception)) 42 with self.assertRaises(argparse.ArgumentTypeError) as cm: 43 PagesConfig.parse("http://foo.com,,") 44 self.assertIn("empty", str(cm.exception)) 45 with self.assertRaises(argparse.ArgumentTypeError) as cm: 46 PagesConfig.parse("http://foo.com,123s,") 47 self.assertIn("empty", str(cm.exception)) 48 with self.assertRaises(argparse.ArgumentTypeError) as cm: 49 PagesConfig.parse("http://foo.com,123s,123s") 50 self.assertIn("Duration", str(cm.exception)) 51 52 def test_parse_single(self): 53 config = PagesConfig.parse("http://a.com") 54 self.assertEqual(len(config.pages), 1) 55 page_config = config.pages[0] 56 self.assertEqual(page_config.first_url, "http://a.com") 57 58 def test_parse_single_with_duration(self): 59 config = PagesConfig.parse("http://a.com,123s") 60 self.assertEqual(len(config.pages), 1) 61 page_config = config.pages[0] 62 self.assertEqual(page_config.first_url, "http://a.com") 63 self.assertEqual(page_config.duration.total_seconds(), 123) 64 65 def test_parse_multiple(self): 66 config = PagesConfig.parse("http://a.com,http://b.com") 67 self.assertEqual(len(config.pages), 2) 68 page_config_0, page_config_1 = config.pages 69 self.assertEqual(page_config_0.first_url, "http://a.com") 70 self.assertEqual(page_config_1.first_url, "http://b.com") 71 72 def test_parse_multiple_short_domain(self): 73 config = PagesConfig.parse("a.com,b.com") 74 self.assertEqual(len(config.pages), 2) 75 page_config_0, page_config_1 = config.pages 76 self.assertEqual(page_config_0.first_url, "https://a.com") 77 self.assertEqual(page_config_1.first_url, "https://b.com") 78 79 def test_parse_multiple_numeric_domain(self): 80 config = PagesConfig.parse("111.a.com,222.b.com") 81 self.assertEqual(len(config.pages), 2) 82 page_config_0, page_config_1 = config.pages 83 self.assertEqual(page_config_0.first_url, "https://111.a.com") 84 self.assertEqual(page_config_1.first_url, "https://222.b.com") 85 86 def test_parse_multiple_numeric_domain_with_duration(self): 87 config = PagesConfig.parse("111.a.com,12s,222.b.com,23s") 88 self.assertEqual(len(config.pages), 2) 89 page_config_0, page_config_1 = config.pages 90 self.assertEqual(page_config_0.first_url, "https://111.a.com") 91 self.assertEqual(page_config_1.first_url, "https://222.b.com") 92 self.assertEqual(page_config_0.duration.total_seconds(), 12) 93 self.assertEqual(page_config_1.duration.total_seconds(), 23) 94 95 def test_parse_multiple_with_duration(self): 96 config = PagesConfig.parse("http://a.com,123s,http://b.com") 97 self.assertEqual(len(config.pages), 2) 98 page_config_0, page_config_1 = config.pages 99 self.assertEqual(page_config_0.first_url, "http://a.com") 100 self.assertEqual(page_config_1.first_url, "http://b.com") 101 self.assertEqual(page_config_0.duration.total_seconds(), 123) 102 self.assertEqual(page_config_1.duration, dt.timedelta()) 103 104 def test_parse_multiple_with_duration_end(self): 105 config = PagesConfig.parse("http://a.com,http://b.com,123s") 106 self.assertEqual(len(config.pages), 2) 107 page_config_0, page_config_1 = config.pages 108 self.assertEqual(page_config_0.first_url, "http://a.com") 109 self.assertEqual(page_config_1.first_url, "http://b.com") 110 self.assertEqual(page_config_0.duration, dt.timedelta()) 111 self.assertEqual(page_config_1.duration.total_seconds(), 123) 112 113 def test_parse_multiple_with_duration_all(self): 114 config = PagesConfig.parse("http://a.com,1s,http://b.com,123s") 115 self.assertEqual(len(config.pages), 2) 116 page_config_0, page_config_1 = config.pages 117 self.assertEqual(page_config_0.first_url, "http://a.com") 118 self.assertEqual(page_config_1.first_url, "http://b.com") 119 self.assertEqual(page_config_0.duration.total_seconds(), 1) 120 self.assertEqual(page_config_1.duration.total_seconds(), 123) 121 122 def test_parse_sequence(self): 123 config_list = PagesConfig.parse(["http://a.com,1s", "http://b.com,123s"]) 124 config_str = PagesConfig.parse("http://a.com,1s,http://b.com,123s") 125 self.assertEqual(config_list, config_str) 126 127 config_list = PagesConfig.parse(["http://a.com", "http://b.com"]) 128 config_str = PagesConfig.parse("http://a.com,http://b.com") 129 self.assertEqual(config_list, config_str) 130 131 def test_parse_empty_actions(self): 132 config_data = {"pages": {"Google Story": []}} 133 with self.assertRaises(argparse.ArgumentTypeError) as cm: 134 PagesConfig.parse(config_data) 135 self.assertIn("empty", str(cm.exception).lower()) 136 config_data = {"pages": {"Google Story": {}}} 137 with self.assertRaises(argparse.ArgumentTypeError) as cm: 138 PagesConfig.parse(config_data) 139 self.assertIn("empty", str(cm.exception).lower()) 140 141 def test_parse_empty_missing_get_action(self): 142 config_data = { 143 "pages": { 144 "Google Story": [{ 145 "action": "wait", 146 "duration": 5 147 }] 148 } 149 } 150 with self.assertRaises(argparse.ArgumentTypeError) as cm: 151 PagesConfig.parse(config_data) 152 self.assertIn("get", str(cm.exception).lower()) 153 154 def test_example(self): 155 config_data = { 156 "pages": { 157 "Google Story": [ 158 { 159 "action": "get", 160 "url": "https://www.google.com" 161 }, 162 { 163 "action": "wait", 164 "duration": 5 165 }, 166 { 167 "action": "scroll", 168 "direction": "down", 169 "duration": 3 170 }, 171 ], 172 } 173 } 174 config = PagesConfig.parse(config_data) 175 self.assert_single_google_story(config.pages) 176 self.assertIsNone(config.pages[0].login) 177 # Loading the same config from a file should result in the same actions. 178 file = pathlib.Path("page.config.hjson") 179 assert not file.exists() 180 with file.open("w", encoding="utf-8") as f: 181 hjson.dump(config_data, f) 182 pages = PagesConfig.parse(str(file)).pages 183 self.assert_single_google_story(pages) 184 self.assertIsNone(config.pages[0].login) 185 186 def test_example_with_login(self): 187 config_data = { 188 "pages": { 189 "Google Story": { 190 "login": [{ 191 "action": "get", 192 "url": "https://www.google.com/login" 193 },], 194 "actions": [ 195 { 196 "action": "get", 197 "url": "https://www.google.com" 198 }, 199 { 200 "action": "wait", 201 "duration": 5 202 }, 203 { 204 "action": "scroll", 205 "direction": "down", 206 "duration": 3 207 }, 208 ] 209 }, 210 } 211 } 212 config = PagesConfig.parse(config_data) 213 self.assert_single_google_story(config.pages) 214 login = config.pages[0].login 215 self.assertEqual(len(login.actions), 1) 216 self.assertEqual(login.actions[0].url, "https://www.google.com/login") 217 218 def test_example_with_login_preset(self): 219 config_data = { 220 "pages": { 221 "Google Story": { 222 "login": 223 "google", 224 "actions": [ 225 { 226 "action": "get", 227 "url": "https://www.google.com" 228 }, 229 { 230 "action": "wait", 231 "duration": 5 232 }, 233 { 234 "action": "scroll", 235 "direction": "down", 236 "duration": 3 237 }, 238 ] 239 }, 240 } 241 } 242 config = PagesConfig.parse(config_data) 243 self.assert_single_google_story(config.pages) 244 page = config.pages[0] 245 self.assertIsInstance(page.login, GoogleLogin) 246 self.assertIsNone(page.setup) 247 248 def assert_single_google_story(self, pages: Sequence[PageConfig]): 249 self.assertTrue(len(pages), 1) 250 page = pages[0] 251 self.assertEqual(page.label, "Google Story") 252 self.assertEqual(page.first_url, "https://www.google.com") 253 self.assertEqual(len(page.blocks), 1) 254 block = page.blocks[0] 255 self.assertListEqual([str(action.TYPE) for action in block], 256 ["get", "wait", "scroll"]) 257 258 def test_secrets(self): 259 config_data = { 260 "secrets": { 261 "google": { 262 "username": "test", 263 "password": "s3cr3t" 264 } 265 }, 266 "pages": { 267 "Google Story": ["http://google.com"], 268 } 269 } 270 pages = PagesConfig.parse(config_data) 271 secret = Secret(SecretType.GOOGLE, "test", "s3cr3t") 272 self.assertEqual(pages.secrets, SecretsConfig({secret.type: secret})) 273 self.assertEqual(pages.pages[0].first_url, "http://google.com") 274 275 def test_no_scenarios(self): 276 with self.assertRaises(argparse.ArgumentTypeError): 277 PagesConfig.parse_dict({}) 278 with self.assertRaises(argparse.ArgumentTypeError): 279 PagesConfig.parse_dict({"pages": {}}) 280 281 def test_scenario_invalid_actions(self): 282 invalid_actions = [None, "", [], {}, "invalid string", 12] 283 invalid_actions = ["invalid string", 12] 284 for invalid_action in invalid_actions: 285 config_dict = {"pages": {"name": invalid_action}} 286 with self.subTest(invalid_action=invalid_action): 287 with self.assertRaises(argparse.ArgumentTypeError): 288 PagesConfig.parse_dict(config_dict) 289 290 def test_missing_action(self): 291 with self.assertRaises(argparse.ArgumentTypeError) as cm: 292 PagesConfig.parse_dict( 293 {"pages": { 294 "TEST": [{ 295 "action___": "wait", 296 "duration": 5.0 297 }] 298 }}) 299 self.assertIn("Invalid data:", str(cm.exception)) 300 301 def test_invalid_action(self): 302 invalid_actions = [None, "", [], {}, "unknown action name", 12] 303 for invalid_action in invalid_actions: 304 config_dict = { 305 "pages": { 306 "TEST": [{ 307 "action": invalid_action, 308 "duration": 5.0 309 }] 310 } 311 } 312 with self.subTest(invalid_action=invalid_action): 313 with self.assertRaises(argparse.ArgumentTypeError): 314 PagesConfig.parse_dict(config_dict) 315 316 def test_missing_get_action_scenario(self): 317 with self.assertRaises(argparse.ArgumentTypeError): 318 PagesConfig.parse_dict( 319 {"pages": { 320 "TEST": [{ 321 "action": "wait", 322 "duration": 5.0 323 }] 324 }}) 325 326 def test_get_action_durations(self): 327 durations = [ 328 ("5", 5), 329 ("5.5", 5.5), 330 (6, 6), 331 (6.1, 6.1), 332 ("5.5", 5.5), 333 ("170ms", 0.17), 334 ("170milliseconds", 0.17), 335 ("170.4ms", 0.1704), 336 ("170.4 millis", 0.1704), 337 ("8s", 8), 338 ("8.1s", 8.1), 339 ("8.1seconds", 8.1), 340 ("1 second", 1), 341 ("1.1 seconds", 1.1), 342 ("9m", 9 * 60), 343 ("9.5m", 9.5 * 60), 344 ("9.5 minutes", 9.5 * 60), 345 ("9.5 mins", 9.5 * 60), 346 ("1 minute", 60), 347 ("1 min", 60), 348 ("1h", 3600), 349 ("1 h", 3600), 350 ("1 hour", 3600), 351 ("0.5h", 1800), 352 ("0.5 hours", 1800), 353 ] 354 for input_value, duration in durations: 355 with self.subTest(duration=duration): 356 page_config = PagesConfig.parse_dict({ 357 "pages": { 358 "TEST": [ 359 { 360 "action": "get", 361 "url": "google.com" 362 }, 363 { 364 "action": "wait", 365 "duration": input_value 366 }, 367 ] 368 } 369 }) 370 self.assertEqual(len(page_config.pages), 1) 371 page = page_config.pages[0] 372 self.assertEqual(len(page.blocks), 1) 373 actions = page.blocks[0].actions 374 self.assertEqual(len(actions), 2) 375 self.assertEqual(actions[1].duration, dt.timedelta(seconds=duration)) 376 377 def test_action_invalid_duration(self): 378 invalid_durations = [ 379 "1.1.1", None, "", -1, "-1", "-1ms", "1msss", "1ss", "2hh", "asdfasd", 380 "---", "1.1.1", "1_123ms", "1'200h", (), [], {}, "-1h" 381 ] 382 for invalid_duration in invalid_durations: 383 with self.subTest(duration=invalid_duration), self.assertRaises( 384 (AssertionError, ValueError, argparse.ArgumentTypeError)): 385 PagesConfig.parse_dict({ 386 "pages": { 387 "TEST": [ 388 { 389 "action": "get", 390 "url": "google.com" 391 }, 392 { 393 "action": "wait", 394 "duration": invalid_duration 395 }, 396 ] 397 } 398 }) 399 400 401DEVTOOLS_RECORDER_EXAMPLE = { 402 "title": 403 "cnn load", 404 "steps": [ 405 { 406 "type": "setViewport", 407 "width": 1628, 408 "height": 397, 409 "deviceScaleFactor": 1, 410 "isMobile": False, 411 "hasTouch": False, 412 "isLandscape": False 413 }, 414 { 415 "type": 416 "navigate", 417 "url": 418 "https://edition.cnn.com/", 419 "assertedEvents": [{ 420 "type": "navigation", 421 "url": "https://edition.cnn.com/", 422 "title": "" 423 }] 424 }, 425 { 426 "type": "click", 427 "target": "main", 428 "selectors": [["aria/Opinion"], 429 [ 430 "#pageHeader > div > div > " 431 "div.header__container div:nth-of-type(5) > a" 432 ], 433 [ 434 "xpath///*[@id=\"pageHeader\"]/" 435 "div/div/div[1]/div[1]/nav/div/div[5]/a" 436 ], 437 [ 438 "pierce/#pageHeader > div > div > " 439 "div.header__container div:nth-of-type(5) > a" 440 ]], 441 "offsetY": 17, 442 "offsetX": 22.515625 443 }, 444 ] 445} 446 447 448class DevToolsRecorderPageConfigTestCase(CrossbenchFakeFsTestCase): 449 450 def test_invalid(self): 451 with self.assertRaises(argparse.ArgumentTypeError) as cm: 452 DevToolsRecorderPagesConfig.parse({}) 453 self.assertIn("empty", str(cm.exception)) 454 455 def test_missing_title(self): 456 with self.assertRaises(argparse.ArgumentTypeError) as cm: 457 DevToolsRecorderPagesConfig.parse({"foo": {}}) 458 self.assertIn("title", str(cm.exception)) 459 460 def test_basic_config(self): 461 config = DevToolsRecorderPagesConfig.parse(DEVTOOLS_RECORDER_EXAMPLE) 462 self.assertEqual(len(config.pages), 1) 463 page = config.pages[0] 464 self.assertEqual(page.label, "cnn load") 465 self.assertEqual(page.first_url, "https://edition.cnn.com/") 466 self.assertEqual(len(page.blocks), 1) 467 self.assertGreater(len(page.blocks[0].actions), 1) 468 469 def test_basic_config_from_file(self): 470 config_path = pathlib.Path("devtools.config.json") 471 with config_path.open("w", encoding="utf-8") as f: 472 json.dump(DEVTOOLS_RECORDER_EXAMPLE, f) 473 config_file = DevToolsRecorderPagesConfig.parse(config_path) 474 config_dict = DevToolsRecorderPagesConfig.parse(DEVTOOLS_RECORDER_EXAMPLE) 475 self.assertEqual(config_file, config_dict) 476 477 def test_parse_click_step(self): 478 config = { 479 "type": "click", 480 "target": "main", 481 "selectors": [["aria/Search Google"],], 482 } 483 actions = DevToolsRecorderPagesConfig.parse_step(config) 484 self.assertEqual(len(actions), 1) 485 action = actions[0] 486 self.assertEqual(action.TYPE, ActionType.CLICK) 487 assert isinstance(action, ClickAction) 488 self.assertEqual(action.selector, "[aria-label='Search Google']") 489 490 config["selectors"] = [["aria/SIMPLE"], ["#rso > div:nth-of-type(3) h3"], 491 ["xpath///*[@id=\"rso\"]"], 492 ["pierce/#rso > div:nth-of-type(3) h3"], 493 ["text/SIMPLE"]] 494 action = DevToolsRecorderPagesConfig.parse_step(config)[0] 495 assert isinstance(action, ClickAction) 496 self.assertEqual(action.selector, "xpath///*[@id=\"rso\"]") 497 498 config["selectors"] = [ 499 ["aria/SIMPLE"], 500 ["css/#rso > div:nth-of-type(3) h3"], 501 ] 502 action = DevToolsRecorderPagesConfig.parse_step(config)[0] 503 assert isinstance(action, ClickAction) 504 self.assertEqual(action.selector, "#rso > div:nth-of-type(3) h3") 505 506 config["selectors"] = [ 507 ["#rso > div:nth-of-type(3) h3"], 508 ] 509 action = DevToolsRecorderPagesConfig.parse_step(config)[0] 510 assert isinstance(action, ClickAction) 511 self.assertEqual(action.selector, "#rso > div:nth-of-type(3) h3") 512 513 config["selectors"] = [ 514 ["aria/SIMPLE", "area/OTHER"], 515 ["#rso > div:nth-of-type(3) h3"], 516 ] 517 action = DevToolsRecorderPagesConfig.parse_step(config)[0] 518 assert isinstance(action, ClickAction) 519 self.assertEqual(action.selector, "#rso > div:nth-of-type(3) h3") 520 521 config["selectors"] = [ 522 ["text/Content"], 523 ] 524 action = DevToolsRecorderPagesConfig.parse_step(config)[0] 525 assert isinstance(action, ClickAction) 526 self.assertEqual(action.selector, "xpath///*[text()='Content']") 527 528 529class ListPageConfigTestCase(CrossbenchFakeFsTestCase): 530 531 def test_invalid(self): 532 with self.assertRaises(argparse.ArgumentTypeError) as cm: 533 ListPagesConfig.parse({}) 534 self.assertIn("empty", str(cm.exception)) 535 with self.assertRaises(argparse.ArgumentTypeError) as cm: 536 ListPagesConfig.parse({"foo": {}}) 537 self.assertIn("pages", str(cm.exception)) 538 539 with self.assertRaises(argparse.ArgumentTypeError) as cm: 540 ListPagesConfig.parse_dict({"pages": None}) 541 self.assertIn("None", str(cm.exception)) 542 543 with self.assertRaises(argparse.ArgumentTypeError) as cm: 544 ListPagesConfig.parse_dict({"pages": []}) 545 self.assertIn("empty", str(cm.exception)) 546 547 def test_direct_string_single(self): 548 with self.assertRaises(argparse.ArgumentTypeError) as cm: 549 ListPagesConfig.parse("http://foo.bar.com,23s") 550 self.assertIn("http://foo.bar.com,23s", str(cm.exception)) 551 552 def test_direct_string_single_dict(self): 553 config_dict = ListPagesConfig.parse({"pages": "http://foo.bar.com,23s"}) 554 config_str = PagesConfig( 555 pages=(PageConfig.parse("http://foo.bar.com,23s"),)) 556 self.assertEqual(config_dict, config_str) 557 558 @unittest.skip("Combined pages per line not supported yet") 559 def test_direct_string_multiple(self): 560 config = ListPagesConfig.parse_dict( 561 {"pages": "http://a.com,12s,http://b.com,13s"}) 562 self.assertEqual(len(config.pages), 2) 563 story_1, story_2 = config.pages 564 self.assertEqual(story_1.first_url, "http://a.com") 565 self.assertEqual(story_2.first_url, "http://b.com") 566 self.assertEqual(story_1.duration.total_seconds(), 12) 567 self.assertEqual(story_2.duration.total_seconds(), 13) 568 569 def test_list(self): 570 page_configs = ["http://a.com,12s", "http://b.com,13s"] 571 config_str = PagesConfig.parse("http://a.com,12s,http://b.com,13s") 572 config_dict_list = ListPagesConfig.parse({"pages": page_configs}) 573 config_list = ListPagesConfig.parse(page_configs) 574 self.assertEqual(config_str, config_dict_list) 575 self.assertEqual(config_str, config_list) 576 577 def test_parse_file(self): 578 page_configs = ["http://a.com,12s", "http://b.com,13s"] 579 config_file = pathlib.Path("page_list.txt") 580 with config_file.open("w", encoding="utf-8") as f: 581 f.write("\n".join(page_configs)) 582 config_file = ListPagesConfig.parse(config_file) 583 config_list = ListPagesConfig.parse(page_configs) 584 self.assertEqual(config_file, config_list) 585 586 def test_parse_file_empty_lines(self): 587 page_configs = ["http://a.com,12s", "http://b.com,13s"] 588 config_file = pathlib.Path("page_list.txt") 589 with config_file.open("w", encoding="utf-8") as f: 590 f.write("\n") 591 f.write(page_configs[0]) 592 f.write("\n\n") 593 f.write(page_configs[1]) 594 f.write("\n\n") 595 config_file = ListPagesConfig.parse(config_file) 596 config_list = ListPagesConfig.parse(page_configs) 597 self.assertEqual(config_file, config_list) 598 599 600if __name__ == "__main__": 601 test_helper.run_pytest(__file__) 602