1#!/usr/bin/env vpython3 2# Copyright 2021 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import argparse 7import contextlib 8import json 9import logging 10import os 11import posixpath 12import re 13import shutil 14import subprocess 15import sys 16import tempfile 17import time 18 19from collections import OrderedDict 20from PIL import Image 21 22SRC_DIR = os.path.abspath( 23 os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) 24PAR_DIR = os.path.join(SRC_DIR, 'testing') 25OUT_DIR = os.path.join(SRC_DIR, 'out', 'Release') 26BLINK_DIR = os.path.join(SRC_DIR, 'third_party', 'blink') 27BLINK_TOOLS = os.path.join(BLINK_DIR, 'tools') 28BLINK_WEB_TESTS = os.path.join(BLINK_DIR, 'web_tests') 29BUILD_ANDROID = os.path.join(SRC_DIR, 'build', 'android') 30CATAPULT_DIR = os.path.join(SRC_DIR, 'third_party', 'catapult') 31PYUTILS = os.path.join(CATAPULT_DIR, 'common', 'py_utils') 32 33# Protocall buffer directories to import 34PYPROTO_LIB = os.path.join(OUT_DIR, 'pyproto', 'google') 35WEBVIEW_VARIATIONS_PROTO = os.path.join(OUT_DIR, 'pyproto', 36 'android_webview', 'proto') 37 38if PYUTILS not in sys.path: 39 sys.path.append(PYUTILS) 40 41if BUILD_ANDROID not in sys.path: 42 sys.path.append(BUILD_ANDROID) 43 44if BLINK_TOOLS not in sys.path: 45 sys.path.append(BLINK_TOOLS) 46 47if PYPROTO_LIB not in sys.path: 48 sys.path.append(PYPROTO_LIB) 49 50if WEBVIEW_VARIATIONS_PROTO not in sys.path: 51 sys.path.append(WEBVIEW_VARIATIONS_PROTO) 52 53sys.path.append(PAR_DIR) 54 55if 'compile_targets' not in sys.argv: 56 import aw_variations_seed_pb2 57 58import devil_chromium 59 60from blinkpy.common.host import Host 61from blinkpy.common.path_finder import PathFinder 62from blinkpy.web_tests.models import test_failures 63from blinkpy.web_tests.port.android import ( 64 ANDROID_WEBVIEW, CHROME_ANDROID) 65from blinkpy.w3c.wpt_results_processor import WPTResultsProcessor 66 67from devil import devil_env 68from devil.android import apk_helper 69from devil.android import device_temp_file 70from devil.android import flag_changer 71from devil.android import logcat_monitor 72from devil.android.tools import script_common 73from devil.android.tools import system_app 74from devil.android.tools import webview_app 75from devil.utils import logging_common 76from pylib.local.device import local_device_environment 77from pylib.local.emulator import avd 78from py_utils.tempfile_ext import NamedTemporaryDirectory 79from scripts import common 80from skia_gold_infra.finch_skia_gold_properties import FinchSkiaGoldProperties 81from skia_gold_infra import finch_skia_gold_session_manager 82from skia_gold_infra import finch_skia_gold_utils 83from run_wpt_tests import get_device 84 85ANDROID_WEBLAYER = 'android_weblayer' 86LOGCAT_TAG = 'finch_test_runner_py' 87LOGCAT_FILTERS = [ 88 'chromium:v', 89 'cr_*:v', 90 'DEBUG:I', 91 'StrictMode:D', 92 'WebView*:v', 93 '%s:I' % LOGCAT_TAG 94] 95logger = logging.getLogger(__name__) 96logger.setLevel(logging.INFO) 97TEST_CASES = {} 98 99def _is_version_greater_than_or_equal(version1, version2): 100 version1_parts = version1.split('.') 101 version2_parts = version2.split('.') 102 for i in range(4): 103 comp = int(version1_parts[i]) - int(version2_parts[i]) 104 if comp != 0: 105 return comp > 0 106 107 return True 108 109 110def _merge_results_dicts(dict_to_merge, test_results_dict): 111 if 'actual' in dict_to_merge: 112 test_results_dict.update(dict_to_merge) 113 return 114 for key in dict_to_merge.keys(): 115 _merge_results_dicts(dict_to_merge[key], 116 test_results_dict.setdefault(key, {})) 117 118 119# pylint: disable=super-with-arguments, abstract-method 120class FinchTestCase(common.BaseIsolatedScriptArgsAdapter): 121 122 def __init__(self, device): 123 self.host = Host() 124 self.fs = self.host.filesystem 125 self.path_finder = PathFinder(self.fs) 126 self.port = self.host.port_factory.get() 127 super(FinchTestCase, self).__init__() 128 self._add_extra_arguments() 129 self._parser = self._override_options(self._parser) 130 self._include_filename = None 131 self.layout_test_results_subdir = 'layout-test-results' 132 self._device = device 133 self.parse_args() 134 self.port.set_option_default('target', self.options.target) 135 self._browser_apk_helper = apk_helper.ToHelper(self.options.browser_apk) 136 137 self.browser_package_name = self._browser_apk_helper.GetPackageName() 138 self.browser_activity_name = (self.options.browser_activity_name or 139 self.default_browser_activity_name) 140 self.layout_test_results_subdir = None 141 self.test_specific_browser_args = [] 142 if self.options.webview_provider_apk: 143 self.webview_provider_package_name = ( 144 apk_helper.GetPackageName(self.options.webview_provider_apk)) 145 146 # Initialize the Skia Gold session manager 147 self._skia_gold_corpus = 'finch-smoke-tests' 148 self._skia_gold_tmp_dir = None 149 self._skia_gold_session_manager = None 150 151 @classmethod 152 def app_user_sub_dir(cls): 153 """Returns sub directory within user directory""" 154 return 'app_%s' % cls.product_name() 155 156 @classmethod 157 def product_name(cls): 158 raise NotImplementedError 159 160 @property 161 def tests(self): 162 return [ 163 'dom/collections/HTMLCollection-delete.html', 164 'dom/collections/HTMLCollection-supported-property-names.html', 165 'dom/collections/HTMLCollection-supported-property-indices.html', 166 ] 167 168 @property 169 def pixel_tests(self): 170 return [] 171 172 @property 173 def default_browser_activity_name(self): 174 raise NotImplementedError 175 176 @property 177 def default_finch_seed_path(self): 178 raise NotImplementedError 179 180 @classmethod 181 def finch_seed_download_args(cls): 182 return [] 183 184 def generate_test_output_args(self, output): 185 return ['--log-chromium=%s' % output] 186 187 def generate_test_filter_args(self, test_filter_str): 188 included_tests, excluded_tests = \ 189 self._resolve_tests_from_isolate_filter(test_filter_str) 190 include_file, self._include_filename = self.fs.open_text_tempfile() 191 with include_file: 192 for test in included_tests: 193 include_file.write(test) 194 include_file.write('\n') 195 wpt_args = ['--include-file=%s' % self._include_filename] 196 for test in excluded_tests: 197 wpt_args.append('--exclude=%s' % test) 198 return wpt_args 199 200 def _override_options(self, base_parser): 201 """Create a parser that overrides existing options. 202 203 `argument.ArgumentParser` can extend other parsers and override their 204 options, with the caveat that the child parser only inherits options 205 that the parent had at the time of the child's initialization. 206 207 See Also: 208 https://docs.python.org/3/library/argparse.html#parents 209 """ 210 parser = argparse.ArgumentParser( 211 parents=[base_parser], 212 # Allow overriding existing options in the parent parser. 213 conflict_handler='resolve', 214 epilog=('All unrecognized arguments are passed through ' 215 "to wptrunner. Use '--wpt-help' to see wptrunner's usage."), 216 ) 217 parser.add_argument( 218 '--isolated-script-test-repeat', 219 '--repeat', 220 '--gtest_repeat', 221 metavar='REPEAT', 222 type=int, 223 default=1, 224 help='Number of times to run the tests') 225 parser.add_argument( 226 '--isolated-script-test-launcher-retry-limit', 227 '--test-launcher-retry-limit', 228 '--retry-unexpected', 229 metavar='RETRIES', 230 type=int, 231 help=( 232 'Maximum number of times to rerun unexpectedly failed tests. ' 233 'Defaults to 3 unless given an explicit list of tests to run.')) 234 # `--gtest_filter` and `--isolated-script-test-filter` have slightly 235 # different formats and behavior, so keep them as separate options. 236 # See: crbug/1316164#c4 237 238 # TODO(crbug.com/1356318): This is a temporary hack to hide the 239 # inherited '--xvfb' option and force Xvfb to run always. 240 parser.add_argument('--xvfb', action='store_true', default=True, 241 help=argparse.SUPPRESS) 242 return parser 243 244 def generate_test_repeat_args(self, repeat_count): 245 return ['--repeat=%d' % repeat_count] 246 247 def generate_test_launcher_retry_limit_args(self, retry_limit): 248 return ['--retry-unexpected=%d' % retry_limit] 249 250 def generate_sharding_args(self, total_shards, shard_index): 251 return ['--total-chunks=%d' % total_shards, 252 # shard_index is 0-based but WPT's this-chunk to be 1-based 253 '--this-chunk=%d' % (shard_index + 1), 254 # The default sharding strategy is to shard by directory. But 255 # we want to hash each test to determine which shard runs it. 256 # This allows running individual directories that have few 257 # tests across many shards. 258 '--chunk-type=hash'] 259 260 def clean_up_after_test_run(self): 261 if self._include_filename: 262 self.fs.remove(self._include_filename) 263 264 def new_seed_downloaded(self): 265 # TODO(crbug.com/1285152): Implement seed download test 266 # for Chrome and WebLayer. 267 return True 268 269 def enable_internet(self): 270 self._device.RunShellCommand( 271 ['settings', 'put', 'global', 'airplane_mode_on', '0']) 272 self._device.RunShellCommand( 273 ['am', 'broadcast', '-a', 274 'android.intent.action.AIRPLANE_MODE']) 275 self._device.RunShellCommand(['svc', 'wifi', 'enable']) 276 self._device.RunShellCommand(['svc', 'data', 'enable']) 277 278 def disable_internet(self): 279 self._device.RunShellCommand( 280 ['settings', 'put', 'global', 'airplane_mode_on', '1']) 281 self._device.RunShellCommand( 282 ['am', 'broadcast', '-a', 283 'android.intent.action.AIRPLANE_MODE']) 284 285 @contextlib.contextmanager 286 def _archive_logcat(self, filename, endpoint_name): 287 start_point = 'START {}'.format(endpoint_name) 288 end_point = 'END {}'.format(endpoint_name) 289 with logcat_monitor.LogcatMonitor( 290 self._device.adb, 291 filter_specs=LOGCAT_FILTERS, 292 output_file=filename, 293 check_error=False): 294 try: 295 self._device.RunShellCommand(['log', '-p', 'i', '-t', LOGCAT_TAG, 296 start_point], 297 check_return=True) 298 yield 299 finally: 300 self._device.RunShellCommand(['log', '-p', 'i', '-t', LOGCAT_TAG, 301 end_point], 302 check_return=True) 303 304 def parse_args(self, args=None): 305 super(FinchTestCase, self).parse_args(args) 306 if (not self.options.finch_seed_path or 307 not os.path.exists(self.options.finch_seed_path)): 308 logger.warning('Could not find the finch seed passed ' 309 'as the argument for --finch-seed-path. ' 310 'Running tests on the default finch seed') 311 self.options.finch_seed_path = self.default_finch_seed_path 312 313 @property 314 def output_directory(self): 315 return self.path_finder.path_from_chromium_base('out', 316 self.options.target) 317 318 @property 319 def mojo_js_directory(self): 320 return self.fs.join(self.output_directory, 'gen') 321 322 @property 323 def wpt_output(self): 324 return self.options.isolated_script_test_output 325 326 @property 327 def _raw_log_path(self): 328 return self.fs.join(self.output_directory, 'finch-smoke-raw-events.log') 329 330 def __enter__(self): 331 self._device.EnableRoot() 332 # Run below commands to ensure that the device can download a seed 333 self.disable_internet() 334 self._device.adb.Emu(['power', 'ac', 'on']) 335 self._skia_gold_tmp_dir = tempfile.mkdtemp() 336 self._skia_gold_session_manager = ( 337 finch_skia_gold_session_manager.FinchSkiaGoldSessionManager( 338 self._skia_gold_tmp_dir, FinchSkiaGoldProperties(self.options))) 339 return self 340 341 def __exit__(self, exc_type, exc_val, exc_tb): 342 self._skia_gold_session_manager = None 343 if self._skia_gold_tmp_dir: 344 shutil.rmtree(self._skia_gold_tmp_dir) 345 self._skia_gold_tmp_dir = None 346 347 @property 348 def rest_args(self): 349 unknown_args = super(FinchTestCase, self).rest_args 350 351 rest_args = list() 352 353 rest_args.extend(self.wpt_rest_args(unknown_args)) 354 355 rest_args.extend([ 356 '--webdriver-arg=--disable-build-check', 357 '--device-serial', 358 self._device.serial, 359 '--webdriver-binary', 360 os.path.join('clang_x64', 'chromedriver'), 361 '--symbols-path', 362 self.output_directory, 363 '--package-name', 364 self.browser_package_name, 365 '--keep-app-data-directory', 366 '--test-type=testharness', 367 ]) 368 369 for binary_arg in self.browser_command_line_args(): 370 rest_args.append('--binary-arg=%s' % binary_arg) 371 372 for test in self.tests: 373 rest_args.extend(['--include', test]) 374 375 return rest_args 376 377 @property 378 def wpt_binary(self): 379 default_wpt_binary = os.path.join( 380 common.SRC_DIR, "third_party", "wpt_tools", "wpt", "wpt") 381 return os.environ.get("WPT_BINARY", default_wpt_binary) 382 383 @property 384 def wpt_root_dir(self): 385 return self.path_finder.path_from_web_tests( 386 self.path_finder.wpt_prefix()) 387 388 @property 389 def _wpt_run_args(self): 390 """The start of a 'wpt run' command.""" 391 return [ 392 self.wpt_binary, 393 # Use virtualenv packages installed by vpython, not wpt. 394 '--venv=%s' % self.path_finder.chromium_base(), 395 '--skip-venv-setup', 396 'run', 397 ] 398 399 def process_and_upload_results(self, test_name_prefix): 400 processor = WPTResultsProcessor( 401 self.host.filesystem, 402 self.port, 403 artifacts_dir=os.path.join(os.path.dirname(self.wpt_output), 404 self.layout_test_results_subdir), 405 test_name_prefix=test_name_prefix) 406 processor.recreate_artifacts_dir() 407 with self.fs.open_text_file_for_reading(self._raw_log_path) as raw_logs: 408 for event in map(json.loads, raw_logs): 409 if event.get('action') != 'shutdown': 410 processor.process_event(event) 411 processor.process_results_json(self.wpt_output) 412 413 def wpt_rest_args(self, unknown_args): 414 rest_args = list(self._wpt_run_args) 415 rest_args.extend([ 416 '--no-pause-after-test', 417 '--no-capture-stdio', 418 '--no-manifest-download', 419 '--tests=%s' % self.wpt_root_dir, 420 '--metadata=%s' % self.wpt_root_dir, 421 '--mojojs-path=%s' % self.mojo_js_directory, 422 '--log-raw=%s' % self._raw_log_path, 423 ]) 424 425 if self.options.default_exclude: 426 rest_args.extend(['--default-exclude']) 427 428 if self.options.verbose >= 3: 429 rest_args.extend([ 430 '--log-mach=-', 431 '--log-mach-level=debug', 432 '--log-mach-verbose', 433 ]) 434 if self.options.verbose >= 4: 435 rest_args.extend([ 436 '--webdriver-arg=--verbose', 437 '--webdriver-arg="--log-path=-"', 438 ]) 439 440 rest_args.append(self.wpt_product_name()) 441 # We pass through unknown args as late as possible so that they can 442 # override earlier options. It also allows users to pass test names as 443 # positional args, which must not have option strings between them. 444 for unknown_arg in unknown_args: 445 # crbug/1274933#c14: Some developers had used the end-of-options 446 # marker '--' to pass through arguments to wptrunner. 447 # crrev.com/c/3573284 makes this no longer necessary. 448 if unknown_arg == '--': 449 logger.warning( 450 'Unrecognized options will automatically fall through ' 451 'to wptrunner.') 452 logger.warning( 453 "There is no need to use the end-of-options marker '--'.") 454 else: 455 rest_args.append(unknown_arg) 456 return rest_args 457 458 @classmethod 459 def add_common_arguments(cls, parser): 460 parser.add_argument('--test-case', 461 choices=TEST_CASES.keys(), 462 # TODO(rmhasan): Remove default values after 463 # adding arguments to test suites. Also make 464 # this argument required. 465 default='webview', 466 help='Name of test case') 467 parser.add_argument('--finch-seed-path', 468 type=os.path.realpath, 469 help='Path to the finch seed') 470 parser.add_argument('--browser-apk', 471 '--webview-shell-apk', 472 '--weblayer-shell-apk', 473 help='Path to the browser apk', 474 type=os.path.realpath, 475 required=True) 476 parser.add_argument('--webview-provider-apk', 477 type=os.path.realpath, 478 help='Path to the WebView provider apk') 479 parser.add_argument('--additional-apk', 480 action='append', 481 type=os.path.realpath, 482 default=[], 483 help='List of additional apk\'s to install') 484 parser.add_argument('--browser-activity-name', 485 action='store', 486 help='Browser activity name') 487 parser.add_argument('--use-webview-installer-tool', 488 action='store_true', 489 help='Use the WebView installer tool.') 490 parser.add_argument('--fake-variations-channel', 491 action='store', 492 default='stable', 493 choices=['dev', 'canary', 'beta', 'stable'], 494 help='Finch seed release channel') 495 parser.add_argument('-j', 496 '--processes', 497 type=lambda processes: max(0, int(processes)), 498 default=1, 499 help='Number of emulator to run.') 500 common.add_emulator_args(parser) 501 # Add arguments used by Skia Gold. 502 FinchSkiaGoldProperties.AddCommandLineArguments(parser) 503 504 def _add_extra_arguments(self): 505 parser = self._parser 506 parser.add_argument( 507 '-t', 508 '--target', 509 default='Release', 510 help='Target build subdirectory under //out') 511 parser.add_argument( 512 '--default-exclude', 513 action='store_true', 514 help=('Only run the tests explicitly given in arguments ' 515 '(can run no tests, which will exit with code 0)')) 516 parser.add_argument( 517 '-v', 518 '--verbose', 519 action='count', 520 default=0, 521 help='Increase verbosity') 522 self.add_product_specific_argument_groups(parser) 523 self.add_common_arguments(parser) 524 525 @classmethod 526 def add_product_specific_argument_groups(cls, _): 527 pass 528 529 def _compare_screenshots_with_baselines(self, all_pixel_tests_results_dict): 530 """Compare pixel tests screenshots with baselines stored in skia gold 531 532 Args: 533 all_pixel_tests_results_dict: Results dictionary for all pixel tests 534 535 Returns: 536 1 if there was an error comparing images otherwise 0 537 """ 538 skia_gold_session = ( 539 self._skia_gold_session_manager.GetSkiaGoldSession( 540 {'platform': 'android'}, self._skia_gold_corpus)) 541 542 def _process_test_leaf(test_result_dict): 543 if ('artifacts' not in test_result_dict or 544 'actual_image' not in test_result_dict['artifacts']): 545 return 0 546 547 return_code = 0 548 artifacts_dict = test_result_dict['artifacts'] 549 curr_artifacts = list(artifacts_dict.keys()) 550 for artifact_name in curr_artifacts: 551 artifact_path = artifacts_dict[artifact_name][0] 552 # Compare screenshots to baselines stored in Skia Gold 553 status, error = skia_gold_session.RunComparison( 554 artifact_path, 555 os.path.join(os.path.dirname(self.wpt_output), artifact_path)) 556 557 if status: 558 test_result_dict['actual'] = 'FAIL' 559 all_pixel_tests_results_dict['num_failures_by_type'].setdefault( 560 'FAIL', 0) 561 all_pixel_tests_results_dict['num_failures_by_type']['FAIL'] += 1 562 triage_link = finch_skia_gold_utils.log_skia_gold_status_code( 563 skia_gold_session, artifact_path, status, error) 564 if triage_link: 565 artifacts_dict['%s_triage_link' % artifact_name] = [triage_link] 566 return_code = 1 567 else: 568 test_result_dict['actual'] = 'PASS' 569 570 return return_code 571 572 def _process_tests(node): 573 return_code = 0 574 if 'actual' in node: 575 return _process_test_leaf(node) 576 for next_node in node.values(): 577 return_code |= _process_tests(next_node) 578 return return_code 579 580 return _process_tests(all_pixel_tests_results_dict['tests']) 581 582 @contextlib.contextmanager 583 def install_apks(self): 584 """Install apks for testing""" 585 self._device.Uninstall(self.browser_package_name) 586 self._device.Install(self.options.browser_apk, reinstall=True) 587 for apk_path in self.options.additional_apk: 588 self._device.Install(apk_path) 589 590 self._device.ClearApplicationState( 591 self.browser_package_name, 592 permissions=self._browser_apk_helper.GetPermissions()) 593 594 # TODO(rmhasan): For R+ test devices, store the files in the 595 # app's data directory. This is needed for R+ devices because 596 # of the scoped storage feature. 597 tests_root_dir = posixpath.join(self._device.GetExternalStoragePath(), 598 'chromium_tests_root') 599 local_device_environment.place_nomedia_on_device(self._device, 600 tests_root_dir) 601 602 # Store screenshot tests on the device's external storage. 603 for test_file in self.pixel_tests: 604 self._device.RunShellCommand( 605 ['mkdir', '-p', 606 posixpath.join(tests_root_dir, 607 'pixel_tests', 608 posixpath.dirname(test_file))], 609 check_return=True) 610 self._device.adb.Push(os.path.join(BLINK_WEB_TESTS, test_file), 611 posixpath.join(tests_root_dir, 612 'pixel_tests', 613 test_file)) 614 615 yield 616 617 def browser_command_line_args(self): 618 return (['--vmodule=variations_field_trial_creator.cc=1', '--v=1', 619 '--disable-field-trial-config', 620 '--fake-variations-channel=%s' % 621 self.options.fake_variations_channel] + 622 self.test_specific_browser_args) 623 624 def run_tests(self, test_run_variation, all_test_results_dict, 625 extra_browser_args=None, check_seed_loaded=False): 626 """Run browser test on test device 627 628 Args: 629 test_run_variation: Test run variation. 630 all_test_results_dict: Main results dictionary containing 631 results for all test variations. 632 extra_browser_args: Extra browser arguments. 633 check_seed_loaded: Check if the finch seed was loaded. 634 635 Returns: 636 The return code of all tests. 637 """ 638 isolate_root_dir = os.path.dirname( 639 self.options.isolated_script_test_output) 640 logcat_filename = '{}_{}_test_run_logcat.txt'.format( 641 self.product_name(), test_run_variation) 642 self.layout_test_results_subdir = ('%s_smoke_test_artifacts' % 643 test_run_variation) 644 self.test_specific_browser_args = extra_browser_args or [] 645 646 with self._archive_logcat(os.path.join(isolate_root_dir, logcat_filename), 647 '{} {} tests'.format(self.product_name(), 648 test_run_variation)): 649 # Make sure the browser is not running before the tests run 650 self.stop_browser() 651 652 if self.tests: 653 ret = super(FinchTestCase, self).run_test() 654 self.stop_browser() 655 656 command_line_file = '%s-command-line' % self.product_name() 657 # Set the browser command line file 658 with flag_changer.CustomCommandLineFlags( 659 self._device, command_line_file, self.browser_command_line_args()): 660 # Run screen shot tests 661 pixel_tests_results_dict, pixel_tests_ret = self._run_pixel_tests() 662 ret |= pixel_tests_ret 663 664 seed_loaded_result_dict = {'num_failures_by_type': {}, 'tests': {}} 665 666 test_harness_results_dict = {'num_failures_by_type': {}, 'tests': {}} 667 # If wpt tests are not run then the file path stored in self.wpt_output 668 # was not created. That is why this check exists. 669 if os.path.exists(self.wpt_output): 670 self.process_and_upload_results(test_run_variation) 671 672 with open(self.wpt_output, 'r') as test_harness_results: 673 test_harness_results_dict = json.load(test_harness_results) 674 # If there are wpt results then add the the test name prefix to the 675 # results metadata dictionary so that the test name prefix is added 676 # to the test name in test results UI. 677 test_harness_results_dict['metadata'] = {'test_name_prefix': 678 test_run_variation} 679 with open(self.wpt_output, 'w+') as test_results_file: 680 json.dump(test_harness_results_dict, test_results_file) 681 682 final_logcat_path = os.path.join(isolate_root_dir, 683 self.layout_test_results_subdir, 684 logcat_filename) 685 os.makedirs(os.path.dirname(final_logcat_path), exist_ok=True) 686 shutil.move(os.path.join(isolate_root_dir, logcat_filename), 687 final_logcat_path) 688 if check_seed_loaded: 689 # Check in the logcat if the seed was loaded 690 ret |= self._finch_seed_loaded(final_logcat_path, seed_loaded_result_dict) 691 692 for test_results_dict in (test_harness_results_dict, 693 pixel_tests_results_dict, 694 seed_loaded_result_dict): 695 _merge_results_dicts( 696 test_results_dict['tests'], 697 all_test_results_dict['tests'].setdefault(test_run_variation, {})) 698 699 for result, count in test_results_dict['num_failures_by_type'].items(): 700 all_test_results_dict['num_failures_by_type'].setdefault(result, 0) 701 all_test_results_dict['num_failures_by_type'][result] += count 702 703 return ret 704 705 def _finch_seed_loaded(self, logcat_path, all_results_dict): 706 raise NotImplementedError 707 708 def _run_pixel_tests(self): 709 """Run pixel tests on device 710 711 Returns: 712 A tuple containing a dictionary of pixel test results 713 and the skia gold status code. 714 """ 715 tests_root_dir = posixpath.join( 716 self._device.GetExternalStoragePath(), 717 'chromium_tests_root', 718 'pixel_tests') 719 720 pixel_tests_results_dict = {'tests':{}, 'num_failures_by_type': {}} 721 for test_file in self.pixel_tests: 722 logger.info('Running pixel test %s', test_file) 723 try: 724 # The test result will for each tests will be set after 725 # comparing the test screenshots to skia gold baselines. 726 url = 'file://{}'.format( 727 posixpath.join(tests_root_dir, test_file)) 728 self.start_browser(url) 729 730 screenshot_artifact_relpath = os.path.join( 731 'pixel_tests_artifacts', 732 self.layout_test_results_subdir.replace('_artifacts', ''), 733 self.port.output_filename(test_file, 734 test_failures.FILENAME_SUFFIX_ACTUAL, 735 '.png')) 736 screenshot_artifact_abspath = os.path.join( 737 os.path.dirname(self.options.isolated_script_test_output), 738 screenshot_artifact_relpath) 739 740 self._device.TakeScreenshot(host_path=screenshot_artifact_abspath) 741 742 # Crop away the Android status bar and the WebView shell's support 743 # action bar. We will do this by removing one fifth of the image 744 # from the top. 745 top_bar_height_factor = 0.2 746 747 # Crop away the bottom navigation bar from the screenshot. We can 748 # do this by cropping away one tenth of the image from the bottom. 749 navigation_bar_height_factor = 0.1 750 751 image = Image.open(screenshot_artifact_abspath) 752 width, height = image.size 753 cropped_image = image.crop( 754 (0, 755 int(height * top_bar_height_factor), 756 width, 757 int(height * (1 - navigation_bar_height_factor)))) 758 image.close() 759 cropped_image.save(screenshot_artifact_abspath) 760 761 test_results_dict = pixel_tests_results_dict['tests'] 762 for key in test_file.split('/'): 763 test_results_dict = test_results_dict.setdefault(key, {}) 764 765 test_results_dict['actual'] = 'PASS' 766 test_results_dict['expected'] = 'PASS' 767 test_results_dict['artifacts'] = { 768 'actual_image': [screenshot_artifact_relpath]} 769 finally: 770 self.stop_browser() 771 772 # Compare screenshots with baselines stored in Skia Gold. 773 return (pixel_tests_results_dict, 774 self._compare_screenshots_with_baselines(pixel_tests_results_dict)) 775 776 def stop_browser(self): 777 logger.info('Stopping package %s', self.browser_package_name) 778 self._device.ForceStop(self.browser_package_name) 779 if self.options.webview_provider_apk: 780 logger.info('Stopping package %s', self.webview_provider_package_name) 781 self._device.ForceStop( 782 self.webview_provider_package_name) 783 784 def start_browser(self, url=None): 785 full_activity_name = '%s/%s' % (self.browser_package_name, 786 self.browser_activity_name) 787 logger.info('Starting activity %s', full_activity_name) 788 url = url or 'www.google.com' 789 790 self._device.RunShellCommand([ 791 'am', 792 'start', 793 '-W', 794 '-n', 795 full_activity_name, 796 '-d', 797 url]) 798 logger.info('Waiting 5 seconds') 799 time.sleep(5) 800 801 def _wait_for_local_state_file(self, local_state_file): 802 """Wait for local state file to be generated""" 803 max_wait_time_secs = 120 804 delta_secs = 10 805 total_wait_time_secs = 0 806 807 self.start_browser() 808 809 while total_wait_time_secs < max_wait_time_secs: 810 if self._device.PathExists(local_state_file): 811 logger.info('Local state file generated') 812 self.stop_browser() 813 return 814 815 logger.info('Waiting %d seconds for the local state file to generate', 816 delta_secs) 817 time.sleep(delta_secs) 818 total_wait_time_secs += delta_secs 819 820 raise Exception('Timed out waiting for the ' 821 'local state file to be generated') 822 823 def install_seed(self): 824 """Install finch seed for testing 825 826 Returns: 827 The path to the new finch seed under the application data folder. 828 """ 829 app_data_dir = posixpath.join( 830 self._device.GetApplicationDataDirectory(self.browser_package_name), 831 self.app_user_sub_dir()) 832 833 device_local_state_file = posixpath.join(app_data_dir, 'Local State') 834 self._wait_for_local_state_file(device_local_state_file) 835 836 seed_path = posixpath.join(app_data_dir, 'local_variations_seed') 837 self._device.adb.Push(self.options.finch_seed_path, seed_path) 838 839 user_id = self._device.GetUidForPackage(self.browser_package_name) 840 self._device.RunShellCommand(['chown', user_id, seed_path], as_root=True) 841 842 return seed_path 843 844 845class ChromeFinchTestCase(FinchTestCase): 846 847 @classmethod 848 def product_name(cls): 849 """Returns name of product being tested""" 850 return 'chrome' 851 852 @property 853 def default_finch_seed_path(self): 854 return os.path.join(SRC_DIR, 'testing', 'scripts', 855 'variations_smoke_test_data', 856 'variations_seed_stable_chrome_android.json') 857 858 @classmethod 859 def wpt_product_name(cls): 860 return CHROME_ANDROID 861 862 @property 863 def default_browser_activity_name(self): 864 return 'org.chromium.chrome.browser.ChromeTabbedActivity' 865 866 867class WebViewFinchTestCase(FinchTestCase): 868 869 @classmethod 870 def product_name(cls): 871 """Returns name of product being tested""" 872 return 'webview' 873 874 @classmethod 875 def wpt_product_name(cls): 876 return ANDROID_WEBVIEW 877 878 @property 879 def pixel_tests(self): 880 return super(WebViewFinchTestCase, self).pixel_tests + [ 881 'external/wpt/svg/render/reftests/blending-001.svg', 882 'external/wpt/svg/render/reftests/blending-svg-foreign-object.html', 883 'external/wpt/svg/render/reftests/filter-effects-on-pattern.html', 884 'external/wpt/svg/pservers/reftests/radialgradient-basic-002.svg', 885 ] 886 887 def _finch_seed_loaded(self, logcat_path, all_results_dict): 888 """Checks the logcat if the seed was loaded 889 890 Args: 891 logcat_path: Path to the logcat. 892 all_results_dict: Dictionary containing test results 893 894 Returns: 895 0 if the seed was loaded and experiments were loaded for finch seeds 896 other than the default seed. Otherwise it returns 1. 897 """ 898 with open(logcat_path, 'r') as logcat: 899 logcat_content = logcat.read() 900 901 seed_loaded = 'cr_VariationsUtils: Loaded seed with age' in logcat_content 902 logcat_relpath = os.path.relpath(logcat_path, 903 os.path.dirname(self.wpt_output)) 904 seed_loaded_results_dict = ( 905 all_results_dict['tests'].setdefault( 906 'check_seed_loaded', 907 {'expected': 'PASS', 908 'artifacts': {'logcat_path': [logcat_relpath]}})) 909 910 if seed_loaded: 911 logger.info('The finch seed was loaded by WebView') 912 seed_loaded_results_dict['actual'] = 'PASS' 913 else: 914 logger.error('The finch seed was not loaded by WebView') 915 seed_loaded_results_dict['actual'] = 'FAIL' 916 all_results_dict['num_failures_by_type']['FAIL'] = 1 917 918 # If the value for the --finch-seed-path argument does not exist, then 919 # a default seed is consumed. The default seed may be too old to have it's 920 # experiments loaded. 921 if self.default_finch_seed_path != self.options.finch_seed_path: 922 # For WebView versions >= 110.0.5463.0 we should check for a new log 923 # message in the logcat that confirms that field trials were loaded 924 # from the seed. This message is guaranteed to be outputted when a valid 925 # seed is loaded. We check for this log for versions >= 110.0.5463.0 926 # because it is the first version of WebView that contains 927 # crrev.com/c/4076271. 928 webview_version = self._device.GetApplicationVersion( 929 self._device.GetWebViewProvider()) 930 check_for_vlog = (webview_version and 931 _is_version_greater_than_or_equal(webview_version, 932 '110.0.5463.0')) 933 field_trial_check_name = 'check_for_logged_field_trials' 934 935 if check_for_vlog: 936 # This log was added in crrev.com/c/4076271, which is part of the 937 # M110 milestone. 938 field_trials_loaded = ( 939 'CreateTrialsFromSeed complete with seed.version=' 940 in logcat_content) 941 field_trial_check_name = 'check_for_variations_field_trial_creator_logs' 942 expected_results = 'PASS' 943 logger.info("Checking for variations_field_trial_creator.cc logs " 944 "in the logcat") 945 else: 946 # Check for a field trial that is guaranteed to be activated by 947 # the finch seed. 948 field_trials_loaded = ('Active field trial ' 949 '"UMA-Uniformity-Trial-100-Percent" ' 950 'in group "group_01"') in logcat_content 951 # It is not guaranteed that the field trials will be logged. That 952 # is why this check is flaky. 953 expected_results = 'PASS FAIL' 954 logger.info("Checking for the UMA uniformity trial in the logcat") 955 956 field_trials_loaded_results_dict = ( 957 all_results_dict['tests'].setdefault( 958 field_trial_check_name, 959 {'expected': expected_results, 960 'artifacts': {'logcat_path': [logcat_relpath]}})) 961 962 if field_trials_loaded: 963 logger.info('Experiments were loaded from the finch seed by WebView') 964 field_trials_loaded_results_dict['actual'] = 'PASS' 965 else: 966 logger.error('Experiments were not loaded from ' 967 'the finch seed by WebView') 968 field_trials_loaded_results_dict['actual'] = 'FAIL' 969 all_results_dict['num_failures_by_type'].setdefault('FAIL', 0) 970 all_results_dict['num_failures_by_type']['FAIL'] += 1 971 972 if 'FAIL' in expected_results: 973 # If the check for field trial configs is flaky then only 974 # use the seed_loaded variable to set the return code. 975 return 0 if seed_loaded else 1 976 977 return 0 if seed_loaded and field_trials_loaded else 1 978 979 logger.warning('The default seed is being tested, ' 980 'skipping checks for active field trials') 981 return 0 if seed_loaded else 1 982 983 @classmethod 984 def finch_seed_download_args(cls): 985 return [ 986 '--finch-seed-expiration-age=0', 987 '--finch-seed-min-update-period=0', 988 '--finch-seed-min-download-period=0', 989 '--finch-seed-ignore-pending-download', 990 '--finch-seed-no-charging-requirement'] 991 992 @property 993 def default_browser_activity_name(self): 994 return 'org.chromium.webview_shell.WebViewBrowserActivity' 995 996 @property 997 def default_finch_seed_path(self): 998 return os.path.join(SRC_DIR, 'testing', 'scripts', 999 'variations_smoke_test_data', 1000 'webview_test_seed') 1001 1002 @classmethod 1003 def add_product_specific_argument_groups(cls, parser): 1004 installer_tool_group = parser.add_argument_group( 1005 'WebView Installer tool arguments') 1006 installer_tool_group.add_argument( 1007 '--webview-installer-tool', type=os.path.realpath, 1008 help='Path to the WebView installer tool') 1009 installer_tool_group.add_argument( 1010 '--chrome-version', '-V', type=str, default=None, 1011 help='Chrome version to install with the WebView installer tool') 1012 installer_tool_group.add_argument( 1013 '--channel', '-c', help='Channel build of WebView to install', 1014 choices=['dev', 'canary', 'beta', 'stable'], default=None) 1015 installer_tool_group.add_argument( 1016 '--milestone', '-M', help='Milestone build of WebView to install') 1017 installer_tool_group.add_argument( 1018 '--package', '-P', default=None, 1019 help='Name of the WebView apk to install') 1020 1021 1022 def new_seed_downloaded(self): 1023 """Checks if a new seed was downloaded 1024 1025 Returns: 1026 True if a new seed was downloaded, otherwise False 1027 """ 1028 app_data_dir = posixpath.join( 1029 self._device.GetApplicationDataDirectory(self.browser_package_name), 1030 self.app_user_sub_dir()) 1031 remote_seed_path = posixpath.join(app_data_dir, 'variations_seed') 1032 1033 with NamedTemporaryDirectory() as tmp_dir: 1034 current_seed_path = os.path.join(tmp_dir, 'current_seed') 1035 self._device.adb.Pull(remote_seed_path, current_seed_path) 1036 with open(current_seed_path, 'rb') as current_seed_obj, \ 1037 open(self.options.finch_seed_path, 'rb') as baseline_seed_obj: 1038 current_seed_content = current_seed_obj.read() 1039 baseline_seed_content = baseline_seed_obj.read() 1040 current_seed = aw_variations_seed_pb2.AwVariationsSeed.FromString( 1041 current_seed_content) 1042 baseline_seed = aw_variations_seed_pb2.AwVariationsSeed.FromString( 1043 baseline_seed_content) 1044 shutil.copy(current_seed_path, os.path.join(OUT_DIR, 'final_seed')) 1045 1046 logger.info("Downloaded seed's signature: %s", current_seed.signature) 1047 logger.info("Baseline seed's signature: %s", baseline_seed.signature) 1048 return current_seed_content != baseline_seed_content 1049 1050 def browser_command_line_args(self): 1051 return (super(WebViewFinchTestCase, self).browser_command_line_args() + 1052 ['--webview-verbose-logging']) 1053 1054 @contextlib.contextmanager 1055 def install_apks(self): 1056 """Install apks for testing""" 1057 with super(WebViewFinchTestCase, self).install_apks(): 1058 if self.options.use_webview_installer_tool: 1059 install_webview = self._install_webview_with_tool() 1060 else: 1061 install_webview = webview_app.UseWebViewProvider( 1062 self._device, self.options.webview_provider_apk) 1063 1064 with install_webview: 1065 yield 1066 1067 @contextlib.contextmanager 1068 def _install_webview_with_tool(self): 1069 """Install WebView with the WebView installer tool""" 1070 original_webview_provider = self._device.GetWebViewProvider() 1071 current_webview_provider = None 1072 1073 try: 1074 cmd = [self.options.webview_installer_tool, '-vvv', 1075 '--product', self.product_name()] 1076 assert (self.options.chrome_version or 1077 self.options.milestone or self.options.channel), ( 1078 'The --chrome-version, --milestone or --channel arguments must be ' 1079 'used when installing WebView with the WebView installer tool') 1080 assert not(self.options.chrome_version and self.options.milestone), ( 1081 'The --chrome-version and --milestone arguments cannot be ' 1082 'used together') 1083 1084 if self.options.chrome_version: 1085 cmd.extend(['--chrome-version', self.options.chrome_version]) 1086 elif self.options.milestone: 1087 cmd.extend(['--milestone', self.options.milestone]) 1088 1089 if self.options.channel: 1090 cmd.extend(['--channel', self.options.channel]) 1091 1092 if self.options.package: 1093 cmd.extend(['--package', self.options.package]) 1094 1095 exit_code = subprocess.call(cmd) 1096 assert exit_code == 0, ( 1097 'The WebView installer tool failed to install WebView') 1098 1099 current_webview_provider = self._device.GetWebViewProvider() 1100 yield 1101 finally: 1102 self._device.SetWebViewImplementation(original_webview_provider) 1103 # Restore the original webview provider 1104 if current_webview_provider: 1105 self._device.Uninstall(current_webview_provider) 1106 1107 def install_seed(self): 1108 """Install finch seed for testing 1109 1110 Returns: 1111 None 1112 """ 1113 logcat_file = os.path.join( 1114 os.path.dirname(self.options.isolated_script_test_output), 1115 'install_seed_for_on_device.txt') 1116 1117 with self._archive_logcat( 1118 logcat_file, 1119 'install seed on device {}'.format(self._device.serial)): 1120 app_data_dir = posixpath.join( 1121 self._device.GetApplicationDataDirectory(self.browser_package_name), 1122 self.app_user_sub_dir()) 1123 self._device.RunShellCommand(['mkdir', '-p', app_data_dir], 1124 run_as=self.browser_package_name) 1125 1126 seed_path = posixpath.join(app_data_dir, 'variations_seed') 1127 seed_new_path = posixpath.join(app_data_dir, 'variations_seed_new') 1128 seed_stamp = posixpath.join(app_data_dir, 'variations_stamp') 1129 1130 self._device.adb.Push(self.options.finch_seed_path, seed_path) 1131 self._device.adb.Push(self.options.finch_seed_path, seed_new_path) 1132 self._device.RunShellCommand( 1133 ['touch', seed_stamp], check_return=True, 1134 run_as=self.browser_package_name) 1135 1136 # We need to make the WebView shell package an owner of the seeds, 1137 # see crbug.com/1191169#c19 1138 user_id = self._device.GetUidForPackage(self.browser_package_name) 1139 logger.info('Setting owner of seed files to %r', user_id) 1140 self._device.RunShellCommand(['chown', user_id, seed_path], as_root=True) 1141 self._device.RunShellCommand( 1142 ['chown', user_id, seed_new_path], as_root=True) 1143 1144 1145class WebLayerFinchTestCase(FinchTestCase): 1146 1147 @classmethod 1148 def product_name(cls): 1149 """Returns name of product being tested""" 1150 return 'weblayer' 1151 1152 @classmethod 1153 def wpt_product_name(cls): 1154 return ANDROID_WEBLAYER 1155 1156 @property 1157 def default_browser_activity_name(self): 1158 return 'org.chromium.weblayer.shell.WebLayerShellActivity' 1159 1160 @property 1161 def default_finch_seed_path(self): 1162 return os.path.join(SRC_DIR, 'testing', 'scripts', 1163 'variations_smoke_test_data', 1164 'variations_seed_stable_weblayer.json') 1165 1166 @contextlib.contextmanager 1167 def install_apks(self): 1168 """Install apks for testing""" 1169 with super(WebLayerFinchTestCase, self).install_apks(), \ 1170 webview_app.UseWebViewProvider(self._device, 1171 self.options.webview_provider_apk): 1172 yield 1173 1174 1175def main(args): 1176 TEST_CASES.update( 1177 {p.product_name(): p 1178 for p in [ChromeFinchTestCase, WebViewFinchTestCase, 1179 WebLayerFinchTestCase]}) 1180 1181 # Unfortunately, there's a circular dependency between the parser made 1182 # available from `FinchTestCase.add_extra_arguments` and the selection of the 1183 # correct test case. The workaround is a second parser used in `main` only 1184 # that shares some arguments with the script adapter parser. The second parser 1185 # handles --help, so not all arguments are documented. Important arguments 1186 # added by the script adapter are re-added here for visibility. 1187 parser = argparse.ArgumentParser() 1188 FinchTestCase.add_common_arguments(parser) 1189 parser.add_argument( 1190 '--isolated-script-test-output', type=str, 1191 required=False, 1192 help='path to write test results JSON object to') 1193 1194 script_common.AddDeviceArguments(parser) 1195 script_common.AddEnvironmentArguments(parser) 1196 logging_common.AddLoggingArguments(parser) 1197 1198 for test_class in TEST_CASES.values(): 1199 test_class.add_product_specific_argument_groups(parser) 1200 1201 options, _ = parser.parse_known_args(args) 1202 1203 with get_device(options) as device, \ 1204 TEST_CASES[options.test_case](device) as test_case, \ 1205 test_case.install_apks(): 1206 devil_chromium.Initialize(adb_path=options.adb_path) 1207 logging_common.InitializeLogging(options) 1208 1209 # TODO(rmhasan): Best practice in Chromium is to allow users to provide 1210 # their own adb binary to avoid adb server restarts. We should add a new 1211 # command line argument to wptrunner so that users can pass the path to 1212 # their adb binary. 1213 platform_tools_path = os.path.dirname(devil_env.config.FetchPath('adb')) 1214 os.environ['PATH'] = os.pathsep.join([platform_tools_path] + 1215 os.environ['PATH'].split(os.pathsep)) 1216 1217 test_results_dict = OrderedDict({'version': 3, 'interrupted': False, 1218 'num_failures_by_type': {}, 'tests': {}}) 1219 1220 if test_case.product_name() == 'webview': 1221 ret = test_case.run_tests('without_finch_seed', test_results_dict) 1222 test_case.install_seed() 1223 ret |= test_case.run_tests('with_finch_seed', test_results_dict, 1224 check_seed_loaded=True) 1225 1226 # enable wifi so that a new seed can be downloaded from the finch server 1227 test_case.enable_internet() 1228 1229 # TODO(b/187185389): Figure out why WebView needs an extra restart 1230 # to fetch and load a new finch seed. 1231 ret |= test_case.run_tests( 1232 'extra_restart', test_results_dict, 1233 extra_browser_args=test_case.finch_seed_download_args(), 1234 check_seed_loaded=True) 1235 1236 # Restart webview+shell to fetch new seed to variations_seed_new 1237 ret |= test_case.run_tests( 1238 'fetch_new_seed_restart', test_results_dict, 1239 extra_browser_args=test_case.finch_seed_download_args(), 1240 check_seed_loaded=True) 1241 # Restart webview+shell to copy from 1242 # variations_seed_new to variations_seed 1243 ret |= test_case.run_tests( 1244 'load_new_seed_restart', test_results_dict, 1245 extra_browser_args=test_case.finch_seed_download_args(), 1246 check_seed_loaded=True) 1247 1248 # Disable wifi so that new updates will not be downloaded which can cause 1249 # timeouts in the adb commands run below. 1250 test_case.disable_internet() 1251 else: 1252 installed_seed = test_case.install_seed() 1253 # If the seed is placed in a local path, we can pass it from the command 1254 # line, e.g. for Android. 1255 if installed_seed: 1256 extra_args = [f'--variations-test-seed-path={installed_seed}'] 1257 ret = test_case.run_tests('with_finch_seed', test_results_dict, 1258 extra_browser_args=extra_args) 1259 else: 1260 ret = test_case.run_tests('with_finch_seed', test_results_dict) 1261 # Clears out the finch seed. Need to run finch_seed tests first. 1262 # See crbug/1305430 1263 device.ClearApplicationState(test_case.browser_package_name) 1264 ret |= test_case.run_tests('without_finch_seed', test_results_dict) 1265 1266 test_results_dict['seconds_since_epoch'] = int(time.time()) 1267 test_results_dict['path_delimiter'] = '/' 1268 1269 with open(test_case.options.isolated_script_test_output, 'w') as json_out: 1270 json_out.write(json.dumps(test_results_dict, indent=4)) 1271 1272 if not test_case.new_seed_downloaded(): 1273 raise Exception('A new seed was not downloaded') 1274 1275 # Return zero exit code if tests pass 1276 return ret 1277 1278 1279def main_compile_targets(args): 1280 json.dump([], args.output) 1281 1282 1283if __name__ == '__main__': 1284 if 'compile_targets' in sys.argv: 1285 funcs = { 1286 'run': None, 1287 'compile_targets': main_compile_targets, 1288 } 1289 sys.exit(common.run_script(sys.argv[1:], funcs)) 1290 sys.exit(main(sys.argv[1:])) 1291
