• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env vpython3
2# Copyright 2021 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""A smoke test to verify Chrome doesn't crash and basic rendering is functional
6when parsing a newly given variations seed.
7"""
8
9import argparse
10import http
11import json
12import logging
13import os
14import shutil
15from struct import pack
16import subprocess
17import sys
18import tempfile
19import time
20from functools import partial
21from http.server import SimpleHTTPRequestHandler
22from pkg_resources import packaging
23from threading import Thread
24
25import pkg_resources
26from skia_gold_infra.finch_skia_gold_properties import FinchSkiaGoldProperties
27from skia_gold_infra import finch_skia_gold_utils
28
29import variations_seed_access_helper as seed_helper
30
31_THIS_DIR = os.path.abspath(os.path.dirname(__file__))
32_VARIATIONS_TEST_DATA = 'variations_smoke_test_data'
33_VERSION_STRING = 'PRODUCT_VERSION'
34_FLAG_RELEASE_VERSION = packaging.version.parse('105.0.5176.3')
35
36
37# Add src/testing/ into sys.path for importing common without pylint errors.
38sys.path.append(
39    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
40from scripts import common
41
42from selenium import webdriver
43from selenium.webdriver import ChromeOptions
44from selenium.common.exceptions import NoSuchElementException
45from selenium.common.exceptions import WebDriverException
46
47# Constants for the waiting for seed from finch server
48_MAX_ATTEMPTS = 2
49_WAIT_TIMEOUT_IN_SEC = 0.5
50
51# Test cases to verify web elements can be rendered correctly.
52_TEST_CASES = [
53    {
54        # data:text/html,<h1 id="success">Success</h1>
55        'url': 'data:text/html,%3Ch1%20id%3D%22success%22%3ESuccess%3C%2Fh1%3E',
56        'expected_id': 'success',
57        'expected_text': 'Success',
58    },
59    {
60        'url': 'http://localhost:8000',
61        'expected_id': 'sites-chrome-userheader-title',
62        'expected_text': 'The Chromium Projects',
63        'skia_gold_image': 'finch_smoke_render_chromium_org_html',
64    },
65]
66
67
68def _get_httpd():
69  """Returns a HTTPServer instance."""
70  hostname = "localhost"
71  port = 8000
72  directory = os.path.join(_THIS_DIR, _VARIATIONS_TEST_DATA, "http_server")
73  httpd = None
74  handler = partial(SimpleHTTPRequestHandler, directory=directory)
75  httpd = http.server.HTTPServer((hostname, port), handler)
76  httpd.timeout = 0.5
77  httpd.allow_reuse_address = True
78  return httpd
79
80
81def _get_platform():
82  """Returns the host platform.
83
84  Returns:
85    One of 'linux', 'win' and 'mac'.
86  """
87  if sys.platform == 'win32' or sys.platform == 'cygwin':
88    return 'win'
89  if sys.platform.startswith('linux'):
90    return 'linux'
91  if sys.platform == 'darwin':
92    return 'mac'
93
94  raise RuntimeError(
95    'Unsupported platform: %s. Only Linux (linux*) and Mac (darwin) and '
96    'Windows (win32 or cygwin) are supported' % sys.platform)
97
98
99def _find_chrome_binary(): #pylint: disable=inconsistent-return-statements
100  """Finds and returns the relative path to the Chrome binary.
101
102  This function assumes that the CWD is the build directory.
103
104  Returns:
105    A relative path to the Chrome binary.
106  """
107  platform = _get_platform()
108  if platform == 'linux':
109    return os.path.join('.', 'chrome')
110  if platform == 'mac':
111    chrome_name = 'Google Chrome'
112    return os.path.join('.', chrome_name + '.app', 'Contents', 'MacOS',
113                            chrome_name)
114  if platform == 'win':
115    return os.path.join('.', 'chrome.exe')
116
117
118def _confirm_new_seed_downloaded(user_data_dir,
119                                 path_chromedriver,
120                                 chrome_options,
121                                 old_seed=None,
122                                 old_signature=None):
123  """Confirms the new seed to be downloaded from finch server.
124
125  Note that Local State does not dump until Chrome has exited.
126
127  Args:
128    user_data_dir: the use directory used to store fetched seed.
129    path_chromedriver: the path of chromedriver binary.
130    chrome_options: the chrome option used to launch Chrome.
131    old_seed: the old seed serves as a baseline. New seed should be different.
132    old_signature: the old signature serves as a baseline. New signature should
133        be different.
134
135  Returns:
136    True if the new seed is downloaded, otherwise False.
137  """
138  driver = None
139  attempt = 0
140  wait_timeout_in_sec = _WAIT_TIMEOUT_IN_SEC
141  while attempt < _MAX_ATTEMPTS:
142    # Starts Chrome to allow it to download a seed or a seed delta.
143    driver = webdriver.Chrome(path_chromedriver, chrome_options=chrome_options)
144    time.sleep(5)
145    # Exits Chrome so that Local State could be serialized to disk.
146    driver.quit()
147    # Checks the seed and signature.
148    current_seed, current_signature = seed_helper.get_current_seed(
149        user_data_dir)
150    if current_seed != old_seed and current_signature != old_signature:
151      return True
152    attempt += 1
153    time.sleep(wait_timeout_in_sec)
154    wait_timeout_in_sec *= 2
155  return False
156
157def _check_chrome_version():
158  path_chrome = os.path.abspath(_find_chrome_binary())
159  OS = _get_platform()
160  #(crbug/158372)
161  if OS == 'win':
162    cmd = ('powershell -command "&{(Get-Item'
163            '\''+ path_chrome + '\').VersionInfo.ProductVersion}"')
164    version = subprocess.run(cmd, check=True,
165                          capture_output=True).stdout.decode('utf-8')
166  else:
167    cmd = [path_chrome, '--version']
168    version = subprocess.run(cmd, check=True,
169                          capture_output=True).stdout.decode('utf-8')
170    #only return the version number portion
171    version = version.strip().split(" ")[-1]
172  return packaging.version.parse(version)
173
174def _inject_seed(user_data_dir, path_chromedriver, chrome_options):
175  # Verify a production version of variations seed was fetched successfully.
176  if not _confirm_new_seed_downloaded(user_data_dir, path_chromedriver,
177                                      chrome_options):
178    logging.error('Failed to fetch variations seed on initial run')
179    # For MacOS, there is sometime the test fail to download seed on initial
180    # run (crbug/1312393)
181    if _get_platform() != 'mac':
182      return 1
183
184  # Inject the test seed.
185  # This is a path as fallback when |seed_helper.load_test_seed_from_file()|
186  # can't find one under src root.
187  hardcoded_seed_path = os.path.join(_THIS_DIR, _VARIATIONS_TEST_DATA,
188                          'variations_seed_beta_%s.json' % _get_platform())
189  seed, signature = seed_helper.load_test_seed_from_file(hardcoded_seed_path)
190  if not seed or not signature:
191    logging.error(
192        'Ill-formed test seed json file: "%s" and "%s" are required',
193        seed_helper.LOCAL_STATE_SEED_NAME,
194        seed_helper.LOCAL_STATE_SEED_SIGNATURE_NAME)
195    return 1
196
197  if not seed_helper.inject_test_seed(seed, signature, user_data_dir):
198    logging.error('Failed to inject the test seed')
199    return 1
200  return 0
201
202def _run_tests(work_dir, skia_util, *args):
203  """Runs the smoke tests.
204
205  Args:
206    work_dir: A working directory to store screenshots and other artifacts.
207    skia_util: A FinchSkiaGoldUtil used to do pixel test.
208    args: Arguments to be passed to the chrome binary.
209
210  Returns:
211    0 if tests passed, otherwise 1.
212  """
213  skia_gold_session = skia_util.SkiaGoldSession
214  path_chrome = _find_chrome_binary()
215  path_chromedriver = os.path.join('.', 'chromedriver')
216  hardcoded_seed_path = os.path.join(_THIS_DIR, _VARIATIONS_TEST_DATA,
217                             'variations_seed_beta_%s.json' % _get_platform())
218  path_seed = seed_helper.get_test_seed_file_path(hardcoded_seed_path)
219
220  user_data_dir = tempfile.mkdtemp()
221  crash_dump_dir = tempfile.mkdtemp()
222  _, log_file = tempfile.mkstemp()
223
224  # Crashpad is a separate process and its dump locations is set via env
225  # variable.
226  os.environ['BREAKPAD_DUMP_LOCATION'] = crash_dump_dir
227
228  chrome_options = ChromeOptions()
229  chrome_options.binary_location = path_chrome
230  chrome_options.add_argument('user-data-dir=' + user_data_dir)
231  chrome_options.add_argument('log-file=' + log_file)
232  chrome_options.add_argument('variations-test-seed-path=' + path_seed)
233  #TODO(crbug/1342057): Remove this line.
234  chrome_options.add_argument("disable-field-trial-config")
235
236  for arg in args:
237    chrome_options.add_argument(arg)
238
239  # By default, ChromeDriver passes in --disable-backgroud-networking, however,
240  # fetching variations seeds requires network connection, so override it.
241  chrome_options.add_experimental_option('excludeSwitches',
242                                         ['disable-background-networking'])
243
244  driver = None
245  try:
246    chrome_verison = _check_chrome_version()
247    # If --variations-test-seed-path flag was not implemented in this version
248    if chrome_verison <= _FLAG_RELEASE_VERSION:
249      if _inject_seed(user_data_dir, path_chromedriver, chrome_options) == 1:
250        return 1
251
252    # Starts Chrome with the test seed injected.
253    driver = webdriver.Chrome(path_chromedriver, chrome_options=chrome_options)
254
255    # Run test cases: visit urls and verify certain web elements are rendered
256    # correctly.
257    for t in _TEST_CASES:
258      driver.get(t['url'])
259      driver.set_window_size(1280, 1024)
260      element = driver.find_element_by_id(t['expected_id'])
261      if not element.is_displayed() or t['expected_text'] != element.text:
262        logging.error(
263            'Test failed because element: "%s" is not visibly found after '
264            'visiting url: "%s"', t['expected_text'], t['url'])
265        return 1
266      if 'skia_gold_image' in t:
267        image_name = t['skia_gold_image']
268        sc_file = os.path.join(work_dir, image_name + '.png')
269        driver.find_element_by_id('body').screenshot(sc_file)
270        force_dryrun = False
271        if skia_util.IsTryjobRun and skia_util.IsRetryWithoutPatch:
272          force_dryrun = True
273        status, error = skia_gold_session.RunComparison(
274            name=image_name, png_file=sc_file, force_dryrun=force_dryrun)
275        if status:
276          finch_skia_gold_utils.log_skia_gold_status_code(
277              skia_gold_session, image_name, status, error)
278          return status
279
280    driver.quit()
281
282  except NoSuchElementException as e:
283    logging.error('Failed to find the expected web element.\n%s', e)
284    return 1
285  except WebDriverException as e:
286    if os.listdir(crash_dump_dir):
287      logging.error('Chrome crashed and exited abnormally.\n%s', e)
288    else:
289      logging.error('Uncaught WebDriver exception thrown.\n%s', e)
290    return 1
291  finally:
292    shutil.rmtree(user_data_dir, ignore_errors=True)
293    shutil.rmtree(crash_dump_dir, ignore_errors=True)
294
295    # Print logs for debugging purpose.
296    with open(log_file) as f:
297      logging.info('Chrome logs for debugging:\n%s', f.read())
298
299    shutil.rmtree(log_file, ignore_errors=True)
300    if driver:
301      driver.quit()
302
303  return 0
304
305
306def _start_local_http_server():
307  """Starts a local http server.
308
309  Returns:
310    A local http.server.HTTPServer.
311  """
312  httpd = _get_httpd()
313  thread = None
314  address = "http://{}:{}".format(httpd.server_name, httpd.server_port)
315  logging.info("%s is used as local http server.", address)
316  thread = Thread(target=httpd.serve_forever)
317  thread.setDaemon(True)
318  thread.start()
319  return httpd
320
321
322def main_run(args):
323  """Runs the variations smoke tests."""
324  logging.basicConfig(level=logging.INFO)
325  parser = argparse.ArgumentParser()
326  parser.add_argument('--isolated-script-test-output', type=str)
327  parser.add_argument('--isolated-script-test-filter', type=str)
328  FinchSkiaGoldProperties.AddCommandLineArguments(parser)
329  args, rest = parser.parse_known_args()
330
331  temp_dir = tempfile.mkdtemp()
332  httpd = _start_local_http_server()
333  skia_util = finch_skia_gold_utils.FinchSkiaGoldUtil(
334      temp_dir, args)
335  try:
336    rc = _run_tests(temp_dir, skia_util, *rest)
337    if args.isolated_script_test_output:
338      with open(args.isolated_script_test_output, 'w') as f:
339        common.record_local_script_results('run_variations_smoke_tests', f, [],
340                                           rc == 0)
341  finally:
342    httpd.shutdown()
343    shutil.rmtree(temp_dir, ignore_errors=True)
344
345  return rc
346
347
348def main_compile_targets(args):
349  """Returns the list of targets to compile in order to run this test."""
350  json.dump(['chrome', 'chromedriver'], args.output)
351  return 0
352
353
354if __name__ == '__main__':
355  if 'compile_targets' in sys.argv:
356    funcs = {
357        'run': None,
358        'compile_targets': main_compile_targets,
359    }
360    sys.exit(common.run_script(sys.argv[1:], funcs))
361  sys.exit(main_run(sys.argv[1:]))
362