• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env vpython3
2# Copyright 2021 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""A smoke test to verify Chrome doesn't crash and basic rendering is functional
6when parsing a newly given variations seed.
7"""
8
9import argparse
10import http
11import json
12import logging
13import os
14import shutil
15import subprocess
16import sys
17import tempfile
18import time
19from functools import partial
20from http.server import SimpleHTTPRequestHandler
21from threading import Thread
22
23import packaging.version
24
25from selenium import webdriver
26from selenium.webdriver import ChromeOptions
27from selenium.common.exceptions import NoSuchElementException
28from selenium.common.exceptions import WebDriverException
29
30import common
31import variations_seed_access_helper as seed_helper
32from skia_gold_infra import finch_skia_gold_utils
33
34_THIS_DIR = os.path.abspath(os.path.dirname(__file__))
35_CHROMIUM_SRC_DIR = os.path.realpath(os.path.join(_THIS_DIR, '..', '..'))
36
37sys.path.append(os.path.join(_CHROMIUM_SRC_DIR, 'build'))
38# //build/skia_gold_common imports.
39from skia_gold_common.skia_gold_properties import SkiaGoldProperties
40
41_VARIATIONS_TEST_DATA = 'variations_smoke_test_data'
42_VERSION_STRING = 'PRODUCT_VERSION'
43_FLAG_RELEASE_VERSION = packaging.version.parse('105.0.5176.3')
44
45# Constants for the waiting for seed from finch server
46_MAX_ATTEMPTS = 2
47_WAIT_TIMEOUT_IN_SEC = 0.5
48
49# Test cases to verify web elements can be rendered correctly.
50_TEST_CASES = [
51    {
52        # data:text/html,<h1 id="success">Success</h1>
53        'url': 'data:text/html,%3Ch1%20id%3D%22success%22%3ESuccess%3C%2Fh1%3E',
54        'expected_id': 'success',
55        'expected_text': 'Success',
56    },
57    {
58        'url': 'http://localhost:8000',
59        'expected_id': 'sites-chrome-userheader-title',
60        'expected_text': 'The Chromium Projects',
61        'skia_gold_image': 'finch_smoke_render_chromium_org_html',
62    },
63]
64
65
66def _get_httpd():
67  """Returns a HTTPServer instance."""
68  hostname = 'localhost'
69  port = 8000
70  directory = os.path.join(_THIS_DIR, _VARIATIONS_TEST_DATA, 'http_server')
71  httpd = None
72  handler = partial(SimpleHTTPRequestHandler, directory=directory)
73  httpd = http.server.HTTPServer((hostname, port), handler)
74  httpd.timeout = 0.5
75  httpd.allow_reuse_address = True
76  return httpd
77
78
79def _get_platform():
80  """Returns the host platform.
81
82  Returns:
83    One of 'linux', 'win' and 'mac'.
84  """
85  if sys.platform == 'win32' or sys.platform == 'cygwin':
86    return 'win'
87  if sys.platform.startswith('linux'):
88    return 'linux'
89  if sys.platform == 'darwin':
90    return 'mac'
91
92  raise RuntimeError(
93      'Unsupported platform: %s. Only Linux (linux*) and Mac (darwin) and '
94      'Windows (win32 or cygwin) are supported' % sys.platform)
95
96
97def _find_chrome_binary():  #pylint: disable=inconsistent-return-statements
98  """Finds and returns the relative path to the Chrome binary.
99
100  This function assumes that the CWD is the build directory.
101
102  Returns:
103    A relative path to the Chrome binary.
104  """
105  platform = _get_platform()
106  if platform == 'linux':
107    return os.path.join('.', 'chrome')
108  if platform == 'mac':
109    chrome_name = 'Google Chrome'
110    return os.path.join('.', chrome_name + '.app', 'Contents', 'MacOS',
111                        chrome_name)
112  if platform == 'win':
113    return os.path.join('.', 'chrome.exe')
114
115
116def _confirm_new_seed_downloaded(user_data_dir,
117                                 path_chromedriver,
118                                 chrome_options,
119                                 old_seed=None,
120                                 old_signature=None):
121  """Confirms the new seed to be downloaded from finch server.
122
123  Note that Local State does not dump until Chrome has exited.
124
125  Args:
126    user_data_dir: the use directory used to store fetched seed.
127    path_chromedriver: the path of chromedriver binary.
128    chrome_options: the chrome option used to launch Chrome.
129    old_seed: the old seed serves as a baseline. New seed should be different.
130    old_signature: the old signature serves as a baseline. New signature should
131        be different.
132
133  Returns:
134    True if the new seed is downloaded, otherwise False.
135  """
136  driver = None
137  attempt = 0
138  wait_timeout_in_sec = _WAIT_TIMEOUT_IN_SEC
139  while attempt < _MAX_ATTEMPTS:
140    # Starts Chrome to allow it to download a seed or a seed delta.
141    driver = webdriver.Chrome(path_chromedriver, chrome_options=chrome_options)
142    time.sleep(5)
143    # Exits Chrome so that Local State could be serialized to disk.
144    driver.quit()
145    # Checks the seed and signature.
146    current_seed, current_signature = seed_helper.get_current_seed(
147        user_data_dir)
148    if current_seed != old_seed and current_signature != old_signature:
149      return True
150    attempt += 1
151    time.sleep(wait_timeout_in_sec)
152    wait_timeout_in_sec *= 2
153  return False
154
155
156def _check_chrome_version():
157  path_chrome = os.path.abspath(_find_chrome_binary())
158  OS = _get_platform()
159  #(crbug/158372)
160  if OS == 'win':
161    cmd = ('powershell -command "&{(Get-Item'
162           "'" + path_chrome + '\').VersionInfo.ProductVersion}"')
163    version = subprocess.run(cmd, check=True,
164                             capture_output=True).stdout.decode('utf-8')
165  else:
166    cmd = [path_chrome, '--version']
167    version = subprocess.run(cmd, check=True,
168                             capture_output=True).stdout.decode('utf-8')
169    #only return the version number portion
170    version = version.strip().split(' ')[-1]
171  return packaging.version.parse(version)
172
173
174def _inject_seed(user_data_dir, path_chromedriver, chrome_options):
175  # Verify a production version of variations seed was fetched successfully.
176  if not _confirm_new_seed_downloaded(user_data_dir, path_chromedriver,
177                                      chrome_options):
178    logging.error('Failed to fetch variations seed on initial run')
179    # For MacOS, there is sometime the test fail to download seed on initial
180    # run (crbug/1312393)
181    if _get_platform() != 'mac':
182      return 1
183
184  # Inject the test seed.
185  # This is a path as fallback when |seed_helper.load_test_seed_from_file()|
186  # can't find one under src root.
187  hardcoded_seed_path = os.path.join(
188      _THIS_DIR, _VARIATIONS_TEST_DATA,
189      'variations_seed_beta_%s.json' % _get_platform())
190  seed, signature = seed_helper.load_test_seed_from_file(hardcoded_seed_path)
191  if not seed or not signature:
192    logging.error('Ill-formed test seed json file: "%s" and "%s" are required',
193                  seed_helper.LOCAL_STATE_SEED_NAME,
194                  seed_helper.LOCAL_STATE_SEED_SIGNATURE_NAME)
195    return 1
196
197  if not seed_helper.inject_test_seed(seed, signature, user_data_dir):
198    logging.error('Failed to inject the test seed')
199    return 1
200  return 0
201
202
203def _run_tests(work_dir, skia_util, *args):
204  """Runs the smoke tests.
205
206  Args:
207    work_dir: A working directory to store screenshots and other artifacts.
208    skia_util: A FinchSkiaGoldUtil used to do pixel test.
209    args: Arguments to be passed to the chrome binary.
210
211  Returns:
212    0 if tests passed, otherwise 1.
213  """
214  skia_gold_session = skia_util.SkiaGoldSession
215  path_chrome = _find_chrome_binary()
216  path_chromedriver = os.path.join('.', 'chromedriver')
217  hardcoded_seed_path = os.path.join(
218      _THIS_DIR, _VARIATIONS_TEST_DATA,
219      'variations_seed_beta_%s.json' % _get_platform())
220  path_seed = seed_helper.get_test_seed_file_path(hardcoded_seed_path)
221
222  user_data_dir = tempfile.mkdtemp()
223  crash_dump_dir = tempfile.mkdtemp()
224  _, log_file = tempfile.mkstemp()
225
226  # Crashpad is a separate process and its dump locations is set via env
227  # variable.
228  os.environ['BREAKPAD_DUMP_LOCATION'] = crash_dump_dir
229
230  chrome_options = ChromeOptions()
231  chrome_options.binary_location = path_chrome
232  chrome_options.add_argument('user-data-dir=' + user_data_dir)
233  chrome_options.add_argument('log-file=' + log_file)
234  chrome_options.add_argument('variations-test-seed-path=' + path_seed)
235  #TODO(crbug.com/40230862): Remove this line.
236  chrome_options.add_argument('disable-field-trial-config')
237
238  for arg in args:
239    chrome_options.add_argument(arg)
240
241  # By default, ChromeDriver passes in --disable-backgroud-networking, however,
242  # fetching variations seeds requires network connection, so override it.
243  chrome_options.add_experimental_option('excludeSwitches',
244                                         ['disable-background-networking'])
245
246  driver = None
247  try:
248    chrome_verison = _check_chrome_version()
249    # If --variations-test-seed-path flag was not implemented in this version
250    if chrome_verison <= _FLAG_RELEASE_VERSION:
251      if _inject_seed(user_data_dir, path_chromedriver, chrome_options) == 1:
252        return 1
253
254    # Starts Chrome with the test seed injected.
255    driver = webdriver.Chrome(path_chromedriver, chrome_options=chrome_options)
256
257    # Run test cases: visit urls and verify certain web elements are rendered
258    # correctly.
259    for t in _TEST_CASES:
260      driver.get(t['url'])
261      driver.set_window_size(1280, 1024)
262      element = driver.find_element_by_id(t['expected_id'])
263      if not element.is_displayed() or t['expected_text'] != element.text:
264        logging.error(
265            'Test failed because element: "%s" is not visibly found after '
266            'visiting url: "%s"', t['expected_text'], t['url'])
267        return 1
268      if 'skia_gold_image' in t:
269        image_name = t['skia_gold_image']
270        sc_file = os.path.join(work_dir, image_name + '.png')
271        driver.find_element_by_id('body').screenshot(sc_file)
272        force_dryrun = False
273        if skia_util.IsTryjobRun and skia_util.IsRetryWithoutPatch:
274          force_dryrun = True
275        status, error = skia_gold_session.RunComparison(
276            name=image_name, png_file=sc_file, force_dryrun=force_dryrun)
277        if status:
278          finch_skia_gold_utils.log_skia_gold_status_code(
279              skia_gold_session, image_name, status, error)
280          return status
281
282    driver.quit()
283
284  except NoSuchElementException as e:
285    logging.error('Failed to find the expected web element.\n%s', e)
286    return 1
287  except WebDriverException as e:
288    if os.listdir(crash_dump_dir):
289      logging.error('Chrome crashed and exited abnormally.\n%s', e)
290    else:
291      logging.error('Uncaught WebDriver exception thrown.\n%s', e)
292    return 1
293  finally:
294    shutil.rmtree(user_data_dir, ignore_errors=True)
295    shutil.rmtree(crash_dump_dir, ignore_errors=True)
296
297    # Print logs for debugging purpose.
298    with open(log_file) as f:
299      logging.info('Chrome logs for debugging:\n%s', f.read())
300
301    shutil.rmtree(log_file, ignore_errors=True)
302    if driver:
303      driver.quit()
304
305  return 0
306
307
308def _start_local_http_server():
309  """Starts a local http server.
310
311  Returns:
312    A local http.server.HTTPServer.
313  """
314  httpd = _get_httpd()
315  thread = None
316  address = 'http://{}:{}'.format(httpd.server_name, httpd.server_port)
317  logging.info('%s is used as local http server.', address)
318  thread = Thread(target=httpd.serve_forever)
319  thread.setDaemon(True)
320  thread.start()
321  return httpd
322
323
324def main_run(args):
325  """Runs the variations smoke tests."""
326  logging.basicConfig(level=logging.INFO)
327  parser = argparse.ArgumentParser()
328  parser.add_argument('--isolated-script-test-output', type=str)
329  parser.add_argument('--isolated-script-test-filter', type=str)
330  SkiaGoldProperties.AddCommandLineArguments(parser)
331  args, rest = parser.parse_known_args()
332
333  temp_dir = tempfile.mkdtemp()
334  httpd = _start_local_http_server()
335  skia_util = finch_skia_gold_utils.FinchSkiaGoldUtil(temp_dir, args)
336  try:
337    rc = _run_tests(temp_dir, skia_util, *rest)
338    if args.isolated_script_test_output:
339      with open(args.isolated_script_test_output, 'w') as f:
340        common.record_local_script_results('run_variations_smoke_tests', f, [],
341                                           rc == 0)
342  finally:
343    httpd.shutdown()
344    shutil.rmtree(temp_dir, ignore_errors=True)
345
346  return rc
347
348
349def main_compile_targets(args):
350  """Returns the list of targets to compile in order to run this test."""
351  json.dump(['chrome', 'chromedriver'], args.output)
352  return 0
353
354
355if __name__ == '__main__':
356  if 'compile_targets' in sys.argv:
357    funcs = {
358        'run': None,
359        'compile_targets': main_compile_targets,
360    }
361    sys.exit(common.run_script(sys.argv[1:], funcs))
362  sys.exit(main_run(sys.argv[1:]))
363