• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2016 Google Inc.
4#
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8
9"""Create the SKP asset."""
10
11
12from __future__ import print_function
13import argparse
14from distutils.dir_util import copy_tree
15import os
16import shutil
17import subprocess
18import sys
19import tempfile
20
21FILE_DIR = os.path.dirname(os.path.abspath(__file__))
22INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir))
23sys.path.insert(0, INFRA_BOTS_DIR)
24import utils
25
26
27BROWSER_EXECUTABLE_ENV_VAR = 'SKP_BROWSER_EXECUTABLE'
28CHROME_SRC_PATH_ENV_VAR = 'SKP_CHROME_SRC_PATH'
29UPLOAD_TO_PARTNER_BUCKET_ENV_VAR = 'SKP_UPLOAD_TO_PARTNER_BUCKET'
30DM_PATH_ENV_VAR = 'DM_PATH'
31
32SKIA_TOOLS = os.path.join(INFRA_BOTS_DIR, os.pardir, os.pardir, 'tools')
33PRIVATE_SKPS_GS = 'gs://skia-skps/private/skps'
34
35
36def getenv(key):
37  val = os.environ.get(key)
38  if not val:
39    print(('Environment variable %s not set; you should run this via '
40           'create_and_upload.py.' % key), file=sys.stderr)
41    sys.exit(1)
42  return val
43
44
45def get_flutter_skps(target_dir):
46  """Creates SKPs using Flutter's skp_generator tool.
47
48  Documentation is at https://github.com/flutter/tests/tree/master/skp_generator
49  """
50  with utils.tmp_dir():
51    print('Retrieving Flutter SKPs...')
52    utils.git_clone('https://github.com/flutter/tests.git', '.')
53    os.chdir('skp_generator')
54    subprocess.check_call(['bash', 'build.sh'])
55    # Fix invalid SKP file names.
56    for f in os.listdir('skps'):
57      original_file_name = os.path.splitext(f)[0]
58      new_file_name = ''.join([x if x.isalnum() else "_"
59                               for x in original_file_name])
60      if new_file_name != original_file_name:
61        os.rename(os.path.join('skps', f),
62                  os.path.join('skps', new_file_name + '.skp'))
63    copy_tree('skps', target_dir)
64    print('Done retrieving Flutter SKPs.')
65
66
67def create_asset(chrome_src_path, browser_executable, target_dir,
68                 upload_to_partner_bucket, dm_path):
69  """Create the SKP asset.
70
71  Creates the asset from 3 sources:
72  1. From Flutter's skp_generator tool.
73  2. The web pages defined in the tools/skp/page_sets/ directory.
74  3. Any private SKPs stored in $PRIVATE_SKPS_GS after running dm on
75     them (see below).
76
77  The script runs the following cmd on the non-generated SKPs stored in
78  $PRIVATE_SKPS_GS -
79  `dm --config skp -w newskps/ --skps oldskps/ --src skp`
80  The cmd updates the version stored in the SKPs so that the versions in
81  them do not eventually become unsupported.
82  """
83  browser_executable = os.path.realpath(browser_executable)
84  chrome_src_path = os.path.realpath(chrome_src_path)
85  dm_path = os.path.realpath(dm_path)
86  target_dir = os.path.realpath(target_dir)
87
88  if not os.path.exists(target_dir):
89    os.makedirs(target_dir)
90
91  # 1. Flutter SKPs
92  get_flutter_skps(target_dir)
93
94  # 2. Skia's SKPs from tools/skp/page_sets/
95  with utils.tmp_dir():
96    if os.environ.get('CHROME_HEADLESS'):
97      print('Starting xvfb')
98      # Start Xvfb if running on a bot.
99      try:
100        xvfb_proc = subprocess.Popen([
101            'sudo', 'Xvfb', ':0', '-screen', '0', '1280x1024x24'])
102      except Exception:
103        # It is ok if the above command fails, it just means that DISPLAY=:0
104        # is already up.
105        xvfb_proc = None
106
107    print('Running webpages_playback to generate SKPs...')
108    webpages_playback_cmd = [
109      'python', '-u', os.path.join(SKIA_TOOLS, 'skp', 'webpages_playback.py'),
110      '--page_sets', 'all',
111      '--browser_executable', browser_executable,
112      '--non-interactive',
113      '--output_dir', os.getcwd(),
114      '--chrome_src_path', chrome_src_path,
115    ]
116    if upload_to_partner_bucket:
117      webpages_playback_cmd.append('--upload_to_partner_bucket')
118    print('Running webpages_playback command:\n$ %s' %
119        ' '.join(webpages_playback_cmd))
120    try:
121      subprocess.check_call(webpages_playback_cmd)
122    finally:
123      if xvfb_proc:
124        try:
125          xvfb_proc.kill()
126        except OSError as e:
127          print('Failed to kill xvfb process via Popen.kill();'
128                ' attempting `sudo kill`...')
129          try:
130            subprocess.check_call(['sudo', 'kill', '-9', str(xvfb_proc.pid)])
131          except subprocess.CalledProcessError as e:
132            print('Failed to kill xvfb process via `sudo kill`;'
133                  'this may cause a hang.')
134      # Clean up any leftover browser instances. This can happen if there are
135      # telemetry crashes, processes are not always cleaned up appropriately by
136      # the webpagereplay and telemetry frameworks.
137      procs = subprocess.check_output(['ps', 'ax']).decode()
138      for line in procs.splitlines():
139        if browser_executable in line:
140          print(line)
141          pid = line.strip().split(' ')[0]
142          if pid != str(os.getpid()) and not 'python' in line:
143            print('Kill browser process %s' % str(pid))
144            try:
145              subprocess.check_call(['kill', '-9', str(pid)])
146            except subprocess.CalledProcessError as e:
147              print(e)
148          else:
149            pass
150        if 'Xvfb' in line:
151          print(line)
152          pid = line.strip().split(' ')[0]
153          print('Kill Xvfb process %s' % str(pid))
154          try:
155            subprocess.check_call(['sudo', 'kill', '-9', str(pid)])
156          except subprocess.CalledProcessError as e:
157            print(e)
158
159    src = os.path.join(os.getcwd(), 'playback', 'skps')
160    for f in os.listdir(src):
161      if f.endswith('.skp'):
162        shutil.copyfile(os.path.join(src, f), os.path.join(target_dir, f))
163    print('Done running webpages_playback.')
164
165  # 3. Copy over private SKPs from Google storage into the target_dir.
166  old_skps_dir = tempfile.mkdtemp()
167  new_skps_dir = tempfile.mkdtemp()
168  print('Copying non-generated SKPs from private GCS bucket...')
169  subprocess.check_call([
170    'gsutil', 'cp', os.path.join(PRIVATE_SKPS_GS, '*'), old_skps_dir])
171  print('Updating non-generated SKP versions')
172  subprocess.check_call([
173      dm_path,
174      '--config', 'skp',
175      '-w', new_skps_dir,
176      '--skps', old_skps_dir,
177      '--src', 'skp'])
178  for f in os.listdir(new_skps_dir):
179    if f.endswith('.skp'):
180      shutil.copyfile(
181          os.path.join(new_skps_dir, f), os.path.join(target_dir, f))
182  shutil.rmtree(old_skps_dir)
183  shutil.rmtree(new_skps_dir)
184
185
186def main():
187  parser = argparse.ArgumentParser()
188  parser.add_argument('--target_dir', '-t', required=True)
189  args = parser.parse_args()
190
191  # Obtain flags from create_and_upload via environment variables, since
192  # this script is called via `sk` and not directly.
193  chrome_src_path = getenv(CHROME_SRC_PATH_ENV_VAR)
194  browser_executable = getenv(BROWSER_EXECUTABLE_ENV_VAR)
195  upload_to_partner_bucket = getenv(UPLOAD_TO_PARTNER_BUCKET_ENV_VAR) == '1'
196  dm_path = getenv(DM_PATH_ENV_VAR)
197
198  create_asset(chrome_src_path, browser_executable, args.target_dir,
199               upload_to_partner_bucket, dm_path)
200
201
202if __name__ == '__main__':
203  main()
204