• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#! /usr/bin/env python
2# Copyright 2020 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Updates the chromium binaries used by devil.
6
7This currently must be called from the top-level chromium src directory.
8"""
9
10import argparse
11import collections
12import json
13import logging
14import os
15import sys
16
17_DEVIL_ROOT_DIR = os.path.abspath(
18    os.path.join(os.path.dirname(__file__), '..', '..'))
19
20sys.path.append(_DEVIL_ROOT_DIR)
21from devil import base_error
22from devil import devil_env
23from devil.utils import cmd_helper
24
25_DEVICE_ARCHS = [
26    {
27        'cpu': 'arm',
28        'platform': 'android_armeabi-v7a',
29    },
30    {
31        'cpu': 'arm64',
32        'platform': 'android_arm64-v8a',
33    },
34    {
35        'cpu': 'x86',
36        'platform': 'android_x86',
37    },
38    {
39        'cpu': 'x64',
40        'platform': 'android_x86_64',
41    },
42]
43_HOST_ARCH = [{
44    # Host binaries use x86_64, not arm, but they build with the
45    # host toolchain within a target_cpu="arm" build.
46    'cpu': 'arm',
47    'platform': 'linux2_x86_64',
48}]
49
50_CHROMIUM_DEPS = {
51    'chromium_commands': {
52        'archs': _HOST_ARCH,
53        'build_path': 'lib.java/chromium_commands.dex.jar',
54        'target_name': 'chromium_commands_java',
55    },
56    'forwarder_device': {
57        'archs': _DEVICE_ARCHS,
58        'build_path': 'device_forwarder',
59        'target_name': 'forwarder2',
60    },
61    'forwarder_host': {
62        'archs': _HOST_ARCH,
63        'build_path': 'clang_x64/host_forwarder',
64        'target_name': 'forwarder2',
65    },
66    'md5sum_device': {
67        'archs': _DEVICE_ARCHS,
68        'build_path': 'md5sum_bin',
69        'target_name': 'md5sum',
70    },
71    'md5sum_host': {
72        'archs': _HOST_ARCH,
73        'build_path': 'clang_x64/md5sum_bin',
74        'target_name': 'md5sum',
75    },
76}
77
78
79def BuildTargetsForCpu(targets, cpu, output_dir):
80  logging.info('Building %s', cpu)
81
82  gn_args = [
83      'ffmpeg_branding="Chrome"',
84      'is_component_build=false',
85      'is_debug=false',
86      'proprietary_codecs=true',
87      'symbol_level=1',
88      'target_cpu="%s"' % cpu,
89      'target_os="android"',
90      'use_goma=true',
91  ]
92
93  cmd = ['gn', 'gen', '--args=%s' % (' '.join(gn_args)), output_dir]
94  ec = cmd_helper.RunCmd(cmd)
95  if ec:
96    raise base_error.BaseError('%s failed with %d' % (cmd, ec))
97
98  ec = cmd_helper.RunCmd(['autoninja', '-C', output_dir] + targets)
99  if ec:
100    raise base_error.BaseError('building %s failed with %d' % (cpu, ec))
101
102
103def UpdateDependency(dependency_name, dependency_info, local_path, platform):
104  bucket = dependency_info['cloud_storage_bucket']
105  folder = dependency_info['cloud_storage_base_folder']
106
107  # determine the hash
108  ec, sha1sum_output = cmd_helper.GetCmdStatusAndOutput(['sha1sum', local_path])
109  if ec:
110    raise base_error.BaseError(
111        'Failed to determine SHA1 for %s: %s' % (local_path, sha1sum_output))
112
113  dependency_sha1 = sha1sum_output.split()[0]
114
115  # upload
116  remote_path = '%s_%s' % (dependency_name, dependency_sha1)
117  gs_dest = 'gs://%s/%s/%s' % (bucket, folder, remote_path)
118  ec, gsutil_output = cmd_helper.GetCmdStatusAndOutput(
119      ['gsutil.py', 'cp', local_path, gs_dest])
120  if ec:
121    raise base_error.BaseError(
122        'Failed to upload %s to %s: %s' % (remote_path, gs_dest, gsutil_output))
123
124  # update entry in json
125  file_info = dependency_info['file_info']
126  if platform not in file_info:
127    file_info[platform] = {
128        'cloud_storage_hash': '',
129        # the user will need to manually update the download path after
130        # uploading a previously unknown dependency.
131        'download_path': 'FIXME',
132    }
133  file_info[platform]['cloud_storage_hash'] = dependency_sha1
134
135
136def UpdateChromiumDependencies(dependencies, args):
137  deps_by_platform = collections.defaultdict(list)
138  for dep_name, dep_info in _CHROMIUM_DEPS.iteritems():
139    archs = dep_info.get('archs', [])
140    for a in archs:
141      deps_by_platform[(a.get('cpu'), a.get('platform'))].append(
142          (dep_name, dep_info.get('build_path'), dep_info.get('target_name')))
143
144  for arch, arch_deps in deps_by_platform.iteritems():
145    targets = [target_name for _n, _b, target_name in arch_deps]
146    cpu, platform = arch
147    output_dir = os.path.join(args.chromium_src_dir, 'out-devil-deps', platform)
148    BuildTargetsForCpu(targets, cpu, output_dir)
149
150    for dep_name, build_path, _ in arch_deps:
151      local_path = os.path.abspath(os.path.join(output_dir, build_path))
152      UpdateDependency(dep_name,
153                       dependencies.get('dependencies', {}).get(dep_name, {}),
154                       local_path, platform)
155
156  return dependencies
157
158
159def UpdateGivenDependency(dependencies, args):
160  dep_name = args.name or os.path.basename(args.path)
161  if not dep_name in dependencies.get('dependencies', {}):
162    raise base_error.BaseError('Could not find dependency "%s" in %s' %
163                               (dep_name, args.dependencies_json))
164
165  UpdateDependency(dep_name,
166                   dependencies.get('dependencies', {}).get(dep_name, {}),
167                   args.path, args.platform)
168
169  return dependencies
170
171
172def main(raw_args):
173  parser = argparse.ArgumentParser(description=__doc__)
174
175  # pylint: disable=protected-access
176  parser.add_argument(
177      '--dependencies-json',
178      type=os.path.abspath,
179      default=devil_env._DEVIL_DEFAULT_CONFIG,
180      help='Binary dependency configuration file to update.')
181  # pylint: enable=protected-access
182
183  subparsers = parser.add_subparsers()
184  chromium_parser = subparsers.add_parser('chromium')
185  chromium_parser.add_argument(
186      '--chromium-src-dir',
187      type=os.path.realpath,
188      default=os.getcwd(),
189      help='Path to chromium/src checkout root.')
190  chromium_parser.set_defaults(update_dependencies=UpdateChromiumDependencies)
191
192  dependency_parser = subparsers.add_parser('dependency')
193  dependency_parser.add_argument('--name', help='Name of dependency to update.')
194  dependency_parser.add_argument(
195      '--path',
196      type=os.path.abspath,
197      help='Path to file to upload as new version of dependency.')
198  dependency_parser.add_argument(
199      '--platform', help='Platform of dependency to update.')
200  dependency_parser.set_defaults(update_dependencies=UpdateGivenDependency)
201
202  args = parser.parse_args(raw_args)
203
204  logging.getLogger().setLevel(logging.INFO)
205
206  with open(args.dependencies_json) as f:
207    dependencies = json.load(f)
208
209  dependencies = args.update_dependencies(dependencies, args)
210
211  with open(args.dependencies_json, 'w') as f:
212    json.dump(dependencies, f, indent=2, separators=(',', ': '), sort_keys=True)
213
214  return 0
215
216
217if __name__ == '__main__':
218  sys.exit(main(sys.argv[1:]))
219