• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright 2020 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Updates the Fuchsia images to the given revision. Should be used in a
6'hooks_os' entry so that it only runs when .gclient's target_os includes
7'fuchsia'."""
8
9import argparse
10import itertools
11import logging
12import os
13import re
14import subprocess
15import sys
16from typing import Dict, Optional
17
18sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
19                                             'test')))
20
21from common import DIR_SRC_ROOT, IMAGES_ROOT, get_host_os, \
22                   make_clean_directory
23
24from gcs_download import DownloadAndUnpackFromCloudStorage
25
26from update_sdk import GetSDKOverrideGCSPath
27
28IMAGE_SIGNATURE_FILE = '.hash'
29
30
31# TODO(crbug.com/1138433): Investigate whether we can deprecate
32# use of sdk_bucket.txt.
33def GetOverrideCloudStorageBucket():
34  """Read bucket entry from sdk_bucket.txt"""
35  return ReadFile('sdk-bucket.txt').strip()
36
37
38def ReadFile(filename):
39  """Read a file in this directory."""
40  with open(os.path.join(os.path.dirname(__file__), filename), 'r') as f:
41    return f.read()
42
43
44def StrExpansion():
45  return lambda str_value: str_value
46
47
48def VarLookup(local_scope):
49  return lambda var_name: local_scope['vars'][var_name]
50
51
52def GetImageHashList(bucket):
53  """Read filename entries from sdk-hash-files.list (one per line), substitute
54  {platform} in each entry if present, and read from each filename."""
55  assert (get_host_os() == 'linux')
56  filenames = [
57      line.strip() for line in ReadFile('sdk-hash-files.list').replace(
58          '{platform}', 'linux_internal').splitlines()
59  ]
60  image_hashes = [ReadFile(filename).strip() for filename in filenames]
61  return image_hashes
62
63
64def ParseDepsDict(deps_content):
65  local_scope = {}
66  global_scope = {
67      'Str': StrExpansion(),
68      'Var': VarLookup(local_scope),
69      'deps_os': {},
70  }
71  exec(deps_content, global_scope, local_scope)
72  return local_scope
73
74
75def ParseDepsFile(filename):
76  with open(filename, 'rb') as f:
77    deps_content = f.read()
78  return ParseDepsDict(deps_content)
79
80
81def GetImageHash(bucket):
82  """Gets the hash identifier of the newest generation of images."""
83  if bucket == 'fuchsia-sdk':
84    hashes = GetImageHashList(bucket)
85    return max(hashes)
86  deps_file = os.path.join(DIR_SRC_ROOT, 'DEPS')
87  return ParseDepsFile(deps_file)['vars']['fuchsia_version'].split(':')[1]
88
89
90def GetImageSignature(image_hash, boot_images):
91  return 'gn:{image_hash}:{boot_images}:'.format(image_hash=image_hash,
92                                                 boot_images=boot_images)
93
94
95def GetAllImages(boot_image_names):
96  if not boot_image_names:
97    return
98
99  all_device_types = ['generic', 'qemu']
100  all_archs = ['x64', 'arm64']
101
102  images_to_download = set()
103
104  for boot_image in boot_image_names.split(','):
105    components = boot_image.split('.')
106    if len(components) != 2:
107      continue
108
109    device_type, arch = components
110    device_images = all_device_types if device_type == '*' else [device_type]
111    arch_images = all_archs if arch == '*' else [arch]
112    images_to_download.update(itertools.product(device_images, arch_images))
113  return images_to_download
114
115
116def DownloadBootImages(bucket, image_hash, boot_image_names, image_root_dir):
117  images_to_download = GetAllImages(boot_image_names)
118  for image_to_download in images_to_download:
119    device_type = image_to_download[0]
120    arch = image_to_download[1]
121    image_output_dir = os.path.join(image_root_dir, arch, device_type)
122    if os.path.exists(image_output_dir):
123      continue
124
125    logging.info('Downloading Fuchsia boot images for %s.%s...', device_type,
126                 arch)
127
128    # Legacy images use different naming conventions. See fxbug.dev/85552.
129    legacy_delimiter_device_types = ['qemu', 'generic']
130    if bucket == 'fuchsia-sdk' or \
131       device_type not in legacy_delimiter_device_types:
132      type_arch_connector = '.'
133    else:
134      type_arch_connector = '-'
135
136    images_tarball_url = 'gs://{bucket}/development/{image_hash}/images/'\
137        '{device_type}{type_arch_connector}{arch}.tgz'.format(
138            bucket=bucket, image_hash=image_hash, device_type=device_type,
139            type_arch_connector=type_arch_connector, arch=arch)
140    try:
141      DownloadAndUnpackFromCloudStorage(images_tarball_url, image_output_dir)
142    except subprocess.CalledProcessError as e:
143      logging.exception('Failed to download image %s from URL: %s',
144                        image_to_download, images_tarball_url)
145      raise e
146
147
148def _GetImageOverrideInfo() -> Optional[Dict[str, str]]:
149  """Get the bucket location from sdk_override.txt."""
150  location = GetSDKOverrideGCSPath()
151  if not location:
152    return None
153
154  m = re.match(r'gs://([^/]+)/development/([^/]+)/?(?:sdk)?', location)
155  if not m:
156    raise ValueError('Badly formatted image override location %s' % location)
157
158  return {
159      'bucket': m.group(1),
160      'image_hash': m.group(2),
161  }
162
163
164def GetImageLocationInfo(default_bucket: str,
165                         allow_override: bool = True) -> Dict[str, str]:
166  """Figures out where to pull the image from.
167
168  Defaults to the provided default bucket and generates the hash from defaults.
169  If sdk_override.txt exists (and is allowed) it uses that bucket instead.
170
171  Args:
172    default_bucket: a given default for what bucket to use
173    allow_override: allow SDK override to be used.
174
175  Returns:
176    A dictionary containing the bucket and image_hash
177  """
178  # if sdk_override.txt exists (and is allowed) use the image from that bucket.
179  if allow_override:
180    override = _GetImageOverrideInfo()
181    if override:
182      return override
183
184  # Use the bucket in sdk-bucket.txt if an entry exists.
185  # Otherwise use the default bucket.
186  bucket = GetOverrideCloudStorageBucket() or default_bucket
187  return {
188      'bucket': bucket,
189      'image_hash': GetImageHash(bucket),
190  }
191
192
193def main():
194  parser = argparse.ArgumentParser()
195  parser.add_argument('--verbose',
196                      '-v',
197                      action='store_true',
198                      help='Enable debug-level logging.')
199  parser.add_argument(
200      '--boot-images',
201      type=str,
202      required=True,
203      help='List of boot images to download, represented as a comma separated '
204      'list. Wildcards are allowed. ')
205  parser.add_argument(
206      '--default-bucket',
207      type=str,
208      default='fuchsia',
209      help='The Google Cloud Storage bucket in which the Fuchsia images are '
210      'stored. Entry in sdk-bucket.txt will override this flag.')
211  parser.add_argument(
212      '--image-root-dir',
213      default=IMAGES_ROOT,
214      help='Specify the root directory of the downloaded images. Optional')
215  parser.add_argument(
216      '--allow-override',
217      action='store_true',
218      help='Whether sdk_override.txt can be used for fetching the image, if '
219      'it exists.')
220  args = parser.parse_args()
221
222  logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
223
224  # If no boot images need to be downloaded, exit.
225  if not args.boot_images:
226    return 0
227
228  # Check whether there's Fuchsia support for this platform.
229  get_host_os()
230  image_info = GetImageLocationInfo(args.default_bucket, args.allow_override)
231
232  bucket = image_info['bucket']
233  image_hash = image_info['image_hash']
234
235  if not image_hash:
236    return 1
237
238  signature_filename = os.path.join(args.image_root_dir, IMAGE_SIGNATURE_FILE)
239  current_signature = (open(signature_filename, 'r').read().strip()
240                       if os.path.exists(signature_filename) else '')
241  new_signature = GetImageSignature(image_hash, args.boot_images)
242  if current_signature != new_signature:
243    logging.info('Downloading Fuchsia images %s from bucket %s...', image_hash,
244                 bucket)
245    make_clean_directory(args.image_root_dir)
246
247    try:
248      DownloadBootImages(bucket, image_hash, args.boot_images,
249                         args.image_root_dir)
250      with open(signature_filename, 'w') as f:
251        f.write(new_signature)
252    except subprocess.CalledProcessError as e:
253      logging.exception("command '%s' failed with status %d.%s",
254                        ' '.join(e.cmd), e.returncode,
255                        ' Details: ' + e.output if e.output else '')
256      raise e
257  else:
258    logging.info('Signatures matched! Got %s', new_signature)
259
260  return 0
261
262
263if __name__ == '__main__':
264  sys.exit(main())
265