• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env vpython3
2# Copyright 2022 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Updates the Fuchsia product bundles to the given revision. Should be used
6in a 'hooks_os' entry so that it only runs when .gclient's target_os includes
7'fuchsia'."""
8
9import argparse
10import json
11import logging
12import os
13import sys
14
15sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
16                                             'test')))
17
18import common
19import update_sdk
20from compatible_utils import running_unattended
21
22
23# TODO(crbug.com/40863468): Remove when the old scripts have been deprecated.
24_IMAGE_TO_PRODUCT_BUNDLE = {
25    'qemu.arm64': 'terminal.qemu-arm64',
26    'qemu.x64': 'terminal.x64',
27}
28
29
30# TODO(crbug.com/40863468): Remove when the old scripts have been deprecated.
31def convert_to_products(images_list):
32  """Convert image names in the SDK to product bundle names."""
33
34  product_bundle_list = []
35  for image in images_list:
36    if image in _IMAGE_TO_PRODUCT_BUNDLE:
37      logging.warning(f'Image name {image} has been deprecated. Use '
38                      f'{_IMAGE_TO_PRODUCT_BUNDLE.get(image)} instead.')
39      product_bundle_list.append(_IMAGE_TO_PRODUCT_BUNDLE[image])
40    else:
41      if image.endswith('-release'):
42        image = image[:-len('-release')]
43        logging.warning(f'Image name {image}-release has been deprecated. Use '
44                        f'{image} instead.')
45      product_bundle_list.append(image)
46  return product_bundle_list
47
48
49def remove_repositories(repo_names_to_remove):
50  """Removes given repos from repo list.
51  Repo MUST be present in list to succeed.
52
53  Args:
54    repo_names_to_remove: List of repo names (as strings) to remove.
55  """
56  for repo_name in repo_names_to_remove:
57    common.run_ffx_command(cmd=('repository', 'remove', repo_name), check=True)
58
59
60def get_repositories():
61  """Lists repositories that are available on disk.
62
63  Also prunes repositories that are listed, but do not have an actual packages
64  directory.
65
66  Returns:
67    List of dictionaries containing info about the repositories. They have the
68    following structure:
69    {
70      'name': <repo name>,
71      'spec': {
72        'type': <type, usually pm>,
73        'path': <path to packages directory>
74      },
75    }
76  """
77
78  repos = json.loads(
79      common.run_ffx_command(cmd=('--machine', 'json', 'repository', 'list'),
80                             check=True,
81                             capture_output=True).stdout.strip())
82  to_prune = set()
83  sdk_root_abspath = os.path.abspath(os.path.dirname(common.SDK_ROOT))
84  for repo in repos:
85    # Confirm the path actually exists. If not, prune list.
86    # Also assert the product-bundle repository is for the current repo
87    # (IE within the same directory).
88    if not os.path.exists(repo['spec']['path']):
89      to_prune.add(repo['name'])
90
91    if not repo['spec']['path'].startswith(sdk_root_abspath):
92      to_prune.add(repo['name'])
93
94  repos = [repo for repo in repos if repo['name'] not in to_prune]
95
96  remove_repositories(to_prune)
97  return repos
98
99
100def get_current_signature(image_dir):
101  """Determines the current version of the image, if it exists.
102
103  Returns:
104    The current version, or None if the image is non-existent.
105  """
106
107  version_file = os.path.join(image_dir, 'product_bundle.json')
108  if os.path.exists(version_file):
109    with open(version_file) as f:
110      return json.load(f)['product_version']
111  return None
112
113
114# VisibleForTesting
115def internal_hash():
116  hash_filename = os.path.join(os.path.dirname(__file__),
117                               'linux_internal.sdk.sha1')
118  return (open(hash_filename, 'r').read().strip()
119          if os.path.exists(hash_filename) else '')
120
121
122def main():
123  parser = argparse.ArgumentParser()
124  parser.add_argument('--verbose',
125                      '-v',
126                      action='store_true',
127                      help='Enable debug-level logging.')
128  parser.add_argument(
129      'products',
130      type=str,
131      help='List of product bundles to download, represented as a comma '
132      'separated list.')
133  parser.add_argument(
134      '--internal',
135      action='store_true',
136      help='Whether the images are coming from internal, it impacts version '
137      'file, bucket and download location.')
138  args = parser.parse_args()
139
140  logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
141
142  # Check whether there's Fuchsia support for this platform.
143  common.get_host_os()
144
145  new_products = convert_to_products(args.products.split(','))
146  logging.debug('Searching for the following products: %s', str(new_products))
147
148  logging.debug('Getting new SDK hash')
149  if args.internal:
150    new_hash = internal_hash()
151  else:
152    new_hash = common.get_hash_from_sdk()
153
154  auth_args = [
155      '--auth',
156      os.path.join(os.path.dirname(__file__), 'get_auth_token.py')
157  ] if args.internal and running_unattended() else []
158  if args.internal and not running_unattended():
159    print('*** product bundle v2 requires authentication with your account and '
160          'it should already open a browser window to do it if you have not '
161          'granted the permission yet.')
162  for product in new_products:
163    prod, board = product.split('.', 1)
164    if prod.startswith('smart_display_') and board in [
165        'astro', 'sherlock', 'nelson'
166    ]:
167      # This is a hacky way of keeping the files into the folders matching
168      # the original image name, since the definition is unfortunately in
169      # src-internal. Likely we can download two copies for a smooth
170      # transition, but it would be easier to keep it as-is during the ffx
171      # product v2 migration.
172      # TODO(crbug.com/40938340): Migrate the image download folder away from the
173      # following hack.
174      prod, board = board + '-release', prod
175    if args.internal:
176      # sdk_override.txt does not work for internal images.
177      override_url = None
178      image_dir = os.path.join(common.INTERNAL_IMAGES_ROOT, prod, board)
179    else:
180      override_url = update_sdk.GetSDKOverrideGCSPath()
181      if override_url:
182        logging.debug('Using override file')
183        # TODO(zijiehe): Convert to removesuffix once python 3.9 is supported.
184        if override_url.endswith('/sdk'):
185          override_url = override_url[:-len('/sdk')]
186      image_dir = os.path.join(common.IMAGES_ROOT, prod, board)
187    curr_signature = get_current_signature(image_dir)
188
189    if not override_url and curr_signature == new_hash:
190      continue
191
192    common.make_clean_directory(image_dir)
193    base_url = 'gs://{bucket}/development/{new_hash}'.format(
194        bucket='fuchsia-sdk' if args.internal else 'fuchsia', new_hash=new_hash)
195    lookup_output = common.run_ffx_command(cmd=[
196        '--machine', 'json', 'product', 'lookup', product, new_hash,
197        '--base-url', override_url or base_url
198    ] + auth_args,
199                                           check=True,
200                                           capture_output=True).stdout.strip()
201    download_url = json.loads(lookup_output)['transfer_manifest_url']
202    # The download_url is purely a timestamp based gs location and is fairly
203    # meaningless, so we log the base_url instead which contains the sdk version
204    # if it's not coming from the sdk_override.txt file.
205    logging.info(f'Downloading {product} from {base_url} and {download_url}.')
206    common.run_ffx_command(
207        cmd=['product', 'download', download_url, image_dir] + auth_args,
208        check=True)
209
210  return 0
211
212
213if __name__ == '__main__':
214  sys.exit(main())
215