1#!/usr/bin/env python 2# 3# Copyright 2013 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Updates the Chrome reference builds. 8 9Usage: 10 $ /path/to/update_reference_build.py 11 $ git commit -a 12 $ git cl upload 13""" 14 15import collections 16import logging 17import os 18import shutil 19import subprocess 20import sys 21import tempfile 22import urllib2 23import zipfile 24 25sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'py_utils')) 26 27from py_utils import cloud_storage 28from dependency_manager import base_config 29 30 31def BuildNotFoundError(error_string): 32 raise ValueError(error_string) 33 34 35_CHROME_BINARIES_CONFIG = os.path.join( 36 os.path.dirname(os.path.abspath(__file__)), '..', '..', 'common', 37 'py_utils', 'py_utils', 'chrome_binaries.json') 38 39CHROME_GS_BUCKET = 'chrome-unsigned' 40 41 42# Remove a platform name from this list to disable updating it. 43# Add one to enable updating it. (Must also update _PLATFORM_MAP.) 44_PLATFORMS_TO_UPDATE = ['mac_x86_64', 'win_x86', 'win_AMD64', 'linux_x86_64', 45 'android_k_armeabi-v7a', 'android_l_arm64-v8a', 46 'android_l_armeabi-v7a', 'android_n_armeabi-v7a', 47 'android_n_arm64-v8a'] 48 49# Remove a channel name from this list to disable updating it. 50# Add one to enable updating it. 51_CHANNELS_TO_UPDATE = ['stable', 'canary', 'dev'] 52 53 54# Omaha is Chrome's autoupdate server. It reports the current versions used 55# by each platform on each channel. 56_OMAHA_PLATFORMS = { 'stable': ['mac', 'linux', 'win', 'android'], 57 'dev': ['linux'], 'canary': ['mac', 'win']} 58 59 60# All of the information we need to update each platform. 61# omaha: name omaha uses for the platforms. 62# zip_name: name of the zip file to be retrieved from cloud storage. 63# gs_build: name of the Chrome build platform used in cloud storage. 64# destination: Name of the folder to download the reference build to. 65UpdateInfo = collections.namedtuple('UpdateInfo', 66 'omaha, gs_folder, gs_build, zip_name') 67_PLATFORM_MAP = {'mac_x86_64': UpdateInfo(omaha='mac', 68 gs_folder='desktop-*', 69 gs_build='mac64', 70 zip_name='chrome-mac.zip'), 71 'win_x86': UpdateInfo(omaha='win', 72 gs_folder='desktop-*', 73 gs_build='win-clang', 74 zip_name='chrome-win-clang.zip'), 75 'win_AMD64': UpdateInfo(omaha='win', 76 gs_folder='desktop-*', 77 gs_build='win64-clang', 78 zip_name='chrome-win64-clang.zip'), 79 'linux_x86_64': UpdateInfo(omaha='linux', 80 gs_folder='desktop-*', 81 gs_build='linux64', 82 zip_name='chrome-linux64.zip'), 83 'android_k_armeabi-v7a': UpdateInfo(omaha='android', 84 gs_folder='android-*', 85 gs_build='arm', 86 zip_name='Chrome.apk'), 87 'android_l_arm64-v8a': UpdateInfo(omaha='android', 88 gs_folder='android-*', 89 gs_build='arm_64', 90 zip_name='ChromeModern.apk'), 91 'android_l_armeabi-v7a': UpdateInfo(omaha='android', 92 gs_folder='android-*', 93 gs_build='arm', 94 zip_name='Chrome.apk'), 95 'android_n_armeabi-v7a': UpdateInfo(omaha='android', 96 gs_folder='android-*', 97 gs_build='arm', 98 zip_name='Monochrome.apk'), 99 'android_n_arm64-v8a': UpdateInfo(omaha='android', 100 gs_folder='android-*', 101 gs_build='arm_64', 102 zip_name='Monochrome.apk'), 103 104} 105 106 107def _ChannelVersionsMap(channel): 108 rows = _OmahaReportVersionInfo(channel) 109 omaha_versions_map = _OmahaVersionsMap(rows, channel) 110 channel_versions_map = {} 111 for platform in _PLATFORMS_TO_UPDATE: 112 omaha_platform = _PLATFORM_MAP[platform].omaha 113 if omaha_platform in omaha_versions_map: 114 channel_versions_map[platform] = omaha_versions_map[omaha_platform] 115 return channel_versions_map 116 117 118def _OmahaReportVersionInfo(channel): 119 url ='https://omahaproxy.appspot.com/all?channel=%s' % channel 120 lines = urllib2.urlopen(url).readlines() 121 return [l.split(',') for l in lines] 122 123 124def _OmahaVersionsMap(rows, channel): 125 platforms = _OMAHA_PLATFORMS.get(channel, []) 126 if (len(rows) < 1 or 127 not rows[0][0:3] == ['os', 'channel', 'current_version']): 128 raise ValueError( 129 'Omaha report is not in the expected form: %s.' % rows) 130 versions_map = {} 131 for row in rows[1:]: 132 if row[1] != channel: 133 raise ValueError( 134 'Omaha report contains a line with the channel %s' % row[1]) 135 if row[0] in platforms: 136 versions_map[row[0]] = row[2] 137 logging.warn('versions map: %s' % versions_map) 138 if not all(platform in versions_map for platform in platforms): 139 raise ValueError( 140 'Omaha report did not contain all desired platforms for channel %s' % channel) 141 return versions_map 142 143 144def _QueuePlatformUpdate(platform, version, config, channel): 145 """ platform: the name of the platform for the browser to 146 be downloaded & updated from cloud storage. """ 147 platform_info = _PLATFORM_MAP[platform] 148 filename = platform_info.zip_name 149 # remote_path example: desktop-*/30.0.1595.0/precise32/chrome-precise32.zip 150 remote_path = '%s/%s/%s/%s' % ( 151 platform_info.gs_folder, version, platform_info.gs_build, filename) 152 if not cloud_storage.Exists(CHROME_GS_BUCKET, remote_path): 153 cloud_storage_path = 'gs://%s/%s' % (CHROME_GS_BUCKET, remote_path) 154 raise BuildNotFoundError( 155 'Failed to find %s build for version %s at path %s.' % ( 156 platform, version, cloud_storage_path)) 157 reference_builds_folder = os.path.join( 158 os.path.dirname(os.path.abspath(__file__)), 'chrome_telemetry_build', 159 'reference_builds', channel) 160 if not os.path.exists(reference_builds_folder): 161 os.makedirs(reference_builds_folder) 162 local_dest_path = os.path.join(reference_builds_folder, filename) 163 cloud_storage.Get(CHROME_GS_BUCKET, remote_path, local_dest_path) 164 _ModifyBuildIfNeeded(local_dest_path, platform) 165 config.AddCloudStorageDependencyUpdateJob( 166 'chrome_%s' % channel, platform, local_dest_path, version=version, 167 execute_job=False) 168 169 170def _ModifyBuildIfNeeded(location, platform): 171 """Hook to modify the build before saving it for Telemetry to use. 172 173 This can be used to remove various utilities that cause noise in a 174 test environment. Right now, it is just used to remove Keystone, 175 which is a tool used to autoupdate Chrome. 176 """ 177 if platform == 'mac_x86_64': 178 _RemoveKeystoneFromBuild(location) 179 return 180 181 if 'mac' in platform: 182 raise NotImplementedError( 183 'Platform <%s> sounds like it is an OSX version. If so, we may need to ' 184 'remove Keystone from it per crbug.com/932615. Please edit this script' 185 ' and teach it what needs to be done :).') 186 187 188def _RemoveKeystoneFromBuild(location): 189 """Removes the Keystone autoupdate binary from the chrome mac zipfile.""" 190 logging.info('Removing keystone from mac build at %s' % location) 191 temp_folder = tempfile.mkdtemp(prefix='RemoveKeystoneFromBuild') 192 try: 193 subprocess.check_call(['unzip', '-q', location, '-d', temp_folder]) 194 keystone_folder = os.path.join( 195 temp_folder, 'chrome-mac', 'Google Chrome.app', 'Contents', 196 'Frameworks', 'Google Chrome Framework.framework', 'Frameworks', 197 'KeystoneRegistration.framework') 198 shutil.rmtree(keystone_folder) 199 os.remove(location) 200 subprocess.check_call(['zip', '--quiet', '--recurse-paths', '--symlinks', 201 location, 'chrome-mac'], 202 cwd=temp_folder) 203 finally: 204 shutil.rmtree(temp_folder) 205 206 207def UpdateBuilds(): 208 config = base_config.BaseConfig(_CHROME_BINARIES_CONFIG, writable=True) 209 for channel in _CHANNELS_TO_UPDATE: 210 channel_versions_map = _ChannelVersionsMap(channel) 211 for platform in channel_versions_map: 212 print 'Downloading Chrome (%s channel) on %s' % (channel, platform) 213 current_version = config.GetVersion('chrome_%s' % channel, platform) 214 channel_version = channel_versions_map.get(platform) 215 print 'current: %s, channel: %s' % (current_version, channel_version) 216 if current_version and current_version == channel_version: 217 continue 218 _QueuePlatformUpdate(platform, channel_version, config, channel) 219 220 print 'Updating chrome builds with downloaded binaries' 221 config.ExecuteUpdateJobs(force=True) 222 223 224def main(): 225 logging.getLogger().setLevel(logging.DEBUG) 226 UpdateBuilds() 227 228if __name__ == '__main__': 229 main() 230