1#!/usr/bin/env python 2# 3# Copyright 2013 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Updates the Chrome reference builds. 8 9Usage: 10 $ /path/to/update_reference_build.py 11 $ git commit -a 12 $ git cl upload 13""" 14 15import collections 16import logging 17import os 18import shutil 19import subprocess 20import sys 21import urllib2 22import zipfile 23 24sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'py_utils')) 25 26from py_utils import cloud_storage 27from dependency_manager import base_config 28 29 30def BuildNotFoundError(error_string): 31 raise ValueError(error_string) 32 33 34_CHROME_BINARIES_CONFIG = os.path.join( 35 os.path.dirname(os.path.abspath(__file__)), '..', '..', 'common', 36 'py_utils', 'py_utils', 'chrome_binaries.json') 37 38CHROME_GS_BUCKET = 'chrome-unsigned' 39 40 41# Remove a platform name from this list to disable updating it. 42# Add one to enable updating it. (Must also update _PLATFORM_MAP.) 43_PLATFORMS_TO_UPDATE = ['mac_x86_64', 'win_x86', 'win_AMD64', 'linux_x86_64', 44 'android_k_armeabi-v7a', 'android_l_arm64-v8a', 45 'android_l_armeabi-v7a', 'android_n_armeabi-v7a'] 46 47# Remove a channal name from this list to disable updating it. 48# Add one to enable updating it. 49_CHANNELS_TO_UPDATE = ['stable', 'canary', 'dev'] 50 51 52# Omaha is Chrome's autoupdate server. It reports the current versions used 53# by each platform on each channel. 54_OMAHA_PLATFORMS = { 'stable': ['mac', 'linux', 'win', 'android'], 55 'dev': ['linux'], 'canary': ['mac', 'win']} 56 57 58# All of the information we need to update each platform. 59# omaha: name omaha uses for the plaftorms. 60# zip_name: name of the zip file to be retrieved from cloud storage. 61# gs_build: name of the Chrome build platform used in cloud storage. 62# destination: Name of the folder to download the reference build to. 63UpdateInfo = collections.namedtuple('UpdateInfo', 64 'omaha, gs_folder, gs_build, zip_name') 65_PLATFORM_MAP = {'mac_x86_64': UpdateInfo(omaha='mac', 66 gs_folder='desktop-*', 67 gs_build='mac64', 68 zip_name='chrome-mac.zip'), 69 'win_x86': UpdateInfo(omaha='win', 70 gs_folder='desktop-*', 71 gs_build='win-clang', 72 zip_name='chrome-win-clang.zip'), 73 'win_AMD64': UpdateInfo(omaha='win', 74 gs_folder='desktop-*', 75 gs_build='win64-clang', 76 zip_name='chrome-win64-clang.zip'), 77 'linux_x86_64': UpdateInfo(omaha='linux', 78 gs_folder='desktop-*', 79 gs_build='linux64', 80 zip_name='chrome-linux64.zip'), 81 'android_k_armeabi-v7a': UpdateInfo(omaha='android', 82 gs_folder='android-*', 83 gs_build='arm', 84 zip_name='Chrome.apk'), 85 'android_l_arm64-v8a': UpdateInfo(omaha='android', 86 gs_folder='android-*', 87 gs_build='arm_64', 88 zip_name='ChromeModern.apk'), 89 'android_l_armeabi-v7a': UpdateInfo(omaha='android', 90 gs_folder='android-*', 91 gs_build='arm', 92 zip_name='Chrome.apk'), 93 'android_n_armeabi-v7a': UpdateInfo(omaha='android', 94 gs_folder='android-*', 95 gs_build='arm', 96 zip_name='Monochrome.apk'), 97 98} 99 100 101def _ChannelVersionsMap(channel): 102 rows = _OmahaReportVersionInfo(channel) 103 omaha_versions_map = _OmahaVersionsMap(rows, channel) 104 channel_versions_map = {} 105 for platform in _PLATFORMS_TO_UPDATE: 106 omaha_platform = _PLATFORM_MAP[platform].omaha 107 if omaha_platform in omaha_versions_map: 108 channel_versions_map[platform] = omaha_versions_map[omaha_platform] 109 return channel_versions_map 110 111 112def _OmahaReportVersionInfo(channel): 113 url ='https://omahaproxy.appspot.com/all?channel=%s' % channel 114 lines = urllib2.urlopen(url).readlines() 115 return [l.split(',') for l in lines] 116 117 118def _OmahaVersionsMap(rows, channel): 119 platforms = _OMAHA_PLATFORMS.get(channel, []) 120 if (len(rows) < 1 or 121 not rows[0][0:3] == ['os', 'channel', 'current_version']): 122 raise ValueError( 123 'Omaha report is not in the expected form: %s.' % rows) 124 versions_map = {} 125 for row in rows[1:]: 126 if row[1] != channel: 127 raise ValueError( 128 'Omaha report contains a line with the channel %s' % row[1]) 129 if row[0] in platforms: 130 versions_map[row[0]] = row[2] 131 logging.warn('versions map: %s' % versions_map) 132 if not all(platform in versions_map for platform in platforms): 133 raise ValueError( 134 'Omaha report did not contain all desired platforms for channel %s' % channel) 135 return versions_map 136 137 138def _QueuePlatformUpdate(platform, version, config, channel): 139 """ platform: the name of the platform for the browser to 140 be downloaded & updated from cloud storage. """ 141 platform_info = _PLATFORM_MAP[platform] 142 filename = platform_info.zip_name 143 # remote_path example: desktop-*/30.0.1595.0/precise32/chrome-precise32.zip 144 remote_path = '%s/%s/%s/%s' % ( 145 platform_info.gs_folder, version, platform_info.gs_build, filename) 146 if not cloud_storage.Exists(CHROME_GS_BUCKET, remote_path): 147 cloud_storage_path = 'gs://%s/%s' % (CHROME_GS_BUCKET, remote_path) 148 raise BuildNotFoundError( 149 'Failed to find %s build for version %s at path %s.' % ( 150 platform, version, cloud_storage_path)) 151 reference_builds_folder = os.path.join( 152 os.path.dirname(os.path.abspath(__file__)), 'chrome_telemetry_build', 153 'reference_builds', channel) 154 if not os.path.exists(reference_builds_folder): 155 os.makedirs(reference_builds_folder) 156 local_dest_path = os.path.join(reference_builds_folder, filename) 157 cloud_storage.Get(CHROME_GS_BUCKET, remote_path, local_dest_path) 158 config.AddCloudStorageDependencyUpdateJob( 159 'chrome_%s' % channel, platform, local_dest_path, version=version, 160 execute_job=False) 161 162 163def UpdateBuilds(): 164 config = base_config.BaseConfig(_CHROME_BINARIES_CONFIG, writable=True) 165 for channel in _CHANNELS_TO_UPDATE: 166 channel_versions_map = _ChannelVersionsMap(channel) 167 for platform in channel_versions_map: 168 print 'Downloading Chrome (%s channel) on %s' % (channel, platform) 169 current_version = config.GetVersion('chrome_%s' % channel, platform) 170 channel_version = channel_versions_map.get(platform) 171 print 'current: %s, channel: %s' % (current_version, channel_version) 172 if current_version and current_version == channel_version: 173 continue 174 _QueuePlatformUpdate(platform, channel_version, config, channel) 175 # TODO: move execute update jobs here, and add committing/uploading the cl. 176 177 print 'Updating chrome builds with downloaded binaries' 178 config.ExecuteUpdateJobs(force=True) 179 180 181def main(): 182 logging.getLogger().setLevel(logging.DEBUG) 183 #TODO(aiolos): alert sheriffs via email when an error is seen. 184 #This should be added when alerts are added when updating the build. 185 UpdateBuilds() 186 # TODO(aiolos): Add --commit flag. crbug.com/547229 187 188if __name__ == '__main__': 189 main() 190