1# Copyright 2013 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" 6 7import cStringIO 8import hashlib 9import logging 10import os 11import subprocess 12import sys 13import tarfile 14import urllib2 15 16from telemetry.core.backends.chrome import cros_interface 17from telemetry.core import util 18 19 20PUBLIC_BUCKET = 'chromium-telemetry' 21PARTNER_BUCKET = 'chrome-partner-telemetry' 22INTERNAL_BUCKET = 'chrome-telemetry' 23 24 25_GSUTIL_URL = 'http://storage.googleapis.com/pub/gsutil.tar.gz' 26_DOWNLOAD_PATH = os.path.join(util.GetTelemetryDir(), 'third_party', 'gsutil') 27# TODO(tbarzic): A workaround for http://crbug.com/386416 and 28# http://crbug.com/359293. See |_RunCommand|. 29_CROS_GSUTIL_HOME_WAR = '/home/chromeos-test/' 30 31class CloudStorageError(Exception): 32 @staticmethod 33 def _GetConfigInstructions(gsutil_path): 34 if SupportsProdaccess(gsutil_path) and _FindExecutableInPath('prodaccess'): 35 return 'Run prodaccess to authenticate.' 36 else: 37 if cros_interface.IsRunningOnCrosDevice(): 38 gsutil_path = ('HOME=%s %s' % (_CROS_GSUTIL_HOME_WAR, gsutil_path)) 39 return ('To configure your credentials:\n' 40 ' 1. Run "%s config" and follow its instructions.\n' 41 ' 2. If you have a @google.com account, use that account.\n' 42 ' 3. For the project-id, just enter 0.' % gsutil_path) 43 44 45class PermissionError(CloudStorageError): 46 def __init__(self, gsutil_path): 47 super(PermissionError, self).__init__( 48 'Attempted to access a file from Cloud Storage but you don\'t ' 49 'have permission. ' + self._GetConfigInstructions(gsutil_path)) 50 51 52class CredentialsError(CloudStorageError): 53 def __init__(self, gsutil_path): 54 super(CredentialsError, self).__init__( 55 'Attempted to access a file from Cloud Storage but you have no ' 56 'configured credentials. ' + self._GetConfigInstructions(gsutil_path)) 57 58 59class NotFoundError(CloudStorageError): 60 pass 61 62 63# TODO(tonyg/dtu): Can this be replaced with distutils.spawn.find_executable()? 64def _FindExecutableInPath(relative_executable_path, *extra_search_paths): 65 for path in list(extra_search_paths) + os.environ['PATH'].split(os.pathsep): 66 executable_path = os.path.join(path, relative_executable_path) 67 if os.path.isfile(executable_path) and os.access(executable_path, os.X_OK): 68 return executable_path 69 return None 70 71 72def _DownloadGsutil(): 73 logging.info('Downloading gsutil') 74 response = urllib2.urlopen(_GSUTIL_URL) 75 with tarfile.open(fileobj=cStringIO.StringIO(response.read())) as tar_file: 76 tar_file.extractall(os.path.dirname(_DOWNLOAD_PATH)) 77 logging.info('Downloaded gsutil to %s' % _DOWNLOAD_PATH) 78 79 return os.path.join(_DOWNLOAD_PATH, 'gsutil') 80 81 82def FindGsutil(): 83 """Return the gsutil executable path. If we can't find it, download it.""" 84 # Look for a depot_tools installation. 85 gsutil_path = _FindExecutableInPath( 86 os.path.join('third_party', 'gsutil', 'gsutil'), _DOWNLOAD_PATH) 87 if gsutil_path: 88 return gsutil_path 89 90 # Look for a gsutil installation. 91 gsutil_path = _FindExecutableInPath('gsutil', _DOWNLOAD_PATH) 92 if gsutil_path: 93 return gsutil_path 94 95 # Failed to find it. Download it! 96 return _DownloadGsutil() 97 98 99def SupportsProdaccess(gsutil_path): 100 with open(gsutil_path, 'r') as gsutil: 101 return 'prodaccess' in gsutil.read() 102 103 104def _RunCommand(args): 105 gsutil_path = FindGsutil() 106 107 # On cros device, as telemetry is running as root, home will be set to /root/, 108 # which is not writable. gsutil will attempt to create a download tracker dir 109 # in home dir and fail. To avoid this, override HOME dir to something writable 110 # when running on cros device. 111 # 112 # TODO(tbarzic): Figure out a better way to handle gsutil on cros. 113 # http://crbug.com/386416, http://crbug.com/359293. 114 gsutil_env = None 115 if cros_interface.IsRunningOnCrosDevice(): 116 gsutil_env = os.environ.copy() 117 gsutil_env['HOME'] = _CROS_GSUTIL_HOME_WAR 118 119 gsutil = subprocess.Popen([sys.executable, gsutil_path] + args, 120 stdout=subprocess.PIPE, stderr=subprocess.PIPE, 121 env=gsutil_env) 122 stdout, stderr = gsutil.communicate() 123 124 if gsutil.returncode: 125 if stderr.startswith(( 126 'You are attempting to access protected data with no configured', 127 'Failure: No handler was ready to authenticate.')): 128 raise CredentialsError(gsutil_path) 129 if 'status=403' in stderr or 'status 403' in stderr: 130 raise PermissionError(gsutil_path) 131 if (stderr.startswith('InvalidUriError') or 'No such object' in stderr or 132 'No URLs matched' in stderr): 133 raise NotFoundError(stderr) 134 raise CloudStorageError(stderr) 135 136 return stdout 137 138 139def List(bucket): 140 query = 'gs://%s/' % bucket 141 stdout = _RunCommand(['ls', query]) 142 return [url[len(query):] for url in stdout.splitlines()] 143 144 145def Exists(bucket, remote_path): 146 try: 147 _RunCommand(['ls', 'gs://%s/%s' % (bucket, remote_path)]) 148 return True 149 except NotFoundError: 150 return False 151 152 153def Move(bucket1, bucket2, remote_path): 154 url1 = 'gs://%s/%s' % (bucket1, remote_path) 155 url2 = 'gs://%s/%s' % (bucket2, remote_path) 156 logging.info('Moving %s to %s' % (url1, url2)) 157 _RunCommand(['mv', url1, url2]) 158 159 160def Delete(bucket, remote_path): 161 url = 'gs://%s/%s' % (bucket, remote_path) 162 logging.info('Deleting %s' % url) 163 _RunCommand(['rm', url]) 164 165 166def Get(bucket, remote_path, local_path): 167 url = 'gs://%s/%s' % (bucket, remote_path) 168 logging.info('Downloading %s to %s' % (url, local_path)) 169 _RunCommand(['cp', url, local_path]) 170 171 172def Insert(bucket, remote_path, local_path, publicly_readable=False): 173 url = 'gs://%s/%s' % (bucket, remote_path) 174 command_and_args = ['cp'] 175 extra_info = '' 176 if publicly_readable: 177 command_and_args += ['-a', 'public-read'] 178 extra_info = ' (publicly readable)' 179 command_and_args += [local_path, url] 180 logging.info('Uploading %s to %s%s' % (local_path, url, extra_info)) 181 _RunCommand(command_and_args) 182 183 184def GetIfChanged(file_path, bucket=None): 185 """Gets the file at file_path if it has a hash file that doesn't match. 186 187 If the file is not in Cloud Storage, log a warning instead of raising an 188 exception. We assume that the user just hasn't uploaded the file yet. 189 190 Returns: 191 True if the binary was changed. 192 """ 193 hash_path = file_path + '.sha1' 194 if not os.path.exists(hash_path): 195 logging.warning('Hash file not found: %s' % hash_path) 196 return False 197 198 expected_hash = ReadHash(hash_path) 199 if os.path.exists(file_path) and CalculateHash(file_path) == expected_hash: 200 return False 201 202 if bucket: 203 buckets = [bucket] 204 else: 205 buckets = [PUBLIC_BUCKET, PARTNER_BUCKET, INTERNAL_BUCKET] 206 207 found = False 208 for bucket in buckets: 209 try: 210 url = 'gs://%s/%s' % (bucket, expected_hash) 211 _RunCommand(['cp', url, file_path]) 212 logging.info('Downloaded %s to %s' % (url, file_path)) 213 found = True 214 except NotFoundError: 215 continue 216 217 if not found: 218 logging.warning('Unable to find file in Cloud Storage: %s', file_path) 219 return found 220 221 222def CalculateHash(file_path): 223 """Calculates and returns the hash of the file at file_path.""" 224 sha1 = hashlib.sha1() 225 with open(file_path, 'rb') as f: 226 while True: 227 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. 228 chunk = f.read(1024*1024) 229 if not chunk: 230 break 231 sha1.update(chunk) 232 return sha1.hexdigest() 233 234 235def ReadHash(hash_path): 236 with open(hash_path, 'rb') as f: 237 return f.read(1024).rstrip() 238