1# Copyright (c) 2013 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Set of operations/utilities related to checking out the depot, and 6outputting annotations on the buildbot waterfall. These are intended to be 7used by the bisection scripts.""" 8 9import errno 10import imp 11import os 12import shutil 13import stat 14import subprocess 15import sys 16 17DEFAULT_GCLIENT_CUSTOM_DEPS = { 18 "src/data/page_cycler": "https://chrome-internal.googlesource.com/" 19 "chrome/data/page_cycler/.git", 20 "src/data/dom_perf": "https://chrome-internal.googlesource.com/" 21 "chrome/data/dom_perf/.git", 22 "src/data/mach_ports": "https://chrome-internal.googlesource.com/" 23 "chrome/data/mach_ports/.git", 24 "src/tools/perf/data": "https://chrome-internal.googlesource.com/" 25 "chrome/tools/perf/data/.git", 26 "src/third_party/adobe/flash/binaries/ppapi/linux": 27 "https://chrome-internal.googlesource.com/" 28 "chrome/deps/adobe/flash/binaries/ppapi/linux/.git", 29 "src/third_party/adobe/flash/binaries/ppapi/linux_x64": 30 "https://chrome-internal.googlesource.com/" 31 "chrome/deps/adobe/flash/binaries/ppapi/linux_x64/.git", 32 "src/third_party/adobe/flash/binaries/ppapi/mac": 33 "https://chrome-internal.googlesource.com/" 34 "chrome/deps/adobe/flash/binaries/ppapi/mac/.git", 35 "src/third_party/adobe/flash/binaries/ppapi/mac_64": 36 "https://chrome-internal.googlesource.com/" 37 "chrome/deps/adobe/flash/binaries/ppapi/mac_64/.git", 38 "src/third_party/adobe/flash/binaries/ppapi/win": 39 "https://chrome-internal.googlesource.com/" 40 "chrome/deps/adobe/flash/binaries/ppapi/win/.git", 41 "src/third_party/adobe/flash/binaries/ppapi/win_x64": 42 "https://chrome-internal.googlesource.com/" 43 "chrome/deps/adobe/flash/binaries/ppapi/win_x64/.git", 44 "src/chrome/tools/test/reference_build/chrome_win": None, 45 "src/chrome/tools/test/reference_build/chrome_mac": None, 46 "src/chrome/tools/test/reference_build/chrome_linux": None, 47 "src/third_party/WebKit/LayoutTests": None, 48 "src/tools/valgrind": None,} 49 50GCLIENT_SPEC_DATA = [ 51 { "name" : "src", 52 "url" : "https://chromium.googlesource.com/chromium/src.git", 53 "deps_file" : ".DEPS.git", 54 "managed" : True, 55 "custom_deps" : {}, 56 "safesync_url": "", 57 }, 58] 59GCLIENT_SPEC_ANDROID = "\ntarget_os = ['android']" 60GCLIENT_CUSTOM_DEPS_V8 = {"src/v8_bleeding_edge": "git://github.com/v8/v8.git"} 61FILE_DEPS_GIT = '.DEPS.git' 62FILE_DEPS = 'DEPS' 63 64REPO_PARAMS = [ 65 'https://chrome-internal.googlesource.com/chromeos/manifest-internal/', 66 '--repo-url', 67 'https://git.chromium.org/external/repo.git' 68] 69 70REPO_SYNC_COMMAND = 'git checkout -f $(git rev-list --max-count=1 '\ 71 '--before=%d remotes/m/master)' 72 73ORIGINAL_ENV = {} 74 75def OutputAnnotationStepStart(name): 76 """Outputs appropriate annotation to signal the start of a step to 77 a trybot. 78 79 Args: 80 name: The name of the step. 81 """ 82 print 83 print '@@@SEED_STEP %s@@@' % name 84 print '@@@STEP_CURSOR %s@@@' % name 85 print '@@@STEP_STARTED@@@' 86 print 87 sys.stdout.flush() 88 89 90def OutputAnnotationStepClosed(): 91 """Outputs appropriate annotation to signal the closing of a step to 92 a trybot.""" 93 print 94 print '@@@STEP_CLOSED@@@' 95 print 96 sys.stdout.flush() 97 98 99def OutputAnnotationStepLink(label, url): 100 """Outputs appropriate annotation to print a link. 101 102 Args: 103 label: The name to print. 104 url: The url to print. 105 """ 106 print 107 print '@@@STEP_LINK@%s@%s@@@' % (label, url) 108 print 109 sys.stdout.flush() 110 111 112def LoadExtraSrc(path_to_file): 113 """Attempts to load an extra source file. If this is successful, uses the 114 new module to override some global values, such as gclient spec data. 115 116 Returns: 117 The loaded src module, or None.""" 118 try: 119 global GCLIENT_SPEC_DATA 120 global GCLIENT_SPEC_ANDROID 121 extra_src = imp.load_source('data', path_to_file) 122 GCLIENT_SPEC_DATA = extra_src.GetGClientSpec() 123 GCLIENT_SPEC_ANDROID = extra_src.GetGClientSpecExtraParams() 124 return extra_src 125 except ImportError, e: 126 return None 127 128 129def IsTelemetryCommand(command): 130 """Attempts to discern whether or not a given command is running telemetry.""" 131 return ('tools/perf/run_' in command or 'tools\\perf\\run_' in command) 132 133 134def CreateAndChangeToSourceDirectory(working_directory): 135 """Creates a directory 'bisect' as a subdirectory of 'working_directory'. If 136 the function is successful, the current working directory will change to that 137 of the new 'bisect' directory. 138 139 Returns: 140 True if the directory was successfully created (or already existed). 141 """ 142 cwd = os.getcwd() 143 os.chdir(working_directory) 144 try: 145 os.mkdir('bisect') 146 except OSError, e: 147 if e.errno != errno.EEXIST: 148 return False 149 os.chdir('bisect') 150 return True 151 152 153def SubprocessCall(cmd, cwd=None): 154 """Runs a subprocess with specified parameters. 155 156 Args: 157 params: A list of parameters to pass to gclient. 158 cwd: Working directory to run from. 159 160 Returns: 161 The return code of the call. 162 """ 163 if os.name == 'nt': 164 # "HOME" isn't normally defined on windows, but is needed 165 # for git to find the user's .netrc file. 166 if not os.getenv('HOME'): 167 os.environ['HOME'] = os.environ['USERPROFILE'] 168 shell = os.name == 'nt' 169 return subprocess.call(cmd, shell=shell, cwd=cwd) 170 171 172def RunGClient(params, cwd=None): 173 """Runs gclient with the specified parameters. 174 175 Args: 176 params: A list of parameters to pass to gclient. 177 cwd: Working directory to run from. 178 179 Returns: 180 The return code of the call. 181 """ 182 cmd = ['gclient'] + params 183 184 return SubprocessCall(cmd, cwd=cwd) 185 186 187def RunRepo(params): 188 """Runs cros repo command with specified parameters. 189 190 Args: 191 params: A list of parameters to pass to gclient. 192 193 Returns: 194 The return code of the call. 195 """ 196 cmd = ['repo'] + params 197 198 return SubprocessCall(cmd) 199 200 201def RunRepoSyncAtTimestamp(timestamp): 202 """Syncs all git depots to the timestamp specified using repo forall. 203 204 Args: 205 params: Unix timestamp to sync to. 206 207 Returns: 208 The return code of the call. 209 """ 210 repo_sync = REPO_SYNC_COMMAND % timestamp 211 cmd = ['forall', '-c', REPO_SYNC_COMMAND % timestamp] 212 return RunRepo(cmd) 213 214 215def RunGClientAndCreateConfig(opts, custom_deps=None, cwd=None): 216 """Runs gclient and creates a config containing both src and src-internal. 217 218 Args: 219 opts: The options parsed from the command line through parse_args(). 220 custom_deps: A dictionary of additional dependencies to add to .gclient. 221 cwd: Working directory to run from. 222 223 Returns: 224 The return code of the call. 225 """ 226 spec = GCLIENT_SPEC_DATA 227 228 if custom_deps: 229 for k, v in custom_deps.iteritems(): 230 spec[0]['custom_deps'][k] = v 231 232 # Cannot have newlines in string on windows 233 spec = 'solutions =' + str(spec) 234 spec = ''.join([l for l in spec.splitlines()]) 235 236 if 'android' in opts.target_platform: 237 spec += GCLIENT_SPEC_ANDROID 238 239 return_code = RunGClient( 240 ['config', '--spec=%s' % spec, '--git-deps'], cwd=cwd) 241 return return_code 242 243 244def IsDepsFileBlink(): 245 """Reads .DEPS.git and returns whether or not we're using blink. 246 247 Returns: 248 True if blink, false if webkit. 249 """ 250 locals = {'Var': lambda _: locals["vars"][_], 251 'From': lambda *args: None} 252 execfile(FILE_DEPS_GIT, {}, locals) 253 return 'blink.git' in locals['vars']['webkit_url'] 254 255 256def OnAccessError(func, path, exc_info): 257 """ 258 Source: http://stackoverflow.com/questions/2656322/python-shutil-rmtree-fails-on-windows-with-access-is-denied 259 260 Error handler for ``shutil.rmtree``. 261 262 If the error is due to an access error (read only file) 263 it attempts to add write permission and then retries. 264 265 If the error is for another reason it re-raises the error. 266 267 Args: 268 func: The function that raised the error. 269 path: The path name passed to func. 270 exc_info: Exception information returned by sys.exc_info(). 271 """ 272 if not os.access(path, os.W_OK): 273 # Is the error an access error ? 274 os.chmod(path, stat.S_IWUSR) 275 func(path) 276 else: 277 raise 278 279 280def RemoveThirdPartyDirectory(dir_name): 281 """Removes third_party directory from the source. 282 283 At some point, some of the third_parties were causing issues to changes in 284 the way they are synced. We remove such folder in order to avoid sync errors 285 while bisecting. 286 287 Returns: 288 True on success, otherwise False. 289 """ 290 path_to_dir = os.path.join(os.getcwd(), 'third_party', dir_name) 291 try: 292 if os.path.exists(path_to_dir): 293 shutil.rmtree(path_to_dir, onerror=OnAccessError) 294 except OSError, e: 295 print 'Error #%d while running shutil.rmtree(%s): %s' % ( 296 e.errno, path_to_dir, str(e)) 297 if e.errno != errno.ENOENT: 298 return False 299 return True 300 301 302def _CleanupPreviousGitRuns(): 303 """Performs necessary cleanup between runs.""" 304 # If a previous run of git crashed, bot was reset, etc... we 305 # might end up with leftover index.lock files. 306 for (path, dir, files) in os.walk(os.getcwd()): 307 for cur_file in files: 308 if cur_file.endswith('index.lock'): 309 path_to_file = os.path.join(path, cur_file) 310 os.remove(path_to_file) 311 312 313def RunGClientAndSync(cwd=None): 314 """Runs gclient and does a normal sync. 315 316 Args: 317 cwd: Working directory to run from. 318 319 Returns: 320 The return code of the call. 321 """ 322 params = ['sync', '--verbose', '--nohooks', '--reset', '--force'] 323 return RunGClient(params, cwd=cwd) 324 325 326def SetupGitDepot(opts, custom_deps): 327 """Sets up the depot for the bisection. The depot will be located in a 328 subdirectory called 'bisect'. 329 330 Args: 331 opts: The options parsed from the command line through parse_args(). 332 custom_deps: A dictionary of additional dependencies to add to .gclient. 333 334 Returns: 335 True if gclient successfully created the config file and did a sync, False 336 otherwise. 337 """ 338 name = 'Setting up Bisection Depot' 339 340 if opts.output_buildbot_annotations: 341 OutputAnnotationStepStart(name) 342 343 passed = False 344 345 if not RunGClientAndCreateConfig(opts, custom_deps): 346 passed_deps_check = True 347 if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)): 348 cwd = os.getcwd() 349 os.chdir('src') 350 if not IsDepsFileBlink(): 351 passed_deps_check = RemoveThirdPartyDirectory('Webkit') 352 else: 353 passed_deps_check = True 354 if passed_deps_check: 355 passed_deps_check = RemoveThirdPartyDirectory('libjingle') 356 if passed_deps_check: 357 passed_deps_check = RemoveThirdPartyDirectory('skia') 358 os.chdir(cwd) 359 360 if passed_deps_check: 361 _CleanupPreviousGitRuns() 362 363 RunGClient(['revert']) 364 if not RunGClientAndSync(): 365 passed = True 366 367 if opts.output_buildbot_annotations: 368 print 369 OutputAnnotationStepClosed() 370 371 return passed 372 373 374def SetupCrosRepo(): 375 """Sets up cros repo for bisecting chromeos. 376 377 Returns: 378 Returns 0 on success. 379 """ 380 cwd = os.getcwd() 381 try: 382 os.mkdir('cros') 383 except OSError, e: 384 if e.errno != errno.EEXIST: 385 return False 386 os.chdir('cros') 387 388 cmd = ['init', '-u'] + REPO_PARAMS 389 390 passed = False 391 392 if not RunRepo(cmd): 393 if not RunRepo(['sync']): 394 passed = True 395 os.chdir(cwd) 396 397 return passed 398 399 400def CopyAndSaveOriginalEnvironmentVars(): 401 """Makes a copy of the current environment variables.""" 402 # TODO: Waiting on crbug.com/255689, will remove this after. 403 vars_to_remove = [] 404 for k, v in os.environ.iteritems(): 405 if 'ANDROID' in k: 406 vars_to_remove.append(k) 407 vars_to_remove.append('CHROME_SRC') 408 vars_to_remove.append('CHROMIUM_GYP_FILE') 409 vars_to_remove.append('GYP_CROSSCOMPILE') 410 vars_to_remove.append('GYP_DEFINES') 411 vars_to_remove.append('GYP_GENERATORS') 412 vars_to_remove.append('GYP_GENERATOR_FLAGS') 413 vars_to_remove.append('OBJCOPY') 414 for k in vars_to_remove: 415 if os.environ.has_key(k): 416 del os.environ[k] 417 418 global ORIGINAL_ENV 419 ORIGINAL_ENV = os.environ.copy() 420 421 422def SetupAndroidBuildEnvironment(opts, path_to_src=None): 423 """Sets up the android build environment. 424 425 Args: 426 opts: The options parsed from the command line through parse_args(). 427 path_to_src: Path to the src checkout. 428 429 Returns: 430 True if successful. 431 """ 432 433 # Revert the environment variables back to default before setting them up 434 # with envsetup.sh. 435 env_vars = os.environ.copy() 436 for k, _ in env_vars.iteritems(): 437 del os.environ[k] 438 for k, v in ORIGINAL_ENV.iteritems(): 439 os.environ[k] = v 440 441 path_to_file = os.path.join('build', 'android', 'envsetup.sh') 442 proc = subprocess.Popen(['bash', '-c', 'source %s && env' % path_to_file], 443 stdout=subprocess.PIPE, 444 stderr=subprocess.PIPE, 445 cwd=path_to_src) 446 (out, _) = proc.communicate() 447 448 for line in out.splitlines(): 449 (k, _, v) = line.partition('=') 450 os.environ[k] = v 451 # envsetup.sh no longer sets OS=android to GYP_DEFINES env variable 452 # (CL/170273005). Set this variable explicitly inorder to build chrome on 453 # android. 454 try: 455 if 'OS=android' not in os.environ['GYP_DEFINES']: 456 os.environ['GYP_DEFINES'] = '%s %s' % (os.environ['GYP_DEFINES'], 457 'OS=android') 458 except KeyError: 459 os.environ['GYP_DEFINES'] = 'OS=android' 460 461 if opts.use_goma: 462 os.environ['GYP_DEFINES'] = '%s %s' % (os.environ['GYP_DEFINES'], 463 'use_goma=1') 464 return not proc.returncode 465 466 467def SetupPlatformBuildEnvironment(opts): 468 """Performs any platform specific setup. 469 470 Args: 471 opts: The options parsed from the command line through parse_args(). 472 473 Returns: 474 True if successful. 475 """ 476 if 'android' in opts.target_platform: 477 CopyAndSaveOriginalEnvironmentVars() 478 return SetupAndroidBuildEnvironment(opts) 479 elif opts.target_platform == 'cros': 480 return SetupCrosRepo() 481 482 return True 483 484 485def CheckIfBisectDepotExists(opts): 486 """Checks if the bisect directory already exists. 487 488 Args: 489 opts: The options parsed from the command line through parse_args(). 490 491 Returns: 492 Returns True if it exists. 493 """ 494 path_to_dir = os.path.join(opts.working_directory, 'bisect', 'src') 495 return os.path.exists(path_to_dir) 496 497 498def CreateBisectDirectoryAndSetupDepot(opts, custom_deps): 499 """Sets up a subdirectory 'bisect' and then retrieves a copy of the depot 500 there using gclient. 501 502 Args: 503 opts: The options parsed from the command line through parse_args(). 504 custom_deps: A dictionary of additional dependencies to add to .gclient. 505 """ 506 if not CreateAndChangeToSourceDirectory(opts.working_directory): 507 raise RuntimeError('Could not create bisect directory.') 508 509 if not SetupGitDepot(opts, custom_deps): 510 raise RuntimeError('Failed to grab source.') 511