1#!/usr/bin/env python 2# Copyright 2016 the V8 project authors. All rights reserved. 3# Copyright 2015 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""MB - the Meta-Build wrapper around GN. 8 9MB is a wrapper script for GN that can be used to generate build files 10for sets of canned configurations and analyze them. 11""" 12 13# for py2/py3 compatibility 14from __future__ import print_function 15 16import argparse 17import ast 18import errno 19import json 20import os 21import pipes 22import platform 23import pprint 24import re 25import shutil 26import sys 27import subprocess 28import tempfile 29import traceback 30 31# for py2/py3 compatibility 32try: 33 from urllib.parse import quote 34except ImportError: 35 from urllib2 import quote 36try: 37 from urllib.request import urlopen 38except ImportError: 39 from urllib2 import urlopen 40 41from collections import OrderedDict 42 43CHROMIUM_SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname( 44 os.path.abspath(__file__)))) 45sys.path = [os.path.join(CHROMIUM_SRC_DIR, 'build')] + sys.path 46 47import gn_helpers 48 49try: 50 cmp # Python 2 51except NameError: # Python 3 52 def cmp(x, y): # pylint: disable=redefined-builtin 53 return (x > y) - (x < y) 54 55 56def _v8_builder_fallback(builder, builder_group): 57 """Fallback to V8 builder names before splitting builder/tester. 58 59 This eases splitting builders and testers on release branches and 60 can be removed as soon as all builder have been split and all MB configs 61 exist on all branches. 62 """ 63 builders = [builder] 64 if builder.endswith(' - builder'): 65 builders.append(builder[:-len(' - builder')]) 66 elif builder.endswith(' builder'): 67 builders.append(builder[:-len(' builder')]) 68 69 for builder in builders: 70 if builder in builder_group: 71 return builder_group[builder] 72 return None 73 74 75def main(args): 76 mbw = MetaBuildWrapper() 77 return mbw.Main(args) 78 79 80class MetaBuildWrapper(object): 81 def __init__(self): 82 self.chromium_src_dir = CHROMIUM_SRC_DIR 83 self.default_config = os.path.join(self.chromium_src_dir, 'infra', 'mb', 84 'mb_config.pyl') 85 self.default_isolate_map = os.path.join(self.chromium_src_dir, 'infra', 86 'mb', 'gn_isolate_map.pyl') 87 self.executable = sys.executable 88 self.platform = sys.platform 89 self.sep = os.sep 90 self.args = argparse.Namespace() 91 self.configs = {} 92 self.luci_tryservers = {} 93 self.builder_groups = {} 94 self.mixins = {} 95 self.isolate_exe = 'isolate.exe' if self.platform.startswith( 96 'win') else 'isolate' 97 98 def Main(self, args): 99 self.ParseArgs(args) 100 try: 101 ret = self.args.func() 102 if ret: 103 self.DumpInputFiles() 104 return ret 105 except KeyboardInterrupt: 106 self.Print('interrupted, exiting') 107 return 130 108 except Exception: 109 self.DumpInputFiles() 110 s = traceback.format_exc() 111 for l in s.splitlines(): 112 self.Print(l) 113 return 1 114 115 def ParseArgs(self, argv): 116 def AddCommonOptions(subp): 117 subp.add_argument('-b', '--builder', 118 help='builder name to look up config from') 119 subp.add_argument( 120 '-m', '--builder-group', 121 help='builder group name to look up config from') 122 subp.add_argument('-c', '--config', 123 help='configuration to analyze') 124 subp.add_argument('--phase', 125 help='optional phase name (used when builders ' 126 'do multiple compiles with different ' 127 'arguments in a single build)') 128 subp.add_argument('-f', '--config-file', metavar='PATH', 129 default=self.default_config, 130 help='path to config file ' 131 '(default is %(default)s)') 132 subp.add_argument('-i', '--isolate-map-file', metavar='PATH', 133 help='path to isolate map file ' 134 '(default is %(default)s)', 135 default=[], 136 action='append', 137 dest='isolate_map_files') 138 subp.add_argument('-g', '--goma-dir', 139 help='path to goma directory') 140 subp.add_argument('--android-version-code', 141 help='Sets GN arg android_default_version_code') 142 subp.add_argument('--android-version-name', 143 help='Sets GN arg android_default_version_name') 144 subp.add_argument('-n', '--dryrun', action='store_true', 145 help='Do a dry run (i.e., do nothing, just print ' 146 'the commands that will run)') 147 subp.add_argument('-v', '--verbose', action='store_true', 148 help='verbose logging') 149 150 parser = argparse.ArgumentParser(prog='mb') 151 subps = parser.add_subparsers() 152 153 subp = subps.add_parser('analyze', 154 help='analyze whether changes to a set of files ' 155 'will cause a set of binaries to be rebuilt.') 156 AddCommonOptions(subp) 157 subp.add_argument('path', nargs=1, 158 help='path build was generated into.') 159 subp.add_argument('input_path', nargs=1, 160 help='path to a file containing the input arguments ' 161 'as a JSON object.') 162 subp.add_argument('output_path', nargs=1, 163 help='path to a file containing the output arguments ' 164 'as a JSON object.') 165 subp.add_argument('--json-output', 166 help='Write errors to json.output') 167 subp.set_defaults(func=self.CmdAnalyze) 168 169 subp = subps.add_parser('export', 170 help='print out the expanded configuration for' 171 'each builder as a JSON object') 172 subp.add_argument('-f', '--config-file', metavar='PATH', 173 default=self.default_config, 174 help='path to config file (default is %(default)s)') 175 subp.add_argument('-g', '--goma-dir', 176 help='path to goma directory') 177 subp.set_defaults(func=self.CmdExport) 178 179 subp = subps.add_parser('gen', 180 help='generate a new set of build files') 181 AddCommonOptions(subp) 182 subp.add_argument('--swarming-targets-file', 183 help='save runtime dependencies for targets listed ' 184 'in file.') 185 subp.add_argument('--json-output', 186 help='Write errors to json.output') 187 subp.add_argument('path', nargs=1, 188 help='path to generate build into') 189 subp.set_defaults(func=self.CmdGen) 190 191 subp = subps.add_parser('isolate', 192 help='generate the .isolate files for a given' 193 'binary') 194 AddCommonOptions(subp) 195 subp.add_argument('path', nargs=1, 196 help='path build was generated into') 197 subp.add_argument('target', nargs=1, 198 help='ninja target to generate the isolate for') 199 subp.set_defaults(func=self.CmdIsolate) 200 201 subp = subps.add_parser('lookup', 202 help='look up the command for a given config or ' 203 'builder') 204 AddCommonOptions(subp) 205 subp.add_argument('--quiet', default=False, action='store_true', 206 help='Print out just the arguments, ' 207 'do not emulate the output of the gen subcommand.') 208 subp.add_argument('--recursive', default=False, action='store_true', 209 help='Lookup arguments from imported files, ' 210 'implies --quiet') 211 subp.set_defaults(func=self.CmdLookup) 212 213 subp = subps.add_parser( 214 'run', 215 help='build and run the isolated version of a ' 216 'binary', 217 formatter_class=argparse.RawDescriptionHelpFormatter) 218 subp.description = ( 219 'Build, isolate, and run the given binary with the command line\n' 220 'listed in the isolate. You may pass extra arguments after the\n' 221 'target; use "--" if the extra arguments need to include switches.\n' 222 '\n' 223 'Examples:\n' 224 '\n' 225 ' % tools/mb/mb.py run -m chromium.linux -b "Linux Builder" \\\n' 226 ' //out/Default content_browsertests\n' 227 '\n' 228 ' % tools/mb/mb.py run out/Default content_browsertests\n' 229 '\n' 230 ' % tools/mb/mb.py run out/Default content_browsertests -- \\\n' 231 ' --test-launcher-retry-limit=0' 232 '\n' 233 ) 234 AddCommonOptions(subp) 235 subp.add_argument('-j', '--jobs', dest='jobs', type=int, 236 help='Number of jobs to pass to ninja') 237 subp.add_argument('--no-build', dest='build', default=True, 238 action='store_false', 239 help='Do not build, just isolate and run') 240 subp.add_argument('path', nargs=1, 241 help=('path to generate build into (or use).' 242 ' This can be either a regular path or a ' 243 'GN-style source-relative path like ' 244 '//out/Default.')) 245 subp.add_argument('-d', '--dimension', default=[], action='append', nargs=2, 246 dest='dimensions', metavar='FOO bar', 247 help='dimension to filter on') 248 subp.add_argument('--no-default-dimensions', action='store_false', 249 dest='default_dimensions', default=True, 250 help='Do not automatically add dimensions to the task') 251 subp.add_argument('target', nargs=1, 252 help='ninja target to build and run') 253 subp.add_argument('extra_args', nargs='*', 254 help=('extra args to pass to the isolate to run. Use ' 255 '"--" as the first arg if you need to pass ' 256 'switches')) 257 subp.set_defaults(func=self.CmdRun) 258 259 subp = subps.add_parser('validate', 260 help='validate the config file') 261 subp.add_argument('-f', '--config-file', metavar='PATH', 262 default=self.default_config, 263 help='path to config file (default is %(default)s)') 264 subp.set_defaults(func=self.CmdValidate) 265 266 subp = subps.add_parser('gerrit-buildbucket-config', 267 help='Print buildbucket.config for gerrit ' 268 '(see MB user guide)') 269 subp.add_argument('-f', '--config-file', metavar='PATH', 270 default=self.default_config, 271 help='path to config file (default is %(default)s)') 272 subp.set_defaults(func=self.CmdBuildbucket) 273 274 subp = subps.add_parser('help', 275 help='Get help on a subcommand.') 276 subp.add_argument(nargs='?', action='store', dest='subcommand', 277 help='The command to get help for.') 278 subp.set_defaults(func=self.CmdHelp) 279 280 self.args = parser.parse_args(argv) 281 282 def DumpInputFiles(self): 283 284 def DumpContentsOfFilePassedTo(arg_name, path): 285 if path and self.Exists(path): 286 self.Print("\n# To recreate the file passed to %s:" % arg_name) 287 self.Print("%% cat > %s <<EOF" % path) 288 contents = self.ReadFile(path) 289 self.Print(contents) 290 self.Print("EOF\n%\n") 291 292 if getattr(self.args, 'input_path', None): 293 DumpContentsOfFilePassedTo( 294 'argv[0] (input_path)', self.args.input_path[0]) 295 if getattr(self.args, 'swarming_targets_file', None): 296 DumpContentsOfFilePassedTo( 297 '--swarming-targets-file', self.args.swarming_targets_file) 298 299 def CmdAnalyze(self): 300 vals = self.Lookup() 301 return self.RunGNAnalyze(vals) 302 303 def CmdExport(self): 304 self.ReadConfigFile() 305 obj = {} 306 for builder_group, builders in self.builder_groups.items(): 307 obj[builder_group] = {} 308 for builder in builders: 309 config = self.builder_groups[builder_group][builder] 310 if not config: 311 continue 312 313 if isinstance(config, dict): 314 args = {k: self.FlattenConfig(v)['gn_args'] 315 for k, v in config.items()} 316 elif config.startswith('//'): 317 args = config 318 else: 319 args = self.FlattenConfig(config)['gn_args'] 320 if 'error' in args: 321 continue 322 323 obj[builder_group][builder] = args 324 325 # Dump object and trim trailing whitespace. 326 s = '\n'.join(l.rstrip() for l in 327 json.dumps(obj, sort_keys=True, indent=2).splitlines()) 328 self.Print(s) 329 return 0 330 331 def CmdGen(self): 332 vals = self.Lookup() 333 return self.RunGNGen(vals) 334 335 def CmdHelp(self): 336 if self.args.subcommand: 337 self.ParseArgs([self.args.subcommand, '--help']) 338 else: 339 self.ParseArgs(['--help']) 340 341 def CmdIsolate(self): 342 vals = self.GetConfig() 343 if not vals: 344 return 1 345 return self.RunGNIsolate() 346 347 def CmdLookup(self): 348 vals = self.Lookup() 349 gn_args = self.GNArgs(vals, expand_imports=self.args.recursive) 350 if self.args.quiet or self.args.recursive: 351 self.Print(gn_args, end='') 352 else: 353 cmd = self.GNCmd('gen', '_path_') 354 self.Print('\nWriting """\\\n%s""" to _path_/args.gn.\n' % gn_args) 355 env = None 356 357 self.PrintCmd(cmd, env) 358 return 0 359 360 def CmdRun(self): 361 vals = self.GetConfig() 362 if not vals: 363 return 1 364 365 build_dir = self.args.path[0] 366 target = self.args.target[0] 367 368 if self.args.build: 369 ret = self.Build(target) 370 if ret: 371 return ret 372 ret = self.RunGNIsolate() 373 if ret: 374 return ret 375 376 return self._RunLocallyIsolated(build_dir, target) 377 378 def _RunLocallyIsolated(self, build_dir, target): 379 cmd = [ 380 self.PathJoin(self.chromium_src_dir, 'tools', 'luci-go', 381 self.isolate_exe), 382 'run', 383 '-i', 384 self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), 385 ] 386 if self.args.extra_args: 387 cmd += ['--'] + self.args.extra_args 388 ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False) 389 return ret 390 391 def _DefaultDimensions(self): 392 if not self.args.default_dimensions: 393 return [] 394 395 # This code is naive and just picks reasonable defaults per platform. 396 if self.platform == 'darwin': 397 os_dim = ('os', 'Mac-10.12') 398 elif self.platform.startswith('linux'): 399 os_dim = ('os', 'Ubuntu-16.04') 400 elif self.platform == 'win32': 401 os_dim = ('os', 'Windows-10') 402 else: 403 raise MBErr('unrecognized platform string "%s"' % self.platform) 404 405 return [('pool', 'Chrome'), 406 ('cpu', 'x86-64'), 407 os_dim] 408 409 def CmdBuildbucket(self): 410 self.ReadConfigFile() 411 412 self.Print('# This file was generated using ' 413 '"tools/mb/mb.py gerrit-buildbucket-config".') 414 415 for luci_tryserver in sorted(self.luci_tryservers): 416 self.Print('[bucket "luci.%s"]' % luci_tryserver) 417 for bot in sorted(self.luci_tryservers[luci_tryserver]): 418 self.Print('\tbuilder = %s' % bot) 419 420 for builder_group in sorted(self.builder_groups): 421 if builder_group.startswith('tryserver.'): 422 self.Print('[bucket "builder_group.%s"]' % builder_group) 423 for bot in sorted(self.builder_groups[builder_group]): 424 self.Print('\tbuilder = %s' % bot) 425 426 return 0 427 428 def CmdValidate(self, print_ok=True): 429 errs = [] 430 431 # Read the file to make sure it parses. 432 self.ReadConfigFile() 433 434 # Build a list of all of the configs referenced by builders. 435 all_configs = {} 436 for builder_group in self.builder_groups: 437 for config in self.builder_groups[builder_group].values(): 438 if isinstance(config, dict): 439 for c in config.values(): 440 all_configs[c] = builder_group 441 else: 442 all_configs[config] = builder_group 443 444 # Check that every referenced args file or config actually exists. 445 for config, loc in all_configs.items(): 446 if config.startswith('//'): 447 if not self.Exists(self.ToAbsPath(config)): 448 errs.append('Unknown args file "%s" referenced from "%s".' % 449 (config, loc)) 450 elif not config in self.configs: 451 errs.append('Unknown config "%s" referenced from "%s".' % 452 (config, loc)) 453 454 # Check that every actual config is actually referenced. 455 for config in self.configs: 456 if not config in all_configs: 457 errs.append('Unused config "%s".' % config) 458 459 # Figure out the whole list of mixins, and check that every mixin 460 # listed by a config or another mixin actually exists. 461 referenced_mixins = set() 462 for config, mixins in self.configs.items(): 463 for mixin in mixins: 464 if not mixin in self.mixins: 465 errs.append('Unknown mixin "%s" referenced by config "%s".' % 466 (mixin, config)) 467 referenced_mixins.add(mixin) 468 469 for mixin in self.mixins: 470 for sub_mixin in self.mixins[mixin].get('mixins', []): 471 if not sub_mixin in self.mixins: 472 errs.append('Unknown mixin "%s" referenced by mixin "%s".' % 473 (sub_mixin, mixin)) 474 referenced_mixins.add(sub_mixin) 475 476 # Check that every mixin defined is actually referenced somewhere. 477 for mixin in self.mixins: 478 if not mixin in referenced_mixins: 479 errs.append('Unreferenced mixin "%s".' % mixin) 480 481 if errs: 482 raise MBErr(('mb config file %s has problems:' % self.args.config_file) + 483 '\n ' + '\n '.join(errs)) 484 485 if print_ok: 486 self.Print('mb config file %s looks ok.' % self.args.config_file) 487 return 0 488 489 def GetConfig(self): 490 build_dir = self.args.path[0] 491 492 vals = self.DefaultVals() 493 if self.args.builder or self.args.builder_group or self.args.config: 494 vals = self.Lookup() 495 # Re-run gn gen in order to ensure the config is consistent with the 496 # build dir. 497 self.RunGNGen(vals) 498 return vals 499 500 toolchain_path = self.PathJoin(self.ToAbsPath(build_dir), 501 'toolchain.ninja') 502 if not self.Exists(toolchain_path): 503 self.Print('Must either specify a path to an existing GN build dir ' 504 'or pass in a -m/-b pair or a -c flag to specify the ' 505 'configuration') 506 return {} 507 508 vals['gn_args'] = self.GNArgsFromDir(build_dir) 509 return vals 510 511 def GNArgsFromDir(self, build_dir): 512 args_contents = "" 513 gn_args_path = self.PathJoin(self.ToAbsPath(build_dir), 'args.gn') 514 if self.Exists(gn_args_path): 515 args_contents = self.ReadFile(gn_args_path) 516 gn_args = [] 517 for l in args_contents.splitlines(): 518 fields = l.split(' ') 519 name = fields[0] 520 val = ' '.join(fields[2:]) 521 gn_args.append('%s=%s' % (name, val)) 522 523 return ' '.join(gn_args) 524 525 def Lookup(self): 526 vals = self.ReadIOSBotConfig() 527 if not vals: 528 self.ReadConfigFile() 529 config = self.ConfigFromArgs() 530 if config.startswith('//'): 531 if not self.Exists(self.ToAbsPath(config)): 532 raise MBErr('args file "%s" not found' % config) 533 vals = self.DefaultVals() 534 vals['args_file'] = config 535 else: 536 if not config in self.configs: 537 raise MBErr('Config "%s" not found in %s' % 538 (config, self.args.config_file)) 539 vals = self.FlattenConfig(config) 540 return vals 541 542 def ReadIOSBotConfig(self): 543 if not self.args.builder_group or not self.args.builder: 544 return {} 545 path = self.PathJoin(self.chromium_src_dir, 'ios', 'build', 'bots', 546 self.args.builder_group, self.args.builder + '.json') 547 if not self.Exists(path): 548 return {} 549 550 contents = json.loads(self.ReadFile(path)) 551 gn_args = ' '.join(contents.get('gn_args', [])) 552 553 vals = self.DefaultVals() 554 vals['gn_args'] = gn_args 555 return vals 556 557 def ReadConfigFile(self): 558 if not self.Exists(self.args.config_file): 559 raise MBErr('config file not found at %s' % self.args.config_file) 560 561 try: 562 contents = ast.literal_eval(self.ReadFile(self.args.config_file)) 563 except SyntaxError as e: 564 raise MBErr('Failed to parse config file "%s": %s' % 565 (self.args.config_file, e)) 566 567 self.configs = contents['configs'] 568 self.luci_tryservers = contents.get('luci_tryservers', {}) 569 self.builder_groups = contents['builder_groups'] 570 self.mixins = contents['mixins'] 571 572 def ReadIsolateMap(self): 573 if not self.args.isolate_map_files: 574 self.args.isolate_map_files = [self.default_isolate_map] 575 576 for f in self.args.isolate_map_files: 577 if not self.Exists(f): 578 raise MBErr('isolate map file not found at %s' % f) 579 isolate_maps = {} 580 for isolate_map in self.args.isolate_map_files: 581 try: 582 isolate_map = ast.literal_eval(self.ReadFile(isolate_map)) 583 duplicates = set(isolate_map).intersection(isolate_maps) 584 if duplicates: 585 raise MBErr( 586 'Duplicate targets in isolate map files: %s.' % 587 ', '.join(duplicates)) 588 isolate_maps.update(isolate_map) 589 except SyntaxError as e: 590 raise MBErr( 591 'Failed to parse isolate map file "%s": %s' % (isolate_map, e)) 592 return isolate_maps 593 594 def ConfigFromArgs(self): 595 if self.args.config: 596 if self.args.builder_group or self.args.builder: 597 raise MBErr( 598 'Can not specific both -c/--config and -m/--builder-group or ' 599 '-b/--builder') 600 601 return self.args.config 602 603 if not self.args.builder_group or not self.args.builder: 604 raise MBErr('Must specify either -c/--config or ' 605 '(-m/--builder-group and -b/--builder)') 606 607 if not self.args.builder_group in self.builder_groups: 608 raise MBErr('Builder groups name "%s" not found in "%s"' % 609 (self.args.builder_group, self.args.config_file)) 610 611 config = _v8_builder_fallback( 612 self.args.builder, self.builder_groups[self.args.builder_group]) 613 614 if not config: 615 raise MBErr( 616 'Builder name "%s" not found under builder_groups[%s] in "%s"' % 617 (self.args.builder, self.args.builder_group, self.args.config_file)) 618 619 if isinstance(config, dict): 620 if self.args.phase is None: 621 raise MBErr('Must specify a build --phase for %s on %s' % 622 (self.args.builder, self.args.builder_group)) 623 phase = str(self.args.phase) 624 if phase not in config: 625 raise MBErr('Phase %s doesn\'t exist for %s on %s' % 626 (phase, self.args.builder, self.args.builder_group)) 627 return config[phase] 628 629 if self.args.phase is not None: 630 raise MBErr('Must not specify a build --phase for %s on %s' % 631 (self.args.builder, self.args.builder_group)) 632 return config 633 634 def FlattenConfig(self, config): 635 mixins = self.configs[config] 636 vals = self.DefaultVals() 637 638 visited = [] 639 self.FlattenMixins(mixins, vals, visited) 640 return vals 641 642 def DefaultVals(self): 643 return { 644 'args_file': '', 645 'cros_passthrough': False, 646 'gn_args': '', 647 } 648 649 def FlattenMixins(self, mixins, vals, visited): 650 for m in mixins: 651 if m not in self.mixins: 652 raise MBErr('Unknown mixin "%s"' % m) 653 654 visited.append(m) 655 656 mixin_vals = self.mixins[m] 657 658 if 'cros_passthrough' in mixin_vals: 659 vals['cros_passthrough'] = mixin_vals['cros_passthrough'] 660 if 'args_file' in mixin_vals: 661 if vals['args_file']: 662 raise MBErr('args_file specified multiple times in mixins ' 663 'for %s on %s' % 664 (self.args.builder, self.args.builder_group)) 665 vals['args_file'] = mixin_vals['args_file'] 666 if 'gn_args' in mixin_vals: 667 if vals['gn_args']: 668 vals['gn_args'] += ' ' + mixin_vals['gn_args'] 669 else: 670 vals['gn_args'] = mixin_vals['gn_args'] 671 672 if 'mixins' in mixin_vals: 673 self.FlattenMixins(mixin_vals['mixins'], vals, visited) 674 return vals 675 676 def RunGNGen(self, vals, compute_grit_inputs_for_analyze=False): 677 build_dir = self.args.path[0] 678 679 cmd = self.GNCmd('gen', build_dir, '--check') 680 gn_args = self.GNArgs(vals) 681 if compute_grit_inputs_for_analyze: 682 gn_args += ' compute_grit_inputs_for_analyze=true' 683 684 # Since GN hasn't run yet, the build directory may not even exist. 685 self.MaybeMakeDirectory(self.ToAbsPath(build_dir)) 686 687 gn_args_path = self.ToAbsPath(build_dir, 'args.gn') 688 self.WriteFile(gn_args_path, gn_args, force_verbose=True) 689 690 swarming_targets = [] 691 if getattr(self.args, 'swarming_targets_file', None): 692 # We need GN to generate the list of runtime dependencies for 693 # the compile targets listed (one per line) in the file so 694 # we can run them via swarming. We use gn_isolate_map.pyl to convert 695 # the compile targets to the matching GN labels. 696 path = self.args.swarming_targets_file 697 if not self.Exists(path): 698 self.WriteFailureAndRaise('"%s" does not exist' % path, 699 output_path=None) 700 contents = self.ReadFile(path) 701 swarming_targets = set(contents.splitlines()) 702 703 isolate_map = self.ReadIsolateMap() 704 err, labels = self.MapTargetsToLabels(isolate_map, swarming_targets) 705 if err: 706 raise MBErr(err) 707 708 gn_runtime_deps_path = self.ToAbsPath(build_dir, 'runtime_deps') 709 self.WriteFile(gn_runtime_deps_path, '\n'.join(labels) + '\n') 710 cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path) 711 712 ret, output, _ = self.Run(cmd) 713 if ret: 714 if self.args.json_output: 715 # write errors to json.output 716 self.WriteJSON({'output': output}, self.args.json_output) 717 # If `gn gen` failed, we should exit early rather than trying to 718 # generate isolates. Run() will have already logged any error output. 719 self.Print('GN gen failed: %d' % ret) 720 return ret 721 722 android = 'target_os="android"' in vals['gn_args'] 723 for target in swarming_targets: 724 if android: 725 # Android targets may be either android_apk or executable. The former 726 # will result in runtime_deps associated with the stamp file, while the 727 # latter will result in runtime_deps associated with the executable. 728 label = isolate_map[target]['label'] 729 runtime_deps_targets = [ 730 target + '.runtime_deps', 731 'obj/%s.stamp.runtime_deps' % label.replace(':', '/')] 732 elif (isolate_map[target]['type'] == 'script' or 733 isolate_map[target].get('label_type') == 'group'): 734 # For script targets, the build target is usually a group, 735 # for which gn generates the runtime_deps next to the stamp file 736 # for the label, which lives under the obj/ directory, but it may 737 # also be an executable. 738 label = isolate_map[target]['label'] 739 runtime_deps_targets = [ 740 'obj/%s.stamp.runtime_deps' % label.replace(':', '/')] 741 if self.platform == 'win32': 742 runtime_deps_targets += [ target + '.exe.runtime_deps' ] 743 else: 744 runtime_deps_targets += [ target + '.runtime_deps' ] 745 elif self.platform == 'win32': 746 runtime_deps_targets = [target + '.exe.runtime_deps'] 747 else: 748 runtime_deps_targets = [target + '.runtime_deps'] 749 750 for r in runtime_deps_targets: 751 runtime_deps_path = self.ToAbsPath(build_dir, r) 752 if self.Exists(runtime_deps_path): 753 break 754 else: 755 raise MBErr('did not generate any of %s' % 756 ', '.join(runtime_deps_targets)) 757 758 runtime_deps = self.ReadFile(runtime_deps_path).splitlines() 759 760 self.WriteIsolateFiles(build_dir, target, runtime_deps) 761 762 return 0 763 764 def RunGNIsolate(self): 765 target = self.args.target[0] 766 isolate_map = self.ReadIsolateMap() 767 err, labels = self.MapTargetsToLabels(isolate_map, [target]) 768 if err: 769 raise MBErr(err) 770 label = labels[0] 771 772 build_dir = self.args.path[0] 773 774 cmd = self.GNCmd('desc', build_dir, label, 'runtime_deps') 775 ret, out, _ = self.Call(cmd) 776 if ret: 777 if out: 778 self.Print(out) 779 return ret 780 781 runtime_deps = out.splitlines() 782 783 self.WriteIsolateFiles(build_dir, target, runtime_deps) 784 785 ret, _, _ = self.Run([ 786 self.PathJoin(self.chromium_src_dir, 'tools', 'luci-go', 787 self.isolate_exe), 788 'check', 789 '-i', 790 self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target))], 791 buffer_output=False) 792 793 return ret 794 795 def WriteIsolateFiles(self, build_dir, target, runtime_deps): 796 isolate_path = self.ToAbsPath(build_dir, target + '.isolate') 797 self.WriteFile(isolate_path, 798 pprint.pformat({ 799 'variables': { 800 'files': sorted(runtime_deps), 801 } 802 }) + '\n') 803 804 self.WriteJSON( 805 { 806 'args': [ 807 '--isolate', 808 self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), 809 ], 810 'dir': self.chromium_src_dir, 811 'version': 1, 812 }, 813 isolate_path + 'd.gen.json', 814 ) 815 816 def MapTargetsToLabels(self, isolate_map, targets): 817 labels = [] 818 err = '' 819 820 for target in targets: 821 if target == 'all': 822 labels.append(target) 823 elif target.startswith('//'): 824 labels.append(target) 825 else: 826 if target in isolate_map: 827 if isolate_map[target]['type'] == 'unknown': 828 err += ('test target "%s" type is unknown\n' % target) 829 else: 830 labels.append(isolate_map[target]['label']) 831 else: 832 err += ('target "%s" not found in ' 833 '//infra/mb/gn_isolate_map.pyl\n' % target) 834 835 return err, labels 836 837 def GNCmd(self, subcommand, path, *args): 838 if self.platform.startswith('linux'): 839 subdir, exe = 'linux64', 'gn' 840 elif self.platform == 'darwin': 841 subdir, exe = 'mac', 'gn' 842 else: 843 subdir, exe = 'win', 'gn.exe' 844 845 arch = platform.machine() 846 if (arch.startswith('s390') or arch.startswith('ppc') or 847 self.platform.startswith('aix')): 848 # use gn in PATH 849 gn_path = 'gn' 850 else: 851 gn_path = self.PathJoin(self.chromium_src_dir, 'buildtools', subdir, exe) 852 return [gn_path, subcommand, path] + list(args) 853 854 855 def GNArgs(self, vals, expand_imports=False): 856 if vals['cros_passthrough']: 857 if not 'GN_ARGS' in os.environ: 858 raise MBErr('MB is expecting GN_ARGS to be in the environment') 859 gn_args = os.environ['GN_ARGS'] 860 if not re.search('target_os.*=.*"chromeos"', gn_args): 861 raise MBErr('GN_ARGS is missing target_os = "chromeos": (GN_ARGS=%s)' % 862 gn_args) 863 else: 864 gn_args = vals['gn_args'] 865 866 if self.args.goma_dir: 867 gn_args += ' goma_dir="%s"' % self.args.goma_dir 868 869 android_version_code = self.args.android_version_code 870 if android_version_code: 871 gn_args += ' android_default_version_code="%s"' % android_version_code 872 873 android_version_name = self.args.android_version_name 874 if android_version_name: 875 gn_args += ' android_default_version_name="%s"' % android_version_name 876 877 args_gn_lines = [] 878 parsed_gn_args = {} 879 880 args_file = vals.get('args_file', None) 881 if args_file: 882 if expand_imports: 883 content = self.ReadFile(self.ToAbsPath(args_file)) 884 parsed_gn_args = gn_helpers.FromGNArgs(content) 885 else: 886 args_gn_lines.append('import("%s")' % args_file) 887 888 # Canonicalize the arg string into a sorted, newline-separated list 889 # of key-value pairs, and de-dup the keys if need be so that only 890 # the last instance of each arg is listed. 891 parsed_gn_args.update(gn_helpers.FromGNArgs(gn_args)) 892 args_gn_lines.append(gn_helpers.ToGNString(parsed_gn_args)) 893 894 return '\n'.join(args_gn_lines) 895 896 def ToAbsPath(self, build_path, *comps): 897 return self.PathJoin(self.chromium_src_dir, 898 self.ToSrcRelPath(build_path), 899 *comps) 900 901 def ToSrcRelPath(self, path): 902 """Returns a relative path from the top of the repo.""" 903 if path.startswith('//'): 904 return path[2:].replace('/', self.sep) 905 return self.RelPath(path, self.chromium_src_dir) 906 907 def RunGNAnalyze(self, vals): 908 # Analyze runs before 'gn gen' now, so we need to run gn gen 909 # in order to ensure that we have a build directory. 910 ret = self.RunGNGen(vals, compute_grit_inputs_for_analyze=True) 911 if ret: 912 return ret 913 914 build_path = self.args.path[0] 915 input_path = self.args.input_path[0] 916 gn_input_path = input_path + '.gn' 917 output_path = self.args.output_path[0] 918 gn_output_path = output_path + '.gn' 919 920 inp = self.ReadInputJSON(['files', 'test_targets', 921 'additional_compile_targets']) 922 if self.args.verbose: 923 self.Print() 924 self.Print('analyze input:') 925 self.PrintJSON(inp) 926 self.Print() 927 928 929 # This shouldn't normally happen, but could due to unusual race conditions, 930 # like a try job that gets scheduled before a patch lands but runs after 931 # the patch has landed. 932 if not inp['files']: 933 self.Print('Warning: No files modified in patch, bailing out early.') 934 self.WriteJSON({ 935 'status': 'No dependency', 936 'compile_targets': [], 937 'test_targets': [], 938 }, output_path) 939 return 0 940 941 gn_inp = {} 942 gn_inp['files'] = ['//' + f for f in inp['files'] if not f.startswith('//')] 943 944 isolate_map = self.ReadIsolateMap() 945 err, gn_inp['additional_compile_targets'] = self.MapTargetsToLabels( 946 isolate_map, inp['additional_compile_targets']) 947 if err: 948 raise MBErr(err) 949 950 err, gn_inp['test_targets'] = self.MapTargetsToLabels( 951 isolate_map, inp['test_targets']) 952 if err: 953 raise MBErr(err) 954 labels_to_targets = {} 955 for i, label in enumerate(gn_inp['test_targets']): 956 labels_to_targets[label] = inp['test_targets'][i] 957 958 try: 959 self.WriteJSON(gn_inp, gn_input_path) 960 cmd = self.GNCmd('analyze', build_path, gn_input_path, gn_output_path) 961 ret, output, _ = self.Run(cmd, force_verbose=True) 962 if ret: 963 if self.args.json_output: 964 # write errors to json.output 965 self.WriteJSON({'output': output}, self.args.json_output) 966 return ret 967 968 gn_outp_str = self.ReadFile(gn_output_path) 969 try: 970 gn_outp = json.loads(gn_outp_str) 971 except Exception as e: 972 self.Print("Failed to parse the JSON string GN returned: %s\n%s" 973 % (repr(gn_outp_str), str(e))) 974 raise 975 976 outp = {} 977 if 'status' in gn_outp: 978 outp['status'] = gn_outp['status'] 979 if 'error' in gn_outp: 980 outp['error'] = gn_outp['error'] 981 if 'invalid_targets' in gn_outp: 982 outp['invalid_targets'] = gn_outp['invalid_targets'] 983 if 'compile_targets' in gn_outp: 984 all_input_compile_targets = sorted( 985 set(inp['test_targets'] + inp['additional_compile_targets'])) 986 987 # If we're building 'all', we can throw away the rest of the targets 988 # since they're redundant. 989 if 'all' in gn_outp['compile_targets']: 990 outp['compile_targets'] = ['all'] 991 else: 992 outp['compile_targets'] = gn_outp['compile_targets'] 993 994 # crbug.com/736215: When GN returns targets back, for targets in 995 # the default toolchain, GN will have generated a phony ninja 996 # target matching the label, and so we can safely (and easily) 997 # transform any GN label into the matching ninja target. For 998 # targets in other toolchains, though, GN doesn't generate the 999 # phony targets, and we don't know how to turn the labels into 1000 # compile targets. In this case, we also conservatively give up 1001 # and build everything. Probably the right thing to do here is 1002 # to have GN return the compile targets directly. 1003 if any("(" in target for target in outp['compile_targets']): 1004 self.Print('WARNING: targets with non-default toolchains were ' 1005 'found, building everything instead.') 1006 outp['compile_targets'] = all_input_compile_targets 1007 else: 1008 outp['compile_targets'] = [ 1009 label.replace('//', '') for label in outp['compile_targets']] 1010 1011 # Windows has a maximum command line length of 8k; even Linux 1012 # maxes out at 128k; if analyze returns a *really long* list of 1013 # targets, we just give up and conservatively build everything instead. 1014 # Probably the right thing here is for ninja to support response 1015 # files as input on the command line 1016 # (see https://github.com/ninja-build/ninja/issues/1355). 1017 if len(' '.join(outp['compile_targets'])) > 7*1024: 1018 self.Print('WARNING: Too many compile targets were affected.') 1019 self.Print('WARNING: Building everything instead to avoid ' 1020 'command-line length issues.') 1021 outp['compile_targets'] = all_input_compile_targets 1022 1023 1024 if 'test_targets' in gn_outp: 1025 outp['test_targets'] = [ 1026 labels_to_targets[label] for label in gn_outp['test_targets']] 1027 1028 if self.args.verbose: 1029 self.Print() 1030 self.Print('analyze output:') 1031 self.PrintJSON(outp) 1032 self.Print() 1033 1034 self.WriteJSON(outp, output_path) 1035 1036 finally: 1037 if self.Exists(gn_input_path): 1038 self.RemoveFile(gn_input_path) 1039 if self.Exists(gn_output_path): 1040 self.RemoveFile(gn_output_path) 1041 1042 return 0 1043 1044 def ReadInputJSON(self, required_keys): 1045 path = self.args.input_path[0] 1046 output_path = self.args.output_path[0] 1047 if not self.Exists(path): 1048 self.WriteFailureAndRaise('"%s" does not exist' % path, output_path) 1049 1050 try: 1051 inp = json.loads(self.ReadFile(path)) 1052 except Exception as e: 1053 self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' % 1054 (path, e), output_path) 1055 1056 for k in required_keys: 1057 if not k in inp: 1058 self.WriteFailureAndRaise('input file is missing a "%s" key' % k, 1059 output_path) 1060 1061 return inp 1062 1063 def WriteFailureAndRaise(self, msg, output_path): 1064 if output_path: 1065 self.WriteJSON({'error': msg}, output_path, force_verbose=True) 1066 raise MBErr(msg) 1067 1068 def WriteJSON(self, obj, path, force_verbose=False): 1069 try: 1070 self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n', 1071 force_verbose=force_verbose) 1072 except Exception as e: 1073 raise MBErr('Error %s writing to the output path "%s"' % 1074 (e, path)) 1075 1076 def CheckCompile(self, builder_group, builder): 1077 url_template = self.args.url_template + '/{builder}/builds/_all?as_text=1' 1078 url = quote( 1079 url_template.format(builder_group=builder_group, builder=builder), 1080 safe=':/()?=') 1081 try: 1082 builds = json.loads(self.Fetch(url)) 1083 except Exception as e: 1084 return str(e) 1085 successes = sorted( 1086 [int(x) for x in builds.keys() if "text" in builds[x] and 1087 cmp(builds[x]["text"][:2], ["build", "successful"]) == 0], 1088 reverse=True) 1089 if not successes: 1090 return "no successful builds" 1091 build = builds[str(successes[0])] 1092 step_names = set([step["name"] for step in build["steps"]]) 1093 compile_indicators = set(["compile", "compile (with patch)", "analyze"]) 1094 if compile_indicators & step_names: 1095 return "compiles" 1096 return "does not compile" 1097 1098 def PrintCmd(self, cmd, env): 1099 if self.platform == 'win32': 1100 env_prefix = 'set ' 1101 env_quoter = QuoteForSet 1102 shell_quoter = QuoteForCmd 1103 else: 1104 env_prefix = '' 1105 env_quoter = pipes.quote 1106 shell_quoter = pipes.quote 1107 1108 def print_env(var): 1109 if env and var in env: 1110 self.Print('%s%s=%s' % (env_prefix, var, env_quoter(env[var]))) 1111 1112 print_env('LLVM_FORCE_HEAD_REVISION') 1113 1114 if cmd[0] == self.executable: 1115 cmd = ['python'] + cmd[1:] 1116 self.Print(*[shell_quoter(arg) for arg in cmd]) 1117 1118 def PrintJSON(self, obj): 1119 self.Print(json.dumps(obj, indent=2, sort_keys=True)) 1120 1121 def Build(self, target): 1122 build_dir = self.ToSrcRelPath(self.args.path[0]) 1123 ninja_cmd = ['ninja', '-C', build_dir] 1124 if self.args.jobs: 1125 ninja_cmd.extend(['-j', '%d' % self.args.jobs]) 1126 ninja_cmd.append(target) 1127 ret, _, _ = self.Run(ninja_cmd, force_verbose=False, buffer_output=False) 1128 return ret 1129 1130 def Run(self, cmd, env=None, force_verbose=True, buffer_output=True): 1131 # This function largely exists so it can be overridden for testing. 1132 if self.args.dryrun or self.args.verbose or force_verbose: 1133 self.PrintCmd(cmd, env) 1134 if self.args.dryrun: 1135 return 0, '', '' 1136 1137 ret, out, err = self.Call(cmd, env=env, buffer_output=buffer_output) 1138 if self.args.verbose or force_verbose: 1139 if ret: 1140 self.Print(' -> returned %d' % ret) 1141 if out: 1142 self.Print(out, end='') 1143 if err: 1144 self.Print(err, end='', file=sys.stderr) 1145 return ret, out, err 1146 1147 def Call(self, cmd, env=None, buffer_output=True): 1148 if buffer_output: 1149 p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir, 1150 stdout=subprocess.PIPE, stderr=subprocess.PIPE, 1151 env=env) 1152 out, err = p.communicate() 1153 out = out.decode('utf-8') 1154 err = err.decode('utf-8') 1155 else: 1156 p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir, 1157 env=env) 1158 p.wait() 1159 out = err = '' 1160 return p.returncode, out, err 1161 1162 def ExpandUser(self, path): 1163 # This function largely exists so it can be overridden for testing. 1164 return os.path.expanduser(path) 1165 1166 def Exists(self, path): 1167 # This function largely exists so it can be overridden for testing. 1168 return os.path.exists(path) 1169 1170 def Fetch(self, url): 1171 # This function largely exists so it can be overridden for testing. 1172 f = urlopen(url) 1173 contents = f.read() 1174 f.close() 1175 return contents 1176 1177 def MaybeMakeDirectory(self, path): 1178 try: 1179 os.makedirs(path) 1180 except OSError as e: 1181 if e.errno != errno.EEXIST: 1182 raise 1183 1184 def PathJoin(self, *comps): 1185 # This function largely exists so it can be overriden for testing. 1186 return os.path.join(*comps) 1187 1188 def Print(self, *args, **kwargs): 1189 # This function largely exists so it can be overridden for testing. 1190 print(*args, **kwargs) 1191 if kwargs.get('stream', sys.stdout) == sys.stdout: 1192 sys.stdout.flush() 1193 1194 def ReadFile(self, path): 1195 # This function largely exists so it can be overriden for testing. 1196 with open(path) as fp: 1197 return fp.read() 1198 1199 def RelPath(self, path, start='.'): 1200 # This function largely exists so it can be overriden for testing. 1201 return os.path.relpath(path, start) 1202 1203 def RemoveFile(self, path): 1204 # This function largely exists so it can be overriden for testing. 1205 os.remove(path) 1206 1207 def RemoveDirectory(self, abs_path): 1208 if self.platform == 'win32': 1209 # In other places in chromium, we often have to retry this command 1210 # because we're worried about other processes still holding on to 1211 # file handles, but when MB is invoked, it will be early enough in the 1212 # build that their should be no other processes to interfere. We 1213 # can change this if need be. 1214 self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path]) 1215 else: 1216 shutil.rmtree(abs_path, ignore_errors=True) 1217 1218 def TempFile(self, mode='w'): 1219 # This function largely exists so it can be overriden for testing. 1220 return tempfile.NamedTemporaryFile(mode=mode, delete=False) 1221 1222 def WriteFile(self, path, contents, force_verbose=False): 1223 # This function largely exists so it can be overriden for testing. 1224 if self.args.dryrun or self.args.verbose or force_verbose: 1225 self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path)) 1226 with open(path, 'w') as fp: 1227 return fp.write(contents) 1228 1229 1230class MBErr(Exception): 1231 pass 1232 1233 1234# See http://goo.gl/l5NPDW and http://goo.gl/4Diozm for the painful 1235# details of this next section, which handles escaping command lines 1236# so that they can be copied and pasted into a cmd window. 1237UNSAFE_FOR_SET = set('^<>&|') 1238UNSAFE_FOR_CMD = UNSAFE_FOR_SET.union(set('()%')) 1239ALL_META_CHARS = UNSAFE_FOR_CMD.union(set('"')) 1240 1241 1242def QuoteForSet(arg): 1243 if any(a in UNSAFE_FOR_SET for a in arg): 1244 arg = ''.join('^' + a if a in UNSAFE_FOR_SET else a for a in arg) 1245 return arg 1246 1247 1248def QuoteForCmd(arg): 1249 # First, escape the arg so that CommandLineToArgvW will parse it properly. 1250 if arg == '' or ' ' in arg or '"' in arg: 1251 quote_re = re.compile(r'(\\*)"') 1252 arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg)) 1253 1254 # Then check to see if the arg contains any metacharacters other than 1255 # double quotes; if it does, quote everything (including the double 1256 # quotes) for safety. 1257 if any(a in UNSAFE_FOR_CMD for a in arg): 1258 arg = ''.join('^' + a if a in ALL_META_CHARS else a for a in arg) 1259 return arg 1260 1261 1262if __name__ == '__main__': 1263 sys.exit(main(sys.argv[1:])) 1264