1#!/usr/bin/env python3 2 3# Copyright 2021 Google, Inc. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at: 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16""" Build BT targets on the host system. 17 18For building, you will first have to stage a platform directory that has the 19following structure: 20|-common-mk 21|-bt 22|-external 23|-|-rust 24|-|-|-vendor 25 26The simplest way to do this is to check out platform2 to another directory (that 27is not a subdir of this bt directory), symlink bt there and symlink the rust 28vendor repository as well. 29""" 30import argparse 31import multiprocessing 32import os 33import platform 34import shutil 35import six 36import subprocess 37import sys 38import tarfile 39import time 40 41# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {}) 42COMMON_MK_USES = [ 43 'asan', 44 'coverage', 45 'cros_host', 46 'cros_debug', 47 'floss_rootcanal', 48 'function_elimination_experiment', 49 'fuzzer', 50 'fuzzer', 51 'lto_experiment', 52 'msan', 53 'profiling', 54 'proto_force_optimize_speed', 55 'tcmalloc', 56 'test', 57 'ubsan', 58] 59 60# Use a specific commit version for common-mk to avoid build surprises. 61COMMON_MK_COMMIT = "d014d561eaf5ece08166edd98b10c145ef81312d" 62 63# Default use flags. 64USE_DEFAULTS = { 65 'android': False, 66 'bt_nonstandard_codecs': False, 67 'test': False, 68} 69 70VALID_TARGETS = [ 71 'all', # All targets except test and clean 72 'clean', # Clean up output directory 73 'docs', # Build Rust docs 74 'hosttools', # Build the host tools (i.e. packetgen) 75 'main', # Build the main C++ codebase 76 'prepare', # Prepare the output directory (gn gen + rust setup) 77 'rust', # Build only the rust components + copy artifacts to output dir 78 'test', # Run the unit tests 79 'clippy', # Run cargo clippy 80 'utils', # Build Floss utils 81] 82 83# TODO(b/190750167) - Host tests are disabled until we are full bazel build 84HOST_TESTS = [ 85 # 'bluetooth_test_common', 86 # 'bluetoothtbd_test', 87 # 'net_test_avrcp', 88 # 'net_test_btcore', 89 # 'net_test_types', 90 # 'net_test_btm_iso', 91 # 'net_test_btpackets', 92] 93 94# Map of git repos to bootstrap and what commit to check them out at. None 95# values will just checkout to HEAD. 96BOOTSTRAP_GIT_REPOS = { 97 'platform2': ('https://chromium.googlesource.com/chromiumos/platform2', COMMON_MK_COMMIT), 98 'rust_crates': ('https://chromium.googlesource.com/chromiumos/third_party/rust_crates', None), 99 'proto_logging': ('https://android.googlesource.com/platform/frameworks/proto_logging', None), 100} 101 102# List of packages required for linux build 103REQUIRED_APT_PACKAGES = [ 104 'bison', 105 'build-essential', 106 'curl', 107 'debmake', 108 'flatbuffers-compiler', 109 'flex', 110 'g++-multilib', 111 'gcc-multilib', 112 'generate-ninja', 113 'gnupg', 114 'gperf', 115 'libabsl-dev', 116 'libc++abi-dev', 117 'libc++-dev', 118 'libdbus-1-dev', 119 'libdouble-conversion-dev', 120 'libevent-dev', 121 'libevent-dev', 122 'libflatbuffers-dev', 123 'libfmt-dev', 124 'libgl1-mesa-dev', 125 'libglib2.0-dev', 126 'libgtest-dev', 127 'libgmock-dev', 128 'liblc3-dev', 129 'liblz4-tool', 130 'libncurses5', 131 'libnss3-dev', 132 'libfmt-dev', 133 'libprotobuf-dev', 134 'libre2-9', 135 'libre2-dev', 136 'libssl-dev', 137 'libtinyxml2-dev', 138 'libx11-dev', 139 'libxml2-utils', 140 'ninja-build', 141 'openssl', 142 'protobuf-compiler', 143 'unzip', 144 'x11proto-core-dev', 145 'xsltproc', 146 'zip', 147 'zlib1g-dev', 148] 149 150# List of cargo packages required for linux build 151REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd', 'pdl-compiler'] 152 153APT_PKG_LIST = ['apt', '-qq', 'list'] 154CARGO_PKG_LIST = ['cargo', 'install', '--list'] 155 156 157class UseFlags(): 158 159 def __init__(self, use_flags): 160 """ Construct the use flags. 161 162 Args: 163 use_flags: List of use flags parsed from the command. 164 """ 165 self.flags = {} 166 167 # Import use flags required by common-mk 168 for use in COMMON_MK_USES: 169 self.set_flag(use, False) 170 171 # Set our defaults 172 for use, value in USE_DEFAULTS.items(): 173 self.set_flag(use, value) 174 175 # Set use flags - value is set to True unless the use starts with - 176 # All given use flags always override the defaults 177 for use in use_flags: 178 value = not use.startswith('-') 179 self.set_flag(use, value) 180 181 def set_flag(self, key, value=True): 182 setattr(self, key, value) 183 self.flags[key] = value 184 185 186class HostBuild(): 187 188 def __init__(self, args): 189 """ Construct the builder. 190 191 Args: 192 args: Parsed arguments from ArgumentParser 193 """ 194 self.args = args 195 196 # Set jobs to number of cpus unless explicitly set 197 self.jobs = self.args.jobs 198 if not self.jobs: 199 self.jobs = multiprocessing.cpu_count() 200 sys.stderr.write("Number of jobs = {}\n".format(self.jobs)) 201 202 # Normalize bootstrap dir and make sure it exists 203 self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir) 204 os.makedirs(self.bootstrap_dir, exist_ok=True) 205 206 # Output and platform directories are based on bootstrap 207 self.output_dir = os.path.join(self.bootstrap_dir, 'output') 208 self.platform_dir = os.path.join(self.bootstrap_dir, 'staging') 209 self.bt_dir = os.path.join(self.platform_dir, 'bt') 210 self.sysroot = self.args.sysroot 211 self.libdir = self.args.libdir 212 self.install_dir = os.path.join(self.output_dir, 'install') 213 214 assert os.path.samefile(self.bt_dir, 215 os.path.dirname(__file__)), "Please rerun bootstrap for the current project!" 216 217 # If default target isn't set, build everything 218 self.target = 'all' 219 if hasattr(self.args, 'target') and self.args.target: 220 self.target = self.args.target 221 222 target_use = self.args.use if self.args.use else [] 223 224 # Unless set, always build test code 225 if not self.args.notest: 226 target_use.append('test') 227 228 self.use = UseFlags(target_use) 229 230 # Validate platform directory 231 assert os.path.isdir(self.platform_dir), 'Platform dir does not exist' 232 assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root' 233 234 # Make sure output directory exists (or create it) 235 os.makedirs(self.output_dir, exist_ok=True) 236 237 # Set some default attributes 238 self.libbase_ver = None 239 240 self.configure_environ() 241 242 def _generate_rustflags(self): 243 """ Rustflags to include for the build. 244 """ 245 rust_flags = [ 246 '-L', 247 '{}/out/Default'.format(self.output_dir), 248 '-C', 249 'link-arg=-Wl,--allow-multiple-definition', 250 # exclude uninteresting warnings 251 '-A improper_ctypes_definitions -A improper_ctypes -A unknown_lints', 252 ] 253 254 return ' '.join(rust_flags) 255 256 def configure_environ(self): 257 """ Configure environment variables for GN and Cargo. 258 """ 259 self.env = os.environ.copy() 260 261 # Make sure cargo home dir exists and has a bin directory 262 cargo_home = os.path.join(self.output_dir, 'cargo_home') 263 os.makedirs(cargo_home, exist_ok=True) 264 os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True) 265 266 # Configure Rust env variables 267 self.custom_env = {} 268 self.custom_env['CARGO_TARGET_DIR'] = self.output_dir 269 self.custom_env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home') 270 self.custom_env['RUSTFLAGS'] = self._generate_rustflags() 271 self.custom_env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt') 272 self.custom_env['CROS_SYSTEM_API_ROOT'] = os.path.join(self.platform_dir, 'system_api') 273 self.custom_env['CXX_OUTDIR'] = self._gn_default_output() 274 275 # On ChromeOS, this is /usr/bin/grpc_rust_plugin 276 # In the container, this is /root/.cargo/bin/grpc_rust_plugin 277 self.custom_env['GRPC_RUST_PLUGIN_PATH'] = shutil.which('grpc_rust_plugin') 278 self.env.update(self.custom_env) 279 280 def print_env(self): 281 """ Print the custom environment variables that are used in build. 282 283 Useful so that external tools can mimic the environment to be the same 284 as build.py, e.g. rust-analyzer. 285 """ 286 for k, v in self.custom_env.items(): 287 print("export {}='{}'".format(k, v)) 288 289 def run_command(self, target, args, cwd=None, env=None): 290 """ Run command and stream the output. 291 """ 292 # Set some defaults 293 if not cwd: 294 cwd = self.platform_dir 295 if not env: 296 env = self.env 297 298 log_file = os.path.join(self.output_dir, '{}.log'.format(target)) 299 with open(log_file, 'wb') as lf: 300 rc = 0 301 process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE) 302 while True: 303 line = process.stdout.readline() 304 print(line.decode('utf-8'), end="") 305 lf.write(line) 306 if not line: 307 rc = process.poll() 308 if rc is not None: 309 break 310 311 time.sleep(0.1) 312 313 if rc != 0: 314 raise Exception("Return code is {}".format(rc)) 315 316 def _get_basever(self): 317 if self.libbase_ver: 318 return self.libbase_ver 319 320 self.libbase_ver = os.environ.get('BASE_VER', '') 321 if not self.libbase_ver: 322 base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER') 323 try: 324 with open(base_file, 'r') as f: 325 self.libbase_ver = f.read().strip('\n') 326 except: 327 self.libbase_ver = 'NOT-INSTALLED' 328 329 return self.libbase_ver 330 331 def _gn_default_output(self): 332 return os.path.join(self.output_dir, 'out/Default') 333 334 def _gn_configure(self): 335 """ Configure all required parameters for platform2. 336 337 Mostly copied from //common-mk/platform2.py 338 """ 339 clang = not self.args.no_clang 340 341 def to_gn_string(s): 342 return '"%s"' % s.replace('"', '\\"') 343 344 def to_gn_list(strs): 345 return '[%s]' % ','.join([to_gn_string(s) for s in strs]) 346 347 def to_gn_args_args(gn_args): 348 for k, v in gn_args.items(): 349 if isinstance(v, bool): 350 v = str(v).lower() 351 elif isinstance(v, list): 352 v = to_gn_list(v) 353 elif isinstance(v, six.string_types): 354 v = to_gn_string(v) 355 else: 356 raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v)) 357 yield '%s=%s' % (k.replace('-', '_'), v) 358 359 gn_args = { 360 'platform_subdir': 'bt', 361 'cc': 'clang' if clang else 'gcc', 362 'cxx': 'clang++' if clang else 'g++', 363 'ar': 'llvm-ar' if clang else 'ar', 364 'pkg-config': 'pkg-config', 365 'clang_cc': clang, 366 'clang_cxx': clang, 367 'OS': 'linux', 368 'sysroot': self.sysroot, 369 'libdir': os.path.join(self.sysroot, self.libdir), 370 'build_root': self.output_dir, 371 'platform2_root': self.platform_dir, 372 'libbase_ver': self._get_basever(), 373 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1', 374 'external_cflags': [], 375 'external_cxxflags': ["-DNDEBUG"], 376 'enable_werror': False, 377 } 378 379 if clang: 380 # Make sure to mark the clang use flag as true 381 self.use.set_flag('clang', True) 382 gn_args['external_cxxflags'] += ['-I/usr/include/'] 383 384 gn_args_args = list(to_gn_args_args(gn_args)) 385 use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()] 386 gn_args_args += ['use={%s}' % (' '.join(use_args))] 387 388 gn_args = [ 389 'gn', 390 'gen', 391 ] 392 393 if self.args.verbose: 394 gn_args.append('-v') 395 396 gn_args += [ 397 '--root=%s' % self.platform_dir, 398 '--args=%s' % ' '.join(gn_args_args), 399 self._gn_default_output(), 400 ] 401 402 if 'PKG_CONFIG_PATH' in self.env: 403 print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH']) 404 405 self.run_command('configure', gn_args) 406 407 def _gn_build(self, target): 408 """ Generate the ninja command for the target and run it. 409 """ 410 args = ['%s:%s' % ('bt', target)] 411 ninja_args = ['ninja', '-C', self._gn_default_output()] 412 if self.jobs: 413 ninja_args += ['-j', str(self.jobs)] 414 ninja_args += args 415 416 if self.args.verbose: 417 ninja_args.append('-v') 418 419 self.run_command('build', ninja_args) 420 421 def _rust_configure(self): 422 """ Generate config file at cargo_home so we use vendored crates. 423 """ 424 template = """ 425 [source.systembt] 426 directory = "{}/external/rust/vendor" 427 428 [source.crates-io] 429 replace-with = "systembt" 430 local-registry = "/nonexistent" 431 """ 432 433 if not self.args.no_vendored_rust: 434 contents = template.format(self.platform_dir) 435 with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f: 436 f.write(contents) 437 438 def _rust_build(self): 439 """ Run `cargo build` from platform2/bt directory. 440 """ 441 cmd = ['cargo', 'build'] 442 if not self.args.rust_debug: 443 cmd.append('--release') 444 445 self.run_command('rust', cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) 446 447 def _target_prepare(self): 448 """ Target to prepare the output directory for building. 449 450 This runs gn gen to generate all rquired files and set up the Rust 451 config properly. This will be run 452 """ 453 self._gn_configure() 454 self._rust_configure() 455 456 def _target_hosttools(self): 457 """ Build the tools target in an already prepared environment. 458 """ 459 self._gn_build('tools') 460 461 # Also copy bluetooth_packetgen to CARGO_HOME so it's available 462 shutil.copy(os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), 463 os.path.join(self.env['CARGO_HOME'], 'bin')) 464 465 def _target_docs(self): 466 """Build the Rust docs.""" 467 self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) 468 469 def _target_rust(self): 470 """ Build rust artifacts in an already prepared environment. 471 """ 472 self._rust_build() 473 474 def _target_main(self): 475 """ Build the main GN artifacts in an already prepared environment. 476 """ 477 self._gn_build('all') 478 479 def _target_test(self): 480 """ Runs the host tests. 481 """ 482 # Rust tests first 483 rust_test_cmd = ['cargo', 'test'] 484 if not self.args.rust_debug: 485 rust_test_cmd.append('--release') 486 487 if self.args.test_name: 488 rust_test_cmd = rust_test_cmd + [self.args.test_name, "--", "--test-threads=1", "--nocapture"] 489 490 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) 491 492 # Host tests second based on host test list 493 for t in HOST_TESTS: 494 self.run_command('test', [os.path.join(self.output_dir, 'out/Default', t)], 495 cwd=os.path.join(self.output_dir), 496 env=self.env) 497 498 def _target_clippy(self): 499 """ Runs cargo clippy, a collection of lints to catch common mistakes. 500 """ 501 cmd = ['cargo', 'clippy'] 502 self.run_command('rust', cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) 503 504 def _target_utils(self): 505 """ Builds the utility applications. 506 """ 507 rust_targets = ['hcidoc'] 508 509 # Build targets 510 for target in rust_targets: 511 self.run_command('utils', ['cargo', 'build', '-p', target], 512 cwd=os.path.join(self.platform_dir, 'bt'), 513 env=self.env) 514 515 def _target_install(self): 516 """ Installs files required to run Floss to install directory. 517 """ 518 # First make the install directory 519 prefix = self.install_dir 520 os.makedirs(prefix, exist_ok=True) 521 522 # Next save the cwd and change to install directory 523 last_cwd = os.getcwd() 524 os.chdir(prefix) 525 526 bindir = os.path.join(self.output_dir, 'debug') 527 srcdir = os.path.dirname(__file__) 528 529 install_map = [ 530 { 531 'src': os.path.join(bindir, 'btadapterd'), 532 'dst': 'usr/libexec/bluetooth/btadapterd', 533 'strip': True 534 }, 535 { 536 'src': os.path.join(bindir, 'btmanagerd'), 537 'dst': 'usr/libexec/bluetooth/btmanagerd', 538 'strip': True 539 }, 540 { 541 'src': os.path.join(bindir, 'btclient'), 542 'dst': 'usr/local/bin/btclient', 543 'strip': True 544 }, 545 ] 546 547 for v in install_map: 548 src, partial_dst, strip = (v['src'], v['dst'], v['strip']) 549 dst = os.path.join(prefix, partial_dst) 550 551 # Create dst directory first and copy file there 552 os.makedirs(os.path.dirname(dst), exist_ok=True) 553 print('Installing {}'.format(dst)) 554 shutil.copy(src, dst) 555 556 # Binary should be marked for strip and no-strip option shouldn't be 557 # set. No-strip is useful while debugging. 558 if strip and not self.args.no_strip: 559 self.run_command('install', ['llvm-strip', dst]) 560 561 # Put all files into a tar.gz for easier installation 562 tar_location = os.path.join(prefix, 'floss.tar.gz') 563 with tarfile.open(tar_location, 'w:gz') as tar: 564 for v in install_map: 565 tar.add(v['dst']) 566 567 print('Tarball created at {}'.format(tar_location)) 568 569 def _target_clean(self): 570 """ Delete the output directory entirely. 571 """ 572 shutil.rmtree(self.output_dir) 573 574 # Remove Cargo.lock that may have become generated 575 cargo_lock_files = [ 576 os.path.join(self.platform_dir, 'bt', 'Cargo.lock'), 577 ] 578 for lock_file in cargo_lock_files: 579 try: 580 os.remove(lock_file) 581 print('Removed {}'.format(lock_file)) 582 except FileNotFoundError: 583 pass 584 585 def _target_all(self): 586 """ Build all common targets (skipping doc, test, and clean). 587 """ 588 self._target_prepare() 589 self._target_hosttools() 590 self._target_main() 591 self._target_rust() 592 593 def build(self): 594 """ Builds according to self.target 595 """ 596 print('Building target ', self.target) 597 598 # Validate that the target is valid 599 if self.target not in VALID_TARGETS: 600 print('Target {} is not valid. Must be in {}'.format(self.target, VALID_TARGETS)) 601 return 602 603 if self.target == 'prepare': 604 self._target_prepare() 605 elif self.target == 'hosttools': 606 self._target_hosttools() 607 elif self.target == 'rust': 608 self._target_rust() 609 elif self.target == 'docs': 610 self._target_docs() 611 elif self.target == 'main': 612 self._target_main() 613 elif self.target == 'test': 614 self._target_test() 615 elif self.target == 'clippy': 616 self._target_clippy() 617 elif self.target == 'clean': 618 self._target_clean() 619 elif self.target == 'install': 620 self._target_install() 621 elif self.target == 'utils': 622 self._target_utils() 623 elif self.target == 'all': 624 self._target_all() 625 626 627# Default to 10 min timeouts on all git operations. 628GIT_TIMEOUT_SEC = 600 629 630 631class Bootstrap(): 632 633 def __init__(self, base_dir, bt_dir, partial_staging, clone_timeout): 634 """ Construct bootstrapper. 635 636 Args: 637 base_dir: Where to stage everything. 638 bt_dir: Where bluetooth source is kept (will be symlinked) 639 partial_staging: Whether to do a partial clone for staging. 640 clone_timeout: Timeout for clone operations. 641 """ 642 self.base_dir = os.path.abspath(base_dir) 643 self.bt_dir = os.path.abspath(bt_dir) 644 self.partial_staging = partial_staging 645 self.clone_timeout = clone_timeout 646 647 # Create base directory if it doesn't already exist 648 os.makedirs(self.base_dir, exist_ok=True) 649 650 if not os.path.isdir(self.bt_dir): 651 raise Exception('{} is not a valid directory'.format(self.bt_dir)) 652 653 self.git_dir = os.path.join(self.base_dir, 'repos') 654 self.staging_dir = os.path.join(self.base_dir, 'staging') 655 self.output_dir = os.path.join(self.base_dir, 'output') 656 self.external_dir = os.path.join(self.base_dir, 'staging', 'external') 657 658 self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete') 659 660 def _run_with_timeout(self, cmd, cwd, timeout=None): 661 """Runs a command using subprocess.check_output. """ 662 print('Running command: {} [at cwd={}]'.format(' '.join(cmd), cwd)) 663 with subprocess.Popen(cmd, cwd=cwd) as proc: 664 try: 665 outs, errs = proc.communicate(timeout=timeout) 666 except subprocess.TimeoutExpired: 667 proc.kill() 668 outs, errs = proc.communicate() 669 print('Timeout on {}'.format(' '.join(cmd)), file=sys.stderr) 670 raise 671 672 if proc.returncode != 0: 673 raise Exception('Cmd {} had return code {}'.format(' '.join(cmd), proc.returncode)) 674 675 def _update_platform2(self): 676 """Updates repositories used for build.""" 677 for project in BOOTSTRAP_GIT_REPOS.keys(): 678 cwd = os.path.join(self.git_dir, project) 679 (repo, commit) = BOOTSTRAP_GIT_REPOS[project] 680 681 # Update to required commit when necessary or pull the latest code. 682 if commit is not None: 683 head = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=cwd).strip() 684 if head != commit: 685 subprocess.check_call(['git', 'fetch'], cwd=cwd) 686 subprocess.check_call(['git', 'checkout', commit], cwd=cwd) 687 else: 688 subprocess.check_call(['git', 'pull'], cwd=cwd) 689 690 def _setup_platform2(self): 691 """ Set up platform2. 692 693 This will check out all the git repos and symlink everything correctly. 694 """ 695 696 # Create all directories we will need to use 697 for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]: 698 os.makedirs(dirpath, exist_ok=True) 699 700 # If already set up, only update platform2 701 if os.path.isfile(self.dir_setup_complete): 702 print('{} already set-up. Updating instead.'.format(self.base_dir)) 703 self._update_platform2() 704 else: 705 clone_options = [] 706 # When doing a partial staging, we use a treeless clone which allows 707 # us to access all commits but downloads things on demand. This 708 # helps speed up the initial git clone during builds but isn't good 709 # for long-term development. 710 if self.partial_staging: 711 clone_options = ['--filter=tree:0'] 712 # Check out all repos in git directory 713 for project in BOOTSTRAP_GIT_REPOS.keys(): 714 (repo, commit) = BOOTSTRAP_GIT_REPOS[project] 715 716 # Try repo clone several times. 717 # Currently, we set timeout on this operation after 718 # |self.clone_timeout|. If it fails, try to recover. 719 tries = 2 720 for x in range(tries): 721 try: 722 self._run_with_timeout(['git', 'clone', repo, project] + clone_options, 723 cwd=self.git_dir, 724 timeout=self.clone_timeout) 725 except subprocess.TimeoutExpired: 726 shutil.rmtree(os.path.join(self.git_dir, project)) 727 if x == tries - 1: 728 raise 729 # All other exceptions should raise 730 except: 731 raise 732 # No exceptions/problems should not retry. 733 else: 734 break 735 736 # Pin to commit. 737 if commit is not None: 738 subprocess.check_call(['git', 'checkout', commit], cwd=os.path.join(self.git_dir, project)) 739 740 # Symlink things 741 symlinks = [ 742 (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')), 743 (os.path.join(self.git_dir, 'platform2', 'system_api'), os.path.join(self.staging_dir, 'system_api')), 744 (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')), 745 (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')), 746 (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')), 747 (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')), 748 ] 749 750 # Create symlinks 751 for pairs in symlinks: 752 (src, dst) = pairs 753 try: 754 os.unlink(dst) 755 except Exception as e: 756 print(e) 757 os.symlink(src, dst) 758 759 # Write to setup complete file so we don't repeat this step 760 with open(self.dir_setup_complete, 'w') as f: 761 f.write('Setup complete.') 762 763 def _pretty_print_install(self, install_cmd, packages, line_limit=80): 764 """ Pretty print an install command. 765 766 Args: 767 install_cmd: Prefixed install command. 768 packages: Enumerate packages and append them to install command. 769 line_limit: Number of characters per line. 770 771 Return: 772 Array of lines to join and print. 773 """ 774 install = [install_cmd] 775 line = ' ' 776 # Remainder needed = space + len(pkg) + space + \ 777 # Assuming 80 character lines, that's 80 - 3 = 77 778 line_limit = line_limit - 3 779 for pkg in packages: 780 if len(line) + len(pkg) < line_limit: 781 line = '{}{} '.format(line, pkg) 782 else: 783 install.append(line) 784 line = ' {} '.format(pkg) 785 786 if len(line) > 0: 787 install.append(line) 788 789 return install 790 791 def _check_package_installed(self, package, cmd, predicate): 792 """Check that the given package is installed. 793 794 Args: 795 package: Check that this package is installed. 796 cmd: Command prefix to check if installed (package appended to end) 797 predicate: Function/lambda to check if package is installed based 798 on output. Takes string output and returns boolean. 799 800 Return: 801 True if package is installed. 802 """ 803 try: 804 output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT) 805 is_installed = predicate(output.decode('utf-8')) 806 print(' {} is {}'.format(package, 'installed' if is_installed else 'missing')) 807 808 return is_installed 809 except Exception as e: 810 print(e) 811 return False 812 813 def _get_command_output(self, cmd): 814 """Runs the command and gets the output. 815 816 Args: 817 cmd: Command to run. 818 819 Return: 820 Tuple (Success, Output). Success represents if the command ran ok. 821 """ 822 try: 823 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) 824 return (True, output.decode('utf-8').split('\n')) 825 except Exception as e: 826 print(e) 827 return (False, "") 828 829 def _print_missing_packages(self): 830 """Print any missing packages found via apt. 831 832 This will find any missing packages necessary for build using apt and 833 print it out as an apt-get install printf. 834 """ 835 print('Checking for any missing packages...') 836 837 (success, output) = self._get_command_output(APT_PKG_LIST) 838 if not success: 839 raise Exception("Could not query apt for packages.") 840 841 packages_installed = {} 842 for line in output: 843 if 'installed' in line: 844 split = line.split('/', 2) 845 packages_installed[split[0]] = True 846 847 need_packages = [] 848 for pkg in REQUIRED_APT_PACKAGES: 849 if pkg not in packages_installed: 850 need_packages.append(pkg) 851 852 # No packages need to be installed 853 if len(need_packages) == 0: 854 print('+ All required packages are installed') 855 return 856 857 install = self._pretty_print_install('sudo apt-get install', need_packages) 858 859 # Print all lines so they can be run in cmdline 860 print('Missing system packages. Run the following command: ') 861 print(' \\\n'.join(install)) 862 863 def _print_missing_rust_packages(self): 864 """Print any missing packages found via cargo. 865 866 This will find any missing packages necessary for build using cargo and 867 print it out as a cargo-install printf. 868 """ 869 print('Checking for any missing cargo packages...') 870 871 (success, output) = self._get_command_output(CARGO_PKG_LIST) 872 if not success: 873 raise Exception("Could not query cargo for packages.") 874 875 packages_installed = {} 876 for line in output: 877 # Cargo installed packages have this format 878 # <crate name> <version>: 879 # <binary name> 880 # We only care about the crates themselves 881 if ':' not in line: 882 continue 883 884 split = line.split(' ', 2) 885 packages_installed[split[0]] = True 886 887 need_packages = [] 888 for pkg in REQUIRED_CARGO_PACKAGES: 889 if pkg not in packages_installed: 890 need_packages.append(pkg) 891 892 # No packages to be installed 893 if len(need_packages) == 0: 894 print('+ All required cargo packages are installed') 895 return 896 897 install = self._pretty_print_install('cargo install', need_packages) 898 print('Missing cargo packages. Run the following command: ') 899 print(' \\\n'.join(install)) 900 901 def bootstrap(self): 902 """ Bootstrap the Linux build.""" 903 self._setup_platform2() 904 self._print_missing_packages() 905 self._print_missing_rust_packages() 906 907 908if __name__ == '__main__': 909 parser = argparse.ArgumentParser(description='Simple build for host.') 910 parser.add_argument('--bootstrap-dir', 911 help='Directory to run bootstrap on (or was previously run on).', 912 default="~/.floss") 913 parser.add_argument('--run-bootstrap', 914 help='Run bootstrap code to verify build env is ok to build.', 915 default=False, 916 action='store_true') 917 parser.add_argument('--print-env', 918 help='Print environment variables used for build.', 919 default=False, 920 action='store_true') 921 parser.add_argument('--no-clang', help='Don\'t use clang compiler.', default=False, action='store_true') 922 parser.add_argument('--no-strip', 923 help='Skip stripping binaries during install.', 924 default=False, 925 action='store_true') 926 parser.add_argument('--use', help='Set a specific use flag.') 927 parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true') 928 parser.add_argument('--test-name', help='Run test with this string in the name.', default=None) 929 parser.add_argument('--target', help='Run specific build target') 930 parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/') 931 parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib') 932 parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int) 933 parser.add_argument('--no-vendored-rust', 934 help='Do not use vendored rust crates', 935 default=False, 936 action='store_true') 937 parser.add_argument('--verbose', help='Verbose logs for build.') 938 parser.add_argument('--rust-debug', help='Build Rust code as debug.', default=False, action='store_true') 939 parser.add_argument( 940 '--partial-staging', 941 help='Bootstrap git repositories with partial clones. Use to speed up initial git clone for automated builds.', 942 default=False, 943 action='store_true') 944 parser.add_argument('--clone-timeout', 945 help='Timeout for repository cloning during bootstrap.', 946 default=GIT_TIMEOUT_SEC, 947 type=int) 948 args = parser.parse_args() 949 950 # Make sure we get absolute path + expanded path for bootstrap directory 951 args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir)) 952 953 # Possible values for machine() come from 'uname -m' 954 # Since this script only runs on Linux, x86_64 machines must have this value 955 if platform.machine() != 'x86_64': 956 raise Exception("Only x86_64 machines are currently supported by this build script.") 957 958 if args.run_bootstrap: 959 bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__), args.partial_staging, args.clone_timeout) 960 bootstrap.bootstrap() 961 elif args.print_env: 962 build = HostBuild(args) 963 build.print_env() 964 else: 965 build = HostBuild(args) 966 build.build() 967