1#!/usr/bin/env python3 2 3# Copyright 2021 Google, Inc. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at: 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16""" Build BT targets on the host system. 17 18For building, you will first have to stage a platform directory that has the 19following structure: 20|-common-mk 21|-bt 22|-external 23|-|-rust 24|-|-|-vendor 25 26The simplest way to do this is to check out platform2 to another directory (that 27is not a subdir of this bt directory), symlink bt there and symlink the rust 28vendor repository as well. 29""" 30import argparse 31import multiprocessing 32import os 33import platform 34import shutil 35import six 36import subprocess 37import sys 38import tarfile 39import time 40 41# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {}) 42COMMON_MK_USES = [ 43 'asan', 44 'coverage', 45 'cros_host', 46 'cros_debug', 47 'floss_rootcanal', 48 'function_elimination_experiment', 49 'fuzzer', 50 'fuzzer', 51 'lto_experiment', 52 'msan', 53 'profiling', 54 'proto_force_optimize_speed', 55 'tcmalloc', 56 'test', 57 'ubsan', 58] 59 60# Use a specific commit version for common-mk to avoid build surprises. 61COMMON_MK_COMMIT = "d014d561eaf5ece08166edd98b10c145ef81312d" 62 63# Default use flags. 64USE_DEFAULTS = { 65 'android': False, 66 'bt_nonstandard_codecs': False, 67 'test': False, 68} 69 70VALID_TARGETS = [ 71 'all', # All targets except test and clean 72 'bloat', # Check bloat of crates 73 'clean', # Clean up output directory 74 'docs', # Build Rust docs 75 'hosttools', # Build the host tools (i.e. packetgen) 76 'main', # Build the main C++ codebase 77 'prepare', # Prepare the output directory (gn gen + rust setup) 78 'rust', # Build only the rust components + copy artifacts to output dir 79 'test', # Run the unit tests 80 'clippy', # Run cargo clippy 81 'utils', # Build Floss utils 82] 83 84# TODO(b/190750167) - Host tests are disabled until we are full bazel build 85HOST_TESTS = [ 86 # 'bluetooth_test_common', 87 # 'bluetoothtbd_test', 88 # 'net_test_avrcp', 89 # 'net_test_types', 90 # 'net_test_btm_iso', 91 # 'net_test_btpackets', 92] 93 94# Map of git repos to bootstrap and what commit to check them out at. None 95# values will just checkout to HEAD. 96BOOTSTRAP_GIT_REPOS = { 97 'platform2': ('https://chromium.googlesource.com/chromiumos/platform2', COMMON_MK_COMMIT), 98 'rust_crates': ('https://chromium.googlesource.com/chromiumos/third_party/rust_crates', None), 99 'proto_logging': ('https://android.googlesource.com/platform/frameworks/proto_logging', None), 100} 101 102# List of packages required for linux build 103REQUIRED_APT_PACKAGES = [ 104 'bison', 105 'build-essential', 106 'curl', 107 'debmake', 108 'flatbuffers-compiler', 109 'flex', 110 'g++-multilib', 111 'gcc-multilib', 112 'generate-ninja', 113 'gnupg', 114 'gperf', 115 'libabsl-dev', 116 'libc++abi-dev', 117 'libc++-dev', 118 'libdbus-1-dev', 119 'libdouble-conversion-dev', 120 'libevent-dev', 121 'libevent-dev', 122 'libflatbuffers-dev', 123 'libfmt-dev', 124 'libgl1-mesa-dev', 125 'libglib2.0-dev', 126 'libgtest-dev', 127 'libgmock-dev', 128 'liblc3-dev', 129 'liblz4-tool', 130 'libncurses5', 131 'libnss3-dev', 132 'libfmt-dev', 133 'libprotobuf-dev', 134 'libre2-9', 135 'libre2-dev', 136 'libssl-dev', 137 'libtinyxml2-dev', 138 'libx11-dev', 139 'libxml2-utils', 140 'ninja-build', 141 'openssl', 142 'protobuf-compiler', 143 'unzip', 144 'x11proto-core-dev', 145 'xsltproc', 146 'zip', 147 'zlib1g-dev', 148] 149 150# List of cargo packages required for linux build 151REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd', 'pdl-compiler', 'grpcio-compiler', 'cargo-bloat'] 152 153APT_PKG_LIST = ['apt', '-qq', 'list'] 154CARGO_PKG_LIST = ['cargo', 'install', '--list'] 155 156 157class UseFlags(): 158 159 def __init__(self, use_flags): 160 """ Construct the use flags. 161 162 Args: 163 use_flags: List of use flags parsed from the command. 164 """ 165 self.flags = {} 166 167 # Import use flags required by common-mk 168 for use in COMMON_MK_USES: 169 self.set_flag(use, False) 170 171 # Set our defaults 172 for use, value in USE_DEFAULTS.items(): 173 self.set_flag(use, value) 174 175 # Set use flags - value is set to True unless the use starts with - 176 # All given use flags always override the defaults 177 for use in use_flags: 178 value = not use.startswith('-') 179 self.set_flag(use, value) 180 181 def set_flag(self, key, value=True): 182 setattr(self, key, value) 183 self.flags[key] = value 184 185 186class HostBuild(): 187 188 def __init__(self, args): 189 """ Construct the builder. 190 191 Args: 192 args: Parsed arguments from ArgumentParser 193 """ 194 self.args = args 195 196 # Set jobs to number of cpus unless explicitly set 197 self.jobs = self.args.jobs 198 if not self.jobs: 199 self.jobs = multiprocessing.cpu_count() 200 sys.stderr.write("Number of jobs = {}\n".format(self.jobs)) 201 202 # Normalize bootstrap dir and make sure it exists 203 self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir) 204 os.makedirs(self.bootstrap_dir, exist_ok=True) 205 206 # Output and platform directories are based on bootstrap 207 self.output_dir = os.path.join(self.bootstrap_dir, 'output') 208 self.platform_dir = os.path.join(self.bootstrap_dir, 'staging') 209 self.bt_dir = os.path.join(self.platform_dir, 'bt') 210 self.sysroot = self.args.sysroot 211 self.libdir = self.args.libdir 212 self.install_dir = os.path.join(self.output_dir, 'install') 213 214 assert os.path.samefile(self.bt_dir, 215 os.path.dirname(__file__)), "Please rerun bootstrap for the current project!" 216 217 # If default target isn't set, build everything 218 self.target = 'all' 219 if hasattr(self.args, 'target') and self.args.target: 220 self.target = self.args.target 221 222 target_use = self.args.use if self.args.use else [] 223 224 # Unless set, always build test code 225 if not self.args.notest: 226 target_use.append('test') 227 228 self.use = UseFlags(target_use) 229 230 # Validate platform directory 231 assert os.path.isdir(self.platform_dir), 'Platform dir does not exist' 232 assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root' 233 234 # Make sure output directory exists (or create it) 235 os.makedirs(self.output_dir, exist_ok=True) 236 237 # Set some default attributes 238 self.libbase_ver = None 239 240 self.configure_environ() 241 242 def _generate_rustflags(self): 243 """ Rustflags to include for the build. 244 """ 245 rust_flags = [ 246 '-L', 247 '{}/out/Default'.format(self.output_dir), 248 '-C', 249 'link-arg=-Wl,--allow-multiple-definition', 250 # exclude uninteresting warnings 251 '-A improper_ctypes_definitions -A improper_ctypes -A unknown_lints', 252 '-Cstrip=debuginfo', 253 '-Copt-level=z', 254 ] 255 256 return ' '.join(rust_flags) 257 258 def configure_environ(self): 259 """ Configure environment variables for GN and Cargo. 260 """ 261 self.env = os.environ.copy() 262 263 # Make sure cargo home dir exists and has a bin directory 264 cargo_home = os.path.join(self.output_dir, 'cargo_home') 265 os.makedirs(cargo_home, exist_ok=True) 266 os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True) 267 268 # Configure Rust env variables 269 self.custom_env = {} 270 self.custom_env['CARGO_TARGET_DIR'] = self.output_dir 271 self.custom_env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home') 272 self.custom_env['RUSTFLAGS'] = self._generate_rustflags() 273 self.custom_env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt') 274 self.custom_env['CROS_SYSTEM_API_ROOT'] = os.path.join(self.platform_dir, 'system_api') 275 self.custom_env['CXX_OUTDIR'] = self._gn_default_output() 276 277 # On ChromeOS, this is /usr/bin/grpc_rust_plugin 278 # In the container, this is /root/.cargo/bin/grpc_rust_plugin 279 self.custom_env['GRPC_RUST_PLUGIN_PATH'] = shutil.which('grpc_rust_plugin') 280 self.env.update(self.custom_env) 281 282 def print_env(self): 283 """ Print the custom environment variables that are used in build. 284 285 Useful so that external tools can mimic the environment to be the same 286 as build.py, e.g. rust-analyzer. 287 """ 288 for k, v in self.custom_env.items(): 289 print("export {}='{}'".format(k, v)) 290 291 def run_command(self, target, args, cwd=None, env=None): 292 """ Run command and stream the output. 293 """ 294 # Set some defaults 295 if not cwd: 296 cwd = self.platform_dir 297 if not env: 298 env = self.env 299 300 for k, v in env.items(): 301 if env[k] is None: 302 env[k] = "" 303 304 log_file = os.path.join(self.output_dir, '{}.log'.format(target)) 305 with open(log_file, 'wb') as lf: 306 rc = 0 307 process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE) 308 while True: 309 line = process.stdout.readline() 310 print(line.decode('utf-8'), end="") 311 lf.write(line) 312 if not line: 313 rc = process.poll() 314 if rc is not None: 315 break 316 317 time.sleep(0.1) 318 319 if rc != 0: 320 raise Exception("Return code is {}".format(rc)) 321 322 def _get_basever(self): 323 if self.libbase_ver: 324 return self.libbase_ver 325 326 self.libbase_ver = os.environ.get('BASE_VER', '') 327 if not self.libbase_ver: 328 base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER') 329 try: 330 with open(base_file, 'r') as f: 331 self.libbase_ver = f.read().strip('\n') 332 except: 333 self.libbase_ver = 'NOT-INSTALLED' 334 335 return self.libbase_ver 336 337 def _gn_default_output(self): 338 return os.path.join(self.output_dir, 'out/Default') 339 340 def _gn_configure(self): 341 """ Configure all required parameters for platform2. 342 343 Mostly copied from //common-mk/platform2.py 344 """ 345 clang = not self.args.no_clang 346 347 def to_gn_string(s): 348 return '"%s"' % s.replace('"', '\\"') 349 350 def to_gn_list(strs): 351 return '[%s]' % ','.join([to_gn_string(s) for s in strs]) 352 353 def to_gn_args_args(gn_args): 354 for k, v in gn_args.items(): 355 if isinstance(v, bool): 356 v = str(v).lower() 357 elif isinstance(v, list): 358 v = to_gn_list(v) 359 elif isinstance(v, six.string_types): 360 v = to_gn_string(v) 361 else: 362 raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v)) 363 yield '%s=%s' % (k.replace('-', '_'), v) 364 365 gn_args = { 366 'platform_subdir': 'bt', 367 'cc': 'clang' if clang else 'gcc', 368 'cxx': 'clang++' if clang else 'g++', 369 'ar': 'llvm-ar' if clang else 'ar', 370 'pkg-config': 'pkg-config', 371 'clang_cc': clang, 372 'clang_cxx': clang, 373 'OS': 'linux', 374 'sysroot': self.sysroot, 375 'libdir': os.path.join(self.sysroot, self.libdir), 376 'build_root': self.output_dir, 377 'platform2_root': self.platform_dir, 378 'libbase_ver': self._get_basever(), 379 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1', 380 'external_cflags': [], 381 'external_cxxflags': ["-DNDEBUG"], 382 'enable_werror': True, 383 } 384 385 if clang: 386 # Make sure to mark the clang use flag as true 387 self.use.set_flag('clang', True) 388 gn_args['external_cxxflags'] += ['-I/usr/include/'] 389 390 gn_args_args = list(to_gn_args_args(gn_args)) 391 use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()] 392 gn_args_args += ['use={%s}' % (' '.join(use_args))] 393 394 gn_args = [ 395 'gn', 396 'gen', 397 ] 398 399 if self.args.verbose: 400 gn_args.append('-v') 401 402 gn_args += [ 403 '--root=%s' % self.platform_dir, 404 '--args=%s' % ' '.join(gn_args_args), 405 self._gn_default_output(), 406 ] 407 408 if 'PKG_CONFIG_PATH' in self.env: 409 print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH']) 410 411 self.run_command('configure', gn_args) 412 413 def _gn_build(self, target): 414 """ Generate the ninja command for the target and run it. 415 """ 416 args = ['%s:%s' % ('bt', target)] 417 ninja_args = ['ninja', '-C', self._gn_default_output()] 418 if self.jobs: 419 ninja_args += ['-j', str(self.jobs)] 420 ninja_args += args 421 422 if self.args.verbose: 423 ninja_args.append('-v') 424 425 self.run_command('build', ninja_args) 426 427 def _rust_configure(self): 428 """ Generate config file at cargo_home so we use vendored crates. 429 """ 430 template = """ 431 [source.systembt] 432 directory = "{}/external/rust/vendor" 433 434 [source.crates-io] 435 replace-with = "systembt" 436 local-registry = "/nonexistent" 437 """ 438 439 if not self.args.no_vendored_rust: 440 contents = template.format(self.platform_dir) 441 with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f: 442 f.write(contents) 443 444 def _rust_build(self): 445 """ Run `cargo build` from platform2/bt directory. 446 """ 447 cmd = ['cargo', 'build'] 448 if not self.args.rust_debug: 449 cmd.append('--release') 450 451 self.run_command('rust', cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) 452 453 def _target_prepare(self): 454 """ Target to prepare the output directory for building. 455 456 This runs gn gen to generate all rquired files and set up the Rust 457 config properly. This will be run 458 """ 459 self._gn_configure() 460 self._rust_configure() 461 462 def _target_hosttools(self): 463 """ Build the tools target in an already prepared environment. 464 """ 465 self._gn_build('tools') 466 467 # Also copy bluetooth_packetgen to CARGO_HOME so it's available 468 shutil.copy(os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), 469 os.path.join(self.env['CARGO_HOME'], 'bin')) 470 471 def _target_docs(self): 472 """Build the Rust docs.""" 473 self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) 474 475 def _target_rust(self): 476 """ Build rust artifacts in an already prepared environment. 477 """ 478 self._rust_build() 479 480 def _target_main(self): 481 """ Build the main GN artifacts in an already prepared environment. 482 """ 483 self._gn_build('all') 484 485 def _target_test(self): 486 """ Runs the host tests. 487 """ 488 # Rust tests first 489 rust_test_cmd = ['cargo', 'test'] 490 if not self.args.rust_debug: 491 rust_test_cmd.append('--release') 492 493 if self.args.test_name: 494 rust_test_cmd = rust_test_cmd + [self.args.test_name, "--", "--test-threads=1", "--nocapture"] 495 496 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) 497 498 # Host tests second based on host test list 499 for t in HOST_TESTS: 500 self.run_command('test', [os.path.join(self.output_dir, 'out/Default', t)], 501 cwd=os.path.join(self.output_dir), 502 env=self.env) 503 504 def _target_clippy(self): 505 """ Runs cargo clippy, a collection of lints to catch common mistakes. 506 """ 507 cmd = ['cargo', 'clippy'] 508 self.run_command('rust', cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) 509 510 def _target_utils(self): 511 """ Builds the utility applications. 512 """ 513 rust_targets = ['hcidoc'] 514 515 # Build targets 516 for target in rust_targets: 517 self.run_command('utils', ['cargo', 'build', '-p', target], 518 cwd=os.path.join(self.platform_dir, 'bt'), 519 env=self.env) 520 521 def _target_install(self): 522 """ Installs files required to run Floss to install directory. 523 """ 524 # First make the install directory 525 prefix = self.install_dir 526 os.makedirs(prefix, exist_ok=True) 527 528 # Next save the cwd and change to install directory 529 last_cwd = os.getcwd() 530 os.chdir(prefix) 531 532 bindir = os.path.join(self.output_dir, 'debug') 533 srcdir = os.path.dirname(__file__) 534 535 install_map = [ 536 { 537 'src': os.path.join(bindir, 'btadapterd'), 538 'dst': 'usr/libexec/bluetooth/btadapterd', 539 'strip': True 540 }, 541 { 542 'src': os.path.join(bindir, 'btmanagerd'), 543 'dst': 'usr/libexec/bluetooth/btmanagerd', 544 'strip': True 545 }, 546 { 547 'src': os.path.join(bindir, 'btclient'), 548 'dst': 'usr/local/bin/btclient', 549 'strip': True 550 }, 551 ] 552 553 for v in install_map: 554 src, partial_dst, strip = (v['src'], v['dst'], v['strip']) 555 dst = os.path.join(prefix, partial_dst) 556 557 # Create dst directory first and copy file there 558 os.makedirs(os.path.dirname(dst), exist_ok=True) 559 print('Installing {}'.format(dst)) 560 shutil.copy(src, dst) 561 562 # Binary should be marked for strip and no-strip option shouldn't be 563 # set. No-strip is useful while debugging. 564 if strip and not self.args.no_strip: 565 self.run_command('install', ['llvm-strip', dst]) 566 567 # Put all files into a tar.gz for easier installation 568 tar_location = os.path.join(prefix, 'floss.tar.gz') 569 with tarfile.open(tar_location, 'w:gz') as tar: 570 for v in install_map: 571 tar.add(v['dst']) 572 573 print('Tarball created at {}'.format(tar_location)) 574 575 def _target_bloat(self): 576 """Run cargo bloat on workspace. 577 """ 578 crate_paths = [ 579 os.path.join(self.platform_dir, 'bt', 'system', 'gd', 'rust', 'linux', 'mgmt'), 580 os.path.join(self.platform_dir, 'bt', 'system', 'gd', 'rust', 'linux', 'service'), 581 os.path.join(self.platform_dir, 'bt', 'system', 'gd', 'rust', 'linux', 'client') 582 ] 583 for crate in crate_paths: 584 self.run_command('bloat', ['cargo', 'bloat', '--release', '--crates', '--wide'], cwd=crate, env=self.env) 585 586 def _target_clean(self): 587 """ Delete the output directory entirely. 588 """ 589 shutil.rmtree(self.output_dir) 590 591 # Remove Cargo.lock that may have become generated 592 cargo_lock_files = [ 593 os.path.join(self.platform_dir, 'bt', 'Cargo.lock'), 594 ] 595 for lock_file in cargo_lock_files: 596 try: 597 os.remove(lock_file) 598 print('Removed {}'.format(lock_file)) 599 except FileNotFoundError: 600 pass 601 602 def _target_all(self): 603 """ Build all common targets (skipping doc, test, and clean). 604 """ 605 self._target_prepare() 606 self._target_hosttools() 607 self._target_main() 608 self._target_rust() 609 610 def build(self): 611 """ Builds according to self.target 612 """ 613 print('Building target ', self.target) 614 615 # Validate that the target is valid 616 if self.target not in VALID_TARGETS: 617 print('Target {} is not valid. Must be in {}'.format(self.target, VALID_TARGETS)) 618 return 619 620 if self.target == 'prepare': 621 self._target_prepare() 622 elif self.target == 'hosttools': 623 self._target_hosttools() 624 elif self.target == 'rust': 625 self._target_rust() 626 elif self.target == 'docs': 627 self._target_docs() 628 elif self.target == 'main': 629 self._target_main() 630 elif self.target == 'test': 631 self._target_test() 632 elif self.target == 'clippy': 633 self._target_clippy() 634 elif self.target == 'clean': 635 self._target_clean() 636 elif self.target == 'install': 637 self._target_install() 638 elif self.target == 'utils': 639 self._target_utils() 640 elif self.target == 'bloat': 641 self._target_bloat() 642 elif self.target == 'all': 643 self._target_all() 644 645 646# Default to 10 min timeouts on all git operations. 647GIT_TIMEOUT_SEC = 600 648 649 650class Bootstrap(): 651 652 def __init__(self, base_dir, bt_dir, partial_staging, clone_timeout): 653 """ Construct bootstrapper. 654 655 Args: 656 base_dir: Where to stage everything. 657 bt_dir: Where bluetooth source is kept (will be symlinked) 658 partial_staging: Whether to do a partial clone for staging. 659 clone_timeout: Timeout for clone operations. 660 """ 661 self.base_dir = os.path.abspath(base_dir) 662 self.bt_dir = os.path.abspath(bt_dir) 663 self.partial_staging = partial_staging 664 self.clone_timeout = clone_timeout 665 666 # Create base directory if it doesn't already exist 667 os.makedirs(self.base_dir, exist_ok=True) 668 669 if not os.path.isdir(self.bt_dir): 670 raise Exception('{} is not a valid directory'.format(self.bt_dir)) 671 672 self.git_dir = os.path.join(self.base_dir, 'repos') 673 self.staging_dir = os.path.join(self.base_dir, 'staging') 674 self.output_dir = os.path.join(self.base_dir, 'output') 675 self.external_dir = os.path.join(self.base_dir, 'staging', 'external') 676 677 self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete') 678 679 def _run_with_timeout(self, cmd, cwd, timeout=None): 680 """Runs a command using subprocess.check_output. """ 681 print('Running command: {} [at cwd={}]'.format(' '.join(cmd), cwd)) 682 with subprocess.Popen(cmd, cwd=cwd) as proc: 683 try: 684 outs, errs = proc.communicate(timeout=timeout) 685 except subprocess.TimeoutExpired: 686 proc.kill() 687 outs, errs = proc.communicate() 688 print('Timeout on {}'.format(' '.join(cmd)), file=sys.stderr) 689 raise 690 691 if proc.returncode != 0: 692 raise Exception('Cmd {} had return code {}'.format(' '.join(cmd), proc.returncode)) 693 694 def _update_platform2(self): 695 """Updates repositories used for build.""" 696 for project in BOOTSTRAP_GIT_REPOS.keys(): 697 cwd = os.path.join(self.git_dir, project) 698 (repo, commit) = BOOTSTRAP_GIT_REPOS[project] 699 700 # Update to required commit when necessary or pull the latest code. 701 if commit is not None: 702 head = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=cwd).strip() 703 if head != commit: 704 subprocess.check_call(['git', 'fetch'], cwd=cwd) 705 subprocess.check_call(['git', 'checkout', commit], cwd=cwd) 706 else: 707 subprocess.check_call(['git', 'pull'], cwd=cwd) 708 709 def _setup_platform2(self): 710 """ Set up platform2. 711 712 This will check out all the git repos and symlink everything correctly. 713 """ 714 715 # Create all directories we will need to use 716 for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]: 717 os.makedirs(dirpath, exist_ok=True) 718 719 # If already set up, only update platform2 720 if os.path.isfile(self.dir_setup_complete): 721 print('{} already set-up. Updating instead.'.format(self.base_dir)) 722 self._update_platform2() 723 else: 724 clone_options = [] 725 # When doing a partial staging, we use a treeless clone which allows 726 # us to access all commits but downloads things on demand. This 727 # helps speed up the initial git clone during builds but isn't good 728 # for long-term development. 729 if self.partial_staging: 730 clone_options = ['--filter=tree:0'] 731 # Check out all repos in git directory 732 for project in BOOTSTRAP_GIT_REPOS.keys(): 733 (repo, commit) = BOOTSTRAP_GIT_REPOS[project] 734 735 # Try repo clone several times. 736 # Currently, we set timeout on this operation after 737 # |self.clone_timeout|. If it fails, try to recover. 738 tries = 2 739 for x in range(tries): 740 try: 741 self._run_with_timeout(['git', 'clone', repo, project] + clone_options, 742 cwd=self.git_dir, 743 timeout=self.clone_timeout) 744 except subprocess.TimeoutExpired: 745 shutil.rmtree(os.path.join(self.git_dir, project)) 746 if x == tries - 1: 747 raise 748 # All other exceptions should raise 749 except: 750 raise 751 # No exceptions/problems should not retry. 752 else: 753 break 754 755 # Pin to commit. 756 if commit is not None: 757 subprocess.check_call(['git', 'checkout', commit], cwd=os.path.join(self.git_dir, project)) 758 759 # Symlink things 760 symlinks = [ 761 (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')), 762 (os.path.join(self.git_dir, 'platform2', 'system_api'), os.path.join(self.staging_dir, 'system_api')), 763 (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')), 764 (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')), 765 (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')), 766 (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')), 767 ] 768 769 # Create symlinks 770 for pairs in symlinks: 771 (src, dst) = pairs 772 try: 773 os.unlink(dst) 774 except Exception as e: 775 print(e) 776 os.symlink(src, dst) 777 778 # Write to setup complete file so we don't repeat this step 779 with open(self.dir_setup_complete, 'w') as f: 780 f.write('Setup complete.') 781 782 def _pretty_print_install(self, install_cmd, packages, line_limit=80): 783 """ Pretty print an install command. 784 785 Args: 786 install_cmd: Prefixed install command. 787 packages: Enumerate packages and append them to install command. 788 line_limit: Number of characters per line. 789 790 Return: 791 Array of lines to join and print. 792 """ 793 install = [install_cmd] 794 line = ' ' 795 # Remainder needed = space + len(pkg) + space + \ 796 # Assuming 80 character lines, that's 80 - 3 = 77 797 line_limit = line_limit - 3 798 for pkg in packages: 799 if len(line) + len(pkg) < line_limit: 800 line = '{}{} '.format(line, pkg) 801 else: 802 install.append(line) 803 line = ' {} '.format(pkg) 804 805 if len(line) > 0: 806 install.append(line) 807 808 return install 809 810 def _check_package_installed(self, package, cmd, predicate): 811 """Check that the given package is installed. 812 813 Args: 814 package: Check that this package is installed. 815 cmd: Command prefix to check if installed (package appended to end) 816 predicate: Function/lambda to check if package is installed based 817 on output. Takes string output and returns boolean. 818 819 Return: 820 True if package is installed. 821 """ 822 try: 823 output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT) 824 is_installed = predicate(output.decode('utf-8')) 825 print(' {} is {}'.format(package, 'installed' if is_installed else 'missing')) 826 827 return is_installed 828 except Exception as e: 829 print(e) 830 return False 831 832 def _get_command_output(self, cmd): 833 """Runs the command and gets the output. 834 835 Args: 836 cmd: Command to run. 837 838 Return: 839 Tuple (Success, Output). Success represents if the command ran ok. 840 """ 841 try: 842 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) 843 return (True, output.decode('utf-8').split('\n')) 844 except Exception as e: 845 print(e) 846 return (False, "") 847 848 def _print_missing_packages(self): 849 """Print any missing packages found via apt. 850 851 This will find any missing packages necessary for build using apt and 852 print it out as an apt-get install printf. 853 """ 854 print('Checking for any missing packages...') 855 856 (success, output) = self._get_command_output(APT_PKG_LIST) 857 if not success: 858 raise Exception("Could not query apt for packages.") 859 860 packages_installed = {} 861 for line in output: 862 if 'installed' in line: 863 split = line.split('/', 2) 864 packages_installed[split[0]] = True 865 866 need_packages = [] 867 for pkg in REQUIRED_APT_PACKAGES: 868 if pkg not in packages_installed: 869 need_packages.append(pkg) 870 871 # No packages need to be installed 872 if len(need_packages) == 0: 873 print('+ All required packages are installed') 874 return 875 876 install = self._pretty_print_install('sudo apt-get install', need_packages) 877 878 # Print all lines so they can be run in cmdline 879 print('Missing system packages. Run the following command: ') 880 print(' \\\n'.join(install)) 881 882 def _print_missing_rust_packages(self): 883 """Print any missing packages found via cargo. 884 885 This will find any missing packages necessary for build using cargo and 886 print it out as a cargo-install printf. 887 """ 888 print('Checking for any missing cargo packages...') 889 890 (success, output) = self._get_command_output(CARGO_PKG_LIST) 891 if not success: 892 raise Exception("Could not query cargo for packages.") 893 894 packages_installed = {} 895 for line in output: 896 # Cargo installed packages have this format 897 # <crate name> <version>: 898 # <binary name> 899 # We only care about the crates themselves 900 if ':' not in line: 901 continue 902 903 split = line.split(' ', 2) 904 packages_installed[split[0]] = True 905 906 need_packages = [] 907 for pkg in REQUIRED_CARGO_PACKAGES: 908 if pkg not in packages_installed: 909 need_packages.append(pkg) 910 911 # No packages to be installed 912 if len(need_packages) == 0: 913 print('+ All required cargo packages are installed') 914 return 915 916 install = self._pretty_print_install('cargo install', need_packages) 917 print('Missing cargo packages. Run the following command: ') 918 print(' \\\n'.join(install)) 919 920 def bootstrap(self): 921 """ Bootstrap the Linux build.""" 922 self._setup_platform2() 923 self._print_missing_packages() 924 self._print_missing_rust_packages() 925 926 927if __name__ == '__main__': 928 parser = argparse.ArgumentParser(description='Simple build for host.') 929 parser.add_argument('--bootstrap-dir', 930 help='Directory to run bootstrap on (or was previously run on).', 931 default="~/.floss") 932 parser.add_argument('--run-bootstrap', 933 help='Run bootstrap code to verify build env is ok to build.', 934 default=False, 935 action='store_true') 936 parser.add_argument('--print-env', 937 help='Print environment variables used for build.', 938 default=False, 939 action='store_true') 940 parser.add_argument('--no-clang', help='Don\'t use clang compiler.', default=False, action='store_true') 941 parser.add_argument('--no-strip', 942 help='Skip stripping binaries during install.', 943 default=False, 944 action='store_true') 945 parser.add_argument('--use', help='Set a specific use flag.') 946 parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true') 947 parser.add_argument('--test-name', help='Run test with this string in the name.', default=None) 948 parser.add_argument('--target', help='Run specific build target') 949 parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/') 950 parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib') 951 parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int) 952 parser.add_argument('--no-vendored-rust', 953 help='Do not use vendored rust crates', 954 default=False, 955 action='store_true') 956 parser.add_argument('--verbose', help='Verbose logs for build.') 957 parser.add_argument('--rust-debug', help='Build Rust code as debug.', default=False, action='store_true') 958 parser.add_argument( 959 '--partial-staging', 960 help='Bootstrap git repositories with partial clones. Use to speed up initial git clone for automated builds.', 961 default=False, 962 action='store_true') 963 parser.add_argument('--clone-timeout', 964 help='Timeout for repository cloning during bootstrap.', 965 default=GIT_TIMEOUT_SEC, 966 type=int) 967 args = parser.parse_args() 968 969 # Make sure we get absolute path + expanded path for bootstrap directory 970 args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir)) 971 972 # Possible values for machine() come from 'uname -m' 973 # Since this script only runs on Linux, x86_64 machines must have this value 974 if platform.machine() != 'x86_64': 975 raise Exception("Only x86_64 machines are currently supported by this build script.") 976 977 if args.run_bootstrap: 978 bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__), args.partial_staging, args.clone_timeout) 979 bootstrap.bootstrap() 980 elif args.print_env: 981 build = HostBuild(args) 982 build.print_env() 983 else: 984 build = HostBuild(args) 985 build.build() 986