1#!/usr/bin/env python 2# 3# Copyright (C) 2015 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17"""Verifies that the build is sane. 18 19Cleans old build artifacts, configures the required environment, determines 20build goals, and invokes the build scripts. 21""" 22from __future__ import print_function 23 24import argparse 25import collections 26import datetime 27import inspect 28import os 29import shutil 30import site 31import subprocess 32import sys 33import tempfile 34import textwrap 35 36site.addsitedir(os.path.join(os.path.dirname(__file__), 'build/lib')) 37 38import build_support # pylint: disable=import-error 39 40 41ALL_MODULES = { 42 'binutils', 43 'build', 44 'clang', 45 'cpufeatures', 46 'gabi++', 47 'gcc', 48 'gcclibs', 49 'gdbserver', 50 'gnustl', 51 'gtest', 52 'host-tools', 53 'libandroid_support', 54 'libc++', 55 'libc++abi', 56 'native_app_glue', 57 'ndk_helper', 58 'platforms', 59 'python-packages', 60 'stlport', 61 'system-stl', 62} 63 64 65class ArgParser(argparse.ArgumentParser): 66 def __init__(self): 67 super(ArgParser, self).__init__( 68 description=inspect.getdoc(sys.modules[__name__])) 69 70 self.add_argument( 71 '--arch', 72 choices=('arm', 'arm64', 'mips', 'mips64', 'x86', 'x86_64'), 73 help='Build for the given architecture. Build all by default.') 74 75 package_group = self.add_mutually_exclusive_group() 76 package_group.add_argument( 77 '--package', action='store_true', dest='package', default=True, 78 help='Package the NDK when done building (default).') 79 package_group.add_argument( 80 '--no-package', action='store_false', dest='package', 81 help='Do not package the NDK when done building.') 82 83 test_group = self.add_mutually_exclusive_group() 84 test_group.add_argument( 85 '--test', action='store_true', dest='test', default=True, 86 help=textwrap.dedent("""\ 87 Run host tests when finished. --package is required. Not supported 88 when targeting Windows. 89 """)) 90 test_group.add_argument( 91 '--no-test', action='store_false', dest='test', 92 help='Do not run host tests when finished.') 93 94 self.add_argument( 95 '--release', 96 help='Release name. Package will be named android-ndk-RELEASE.') 97 98 self.add_argument( 99 '--system', choices=('darwin', 'linux', 'windows', 'windows64'), 100 default=build_support.get_default_host(), 101 help='Build for the given OS.') 102 103 module_group = self.add_mutually_exclusive_group() 104 105 module_group.add_argument( 106 '--module', choices=sorted(ALL_MODULES), 107 help='NDK modules to build.') 108 109 module_group.add_argument( 110 '--host-only', action='store_true', 111 help='Skip building target components.') 112 113 114def _invoke_build(script, args): 115 if args is None: 116 args = [] 117 subprocess.check_call([build_support.android_path(script)] + args) 118 119 120def invoke_build(script, args=None): 121 script_path = os.path.join('build/tools', script) 122 _invoke_build(build_support.ndk_path(script_path), args) 123 124 125def invoke_external_build(script, args=None): 126 _invoke_build(build_support.android_path(script), args) 127 128 129def package_ndk(out_dir, dist_dir, args): 130 package_args = common_build_args(out_dir, dist_dir, args) 131 package_args.append(dist_dir) 132 133 if args.release is not None: 134 package_args.append('--release={}'.format(args.release)) 135 136 if args.arch is not None: 137 package_args.append('--arch={}'.format(args.arch)) 138 139 invoke_build('package.py', package_args) 140 141 142def test_ndk(out_dir, args): 143 release = args.release 144 if args.release is None: 145 release = datetime.date.today().strftime('%Y%m%d') 146 147 # The packaging step extracts all the modules to a known directory for 148 # packaging. This directory is not cleaned up after packaging, so we can 149 # reuse that for testing. 150 test_dir = os.path.join(out_dir, 'android-ndk-{}'.format(release)) 151 152 test_env = dict(os.environ) 153 test_env['NDK'] = test_dir 154 155 abis = build_support.ALL_ABIS 156 if args.arch is not None: 157 abis = build_support.arch_to_abis(args.arch) 158 159 results = {} 160 for abi in abis: 161 cmd = [ 162 'python', build_support.ndk_path('tests/run-all.py'), 163 '--abi', abi, '--suite', 'build' 164 ] 165 print('Running tests: {}'.format(' '.join(cmd))) 166 result = subprocess.call(cmd, env=test_env) 167 results[abi] = result == 0 168 169 print('Results:') 170 for abi, result in results.iteritems(): 171 print('{}: {}'.format(abi, 'PASS' if result else 'FAIL')) 172 return all(results.values()) 173 174 175def common_build_args(out_dir, dist_dir, args): 176 build_args = ['--out-dir={}'.format(out_dir)] 177 build_args = ['--dist-dir={}'.format(dist_dir)] 178 build_args.append('--host={}'.format(args.system)) 179 return build_args 180 181 182def fixup_toolchain_triple(toolchain): 183 """Maps toolchain names to their proper triple. 184 185 The x86 toolchains are named stupidly and aren't a proper triple. 186 """ 187 return { 188 'x86': 'i686-linux-android', 189 'x86_64': 'x86_64-linux-android', 190 }.get(toolchain, toolchain) 191 192 193def get_binutils_files(triple, has_gold, is_windows): 194 files = [ 195 'ld.bfd', 196 'nm', 197 'as', 198 'objcopy', 199 'strip', 200 'objdump', 201 'ld', 202 'ar', 203 'ranlib', 204 ] 205 206 if has_gold: 207 files.append('ld.gold') 208 209 if is_windows: 210 files = [f + '.exe' for f in files] 211 212 # binutils programs get installed to two locations: 213 # 1: $INSTALL_DIR/bin/$TRIPLE-$PROGRAM 214 # 2: $INSTALL_DIR/$TRIPLE/bin/$PROGRAM 215 # 216 # We need to copy both. 217 218 prefixed_files = [] 219 for file_name in files: 220 prefixed_name = '-'.join([triple, file_name]) 221 prefixed_files.append(os.path.join('bin', prefixed_name)) 222 223 dir_prefixed_files = [] 224 for file_name in files: 225 dir_prefixed_files.append(os.path.join(triple, 'bin', file_name)) 226 227 ldscripts_dir = os.path.join(triple, 'lib/ldscripts') 228 return prefixed_files + dir_prefixed_files + [ldscripts_dir] 229 230 231def install_file(file_name, src_dir, dst_dir): 232 src_file = os.path.join(src_dir, file_name) 233 dst_file = os.path.join(dst_dir, file_name) 234 235 print('Copying {} to {}...'.format(src_file, dst_file)) 236 if os.path.isdir(src_file): 237 _install_dir(src_file, dst_file) 238 elif os.path.islink(src_file): 239 _install_symlink(src_file, dst_file) 240 else: 241 _install_file(src_file, dst_file) 242 243 244def _install_dir(src_dir, dst_dir): 245 parent_dir = os.path.normpath(os.path.join(dst_dir, '..')) 246 if not os.path.exists(parent_dir): 247 os.makedirs(parent_dir) 248 shutil.copytree(src_dir, dst_dir, symlinks=True) 249 250 251def _install_symlink(src_file, dst_file): 252 dirname = os.path.dirname(dst_file) 253 if not os.path.exists(dirname): 254 os.makedirs(dirname) 255 link_target = os.readlink(src_file) 256 os.symlink(link_target, dst_file) 257 258 259def _install_file(src_file, dst_file): 260 dirname = os.path.dirname(dst_file) 261 if not os.path.exists(dirname): 262 os.makedirs(dirname) 263 # copy2 is just copy followed by copystat (preserves file metadata). 264 shutil.copy2(src_file, dst_file) 265 266 267def pack_binutils(arch, host_tag, out_dir, binutils_path): 268 archive_name = '-'.join(['binutils', arch, host_tag]) 269 build_support.make_package(archive_name, binutils_path, out_dir) 270 271 272def get_prebuilt_gcc(host, arch): 273 tag = build_support.host_to_tag(host) 274 system_subdir = 'prebuilts/ndk/current/toolchains/{}'.format(tag) 275 system_path = build_support.android_path(system_subdir) 276 toolchain = build_support.arch_to_toolchain(arch) 277 toolchain_dir = toolchain + '-4.9' 278 return os.path.join(system_path, toolchain_dir) 279 280 281def build_binutils(out_dir, dist_dir, args): 282 print('Extracting binutils package from GCC...') 283 284 arches = build_support.ALL_ARCHITECTURES 285 if args.arch is not None: 286 arches = [args.arch] 287 288 host_tag = build_support.host_to_tag(args.system) 289 290 for arch in arches: 291 toolchain = build_support.arch_to_toolchain(arch) 292 toolchain_path = get_prebuilt_gcc(args.system, arch) 293 294 triple = fixup_toolchain_triple(toolchain) 295 296 install_dir = os.path.join(out_dir, 'binutils', triple) 297 if os.path.exists(install_dir): 298 shutil.rmtree(install_dir) 299 os.makedirs(install_dir) 300 301 has_gold = True 302 if host_tag == 'windows': 303 # Note: 64-bit Windows is fine. 304 has_gold = False 305 if arch in ('mips', 'mips64'): 306 has_gold = False 307 308 is_windows = host_tag.startswith('windows') 309 for file_name in get_binutils_files(triple, has_gold, is_windows): 310 install_file(file_name, toolchain_path, install_dir) 311 312 license_path = build_support.android_path( 313 'toolchain/binutils/binutils-2.25/COPYING') 314 shutil.copy2(license_path, os.path.join(install_dir, 'NOTICE')) 315 316 pack_binutils(arch, host_tag, dist_dir, install_dir) 317 318 319def build_clang(out_dir, dist_dir, args): 320 print('Building Clang...') 321 invoke_build('build-llvm.py', common_build_args(out_dir, dist_dir, args)) 322 323 324def build_gcc(out_dir, dist_dir, args): 325 print('Building GCC...') 326 build_args = common_build_args(out_dir, dist_dir, args) 327 if args.arch is not None: 328 build_args.append('--arch={}'.format(args.arch)) 329 invoke_build('build-gcc.py', build_args) 330 331 332def build_gcc_libs(out_dir, dist_dir, args): 333 print('Packaging GCC libs...') 334 335 arches = build_support.ALL_ARCHITECTURES 336 if args.arch is not None: 337 arches = [args.arch] 338 339 for arch in arches: 340 toolchain = build_support.arch_to_toolchain(arch) 341 triple = fixup_toolchain_triple(toolchain) 342 libgcc_subdir = 'lib/gcc/{}/4.9'.format(triple) 343 is64 = arch.endswith('64') 344 libatomic_subdir = '{}/lib{}'.format(triple, '64' if is64 else '') 345 346 lib_names = [ 347 (libatomic_subdir, 'libatomic.a'), 348 (libgcc_subdir, 'libgcc.a'), 349 ] 350 351 lib_dirs = [''] 352 if arch == 'arm': 353 lib_dirs += [ 354 'armv7-a', 355 'armv7-a/hard', 356 'armv7-a/thumb', 357 'armv7-a/thumb/hard', 358 'thumb', 359 ] 360 361 libs = [] 362 for lib_dir in lib_dirs: 363 for subdir, lib in lib_names: 364 libs.append((subdir, os.path.join(lib_dir, lib))) 365 366 install_dir = os.path.join(out_dir, 'gcclibs', triple) 367 if os.path.exists(install_dir): 368 shutil.rmtree(install_dir) 369 os.makedirs(install_dir) 370 371 # These are target libraries, so the OS we use here is not 372 # important. We explicitly use Linux because for whatever reason 373 # the Windows aarch64 toolchain doesn't include libatomic. 374 gcc_path = get_prebuilt_gcc('linux', arch) 375 for gcc_subdir, lib in libs: 376 src = os.path.join(gcc_path, gcc_subdir, lib) 377 dst = os.path.join(install_dir, lib) 378 dst_dir = os.path.dirname(dst) 379 if not os.path.exists(dst_dir): 380 os.makedirs(dst_dir) 381 shutil.copy2(src, dst) 382 383 shutil.copy2( 384 os.path.join(gcc_path, 'NOTICE'), 385 os.path.join(install_dir, 'NOTICE')) 386 387 archive_name = os.path.join('gcclibs-' + arch) 388 build_support.make_package(archive_name, install_dir, dist_dir) 389 390 391def build_host_tools(out_dir, dist_dir, args): 392 build_args = common_build_args(out_dir, dist_dir, args) 393 394 print('Building ndk-stack...') 395 invoke_external_build( 396 'ndk/sources/host-tools/ndk-stack/build.py', build_args) 397 398 print('Building ndk-depends...') 399 invoke_external_build( 400 'ndk/sources/host-tools/ndk-depends/build.py', build_args) 401 402 print('Building awk...') 403 invoke_external_build( 404 'ndk/sources/host-tools/nawk-20071023/build.py', build_args) 405 406 print('Building make...') 407 invoke_external_build( 408 'ndk/sources/host-tools/make-3.81/build.py', build_args) 409 410 if args.system in ('windows', 'windows64'): 411 print('Building toolbox...') 412 invoke_external_build( 413 'ndk/sources/host-tools/toolbox/build.py', build_args) 414 415 print('Building Python...') 416 invoke_external_build('toolchain/python/build.py', build_args) 417 418 print('Building GDB...') 419 invoke_external_build('toolchain/gdb/build.py', build_args) 420 421 print('Building YASM...') 422 invoke_external_build('toolchain/yasm/build.py', build_args) 423 424 package_host_tools(out_dir, dist_dir, args.system) 425 426 427def merge_license_files(output_path, files): 428 licenses = [] 429 for license_path in files: 430 with open(license_path) as license_file: 431 licenses.append(license_file.read()) 432 433 with open(output_path, 'w') as output_file: 434 output_file.write('\n'.join(licenses)) 435 436 437def package_host_tools(out_dir, dist_dir, host): 438 packages = [ 439 'gdb-multiarch-7.10', 440 'ndk-awk', 441 'ndk-depends', 442 'ndk-make', 443 'ndk-python', 444 'ndk-stack', 445 'ndk-yasm', 446 ] 447 448 files = [ 449 'ndk-gdb', 450 'ndk-gdb.cmd', 451 'ndk-gdb.py', 452 ] 453 454 if host in ('windows', 'windows64'): 455 packages.append('toolbox') 456 457 host_tag = build_support.host_to_tag(host) 458 459 package_names = [p + '-' + host_tag + '.tar.bz2' for p in packages] 460 for package_name in package_names: 461 package_path = os.path.join(out_dir, package_name) 462 subprocess.check_call(['tar', 'xf', package_path, '-C', out_dir]) 463 464 for f in files: 465 shutil.copy2(f, os.path.join(out_dir, 'host-tools/bin')) 466 467 merge_license_files(os.path.join(out_dir, 'host-tools/NOTICE'), [ 468 build_support.android_path('toolchain/gdb/gdb-7.10/COPYING'), 469 build_support.ndk_path('sources/host-tools/nawk-20071023/NOTICE'), 470 build_support.ndk_path('sources/host-tools/ndk-depends/NOTICE'), 471 build_support.ndk_path('sources/host-tools/make-3.81/COPYING'), 472 build_support.android_path( 473 'toolchain/python/Python-2.7.5/LICENSE'), 474 build_support.ndk_path('sources/host-tools/ndk-stack/NOTICE'), 475 build_support.ndk_path('sources/host-tools/toolbox/NOTICE'), 476 build_support.android_path('toolchain/yasm/COPYING'), 477 build_support.android_path('toolchain/yasm/BSD.txt'), 478 build_support.android_path('toolchain/yasm/Artistic.txt'), 479 build_support.android_path('toolchain/yasm/GNU_GPL-2.0'), 480 build_support.android_path('toolchain/yasm/GNU_LGPL-2.0'), 481 ]) 482 483 package_name = 'host-tools-' + host_tag 484 path = os.path.join(out_dir, 'host-tools') 485 build_support.make_package(package_name, path, dist_dir) 486 487 488def build_gdbserver(out_dir, dist_dir, args): 489 print('Building gdbserver...') 490 build_args = common_build_args(out_dir, dist_dir, args) 491 if args.arch is not None: 492 build_args.append('--arch={}'.format(args.arch)) 493 invoke_build('build-gdbserver.py', build_args) 494 495 496def _build_stl(out_dir, dist_dir, args, stl): 497 build_args = common_build_args(out_dir, dist_dir, args) 498 if args.arch is not None: 499 build_args.append('--arch={}'.format(args.arch)) 500 script = 'ndk/sources/cxx-stl/{}/build.py'.format(stl) 501 invoke_external_build(script, build_args) 502 503 504def build_gnustl(out_dir, dist_dir, args): 505 print('Building gnustl...') 506 _build_stl(out_dir, dist_dir, args, 'gnu-libstdc++') 507 508 509def build_libcxx(out_dir, dist_dir, args): 510 print('Building libc++...') 511 _build_stl(out_dir, dist_dir, args, 'llvm-libc++') 512 513 514def build_stlport(out_dir, dist_dir, args): 515 print('Building stlport...') 516 _build_stl(out_dir, dist_dir, args, 'stlport') 517 518 519def build_platforms(out_dir, dist_dir, args): 520 print('Building platforms...') 521 build_args = common_build_args(out_dir, dist_dir, args) 522 invoke_build('build-platforms.py', build_args) 523 524 525def build_cpufeatures(_, dist_dir, __): 526 path = build_support.ndk_path('sources/android/cpufeatures') 527 build_support.make_package('cpufeatures', path, dist_dir) 528 529 530def build_native_app_glue(_, dist_dir, __): 531 path = build_support.android_path( 532 'development/ndk/sources/android/native_app_glue') 533 build_support.make_package('native_app_glue', path, dist_dir) 534 535 536def build_ndk_helper(_, dist_dir, __): 537 path = build_support.android_path( 538 'development/ndk/sources/android/ndk_helper') 539 build_support.make_package('ndk_helper', path, dist_dir) 540 541 542def build_gtest(_, dist_dir, __): 543 path = build_support.ndk_path('sources/third_party/googletest') 544 build_support.make_package('gtest', path, dist_dir) 545 546 547def build_build(_, dist_dir, __): 548 path = build_support.ndk_path('build') 549 build_support.make_package('build', path, dist_dir) 550 551 552def build_python_packages(_, dist_dir, __): 553 # Stage the files in a temporary directory to make things easier. 554 temp_dir = tempfile.mkdtemp() 555 try: 556 path = os.path.join(temp_dir, 'python-packages') 557 shutil.copytree( 558 build_support.android_path('development/python-packages'), path) 559 build_support.make_package('python-packages', path, dist_dir) 560 finally: 561 shutil.rmtree(temp_dir) 562 563 564def build_gabixx(_out_dir, dist_dir, _args): 565 print('Building gabi++...') 566 path = build_support.ndk_path('sources/cxx-stl/gabi++') 567 build_support.make_package('gabixx', path, dist_dir) 568 569 570def build_system_stl(_out_dir, dist_dir, _args): 571 print('Building system-stl...') 572 path = build_support.ndk_path('sources/cxx-stl/system') 573 build_support.make_package('system-stl', path, dist_dir) 574 575 576def build_libandroid_support(_out_dir, dist_dir, _args): 577 print('Building libandroid_support...') 578 path = build_support.ndk_path('sources/android/support') 579 build_support.make_package('libandroid_support', path, dist_dir) 580 581 582def build_libcxxabi(_out_dir, dist_dir, _args): 583 print('Building libc++abi...') 584 path = build_support.ndk_path('sources/cxx-stl/llvm-libc++abi') 585 build_support.make_package('libcxxabi', path, dist_dir) 586 587 588def main(): 589 parser = ArgParser() 590 args = parser.parse_args() 591 592 if args.module is None: 593 modules = ALL_MODULES 594 else: 595 modules = {args.module} 596 597 if args.host_only: 598 modules = { 599 'clang', 600 'gcc', 601 'host-tools', 602 } 603 604 required_package_modules = ALL_MODULES 605 if args.package and required_package_modules <= modules: 606 do_package = True 607 else: 608 do_package = False 609 610 # TODO(danalbert): wine? 611 # We're building the Windows packages from Linux, so we can't actually run 612 # any of the tests from here. 613 if args.system.startswith('windows') or not do_package: 614 args.test = False 615 616 # Disable buffering on stdout so the build output doesn't hide all of our 617 # "Building..." messages. 618 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) 619 620 os.chdir(os.path.dirname(os.path.realpath(__file__))) 621 622 # Set ANDROID_BUILD_TOP. 623 if 'ANDROID_BUILD_TOP' not in os.environ: 624 os.environ['ANDROID_BUILD_TOP'] = os.path.realpath('..') 625 626 out_dir = build_support.get_out_dir() 627 dist_dir = build_support.get_dist_dir(out_dir) 628 629 print('Cleaning up...') 630 invoke_build('dev-cleanup.sh') 631 632 module_builds = collections.OrderedDict([ 633 ('binutils', build_binutils), 634 ('build', build_build), 635 ('clang', build_clang), 636 ('cpufeatures', build_cpufeatures), 637 ('gabi++', build_gabixx), 638 ('gcc', build_gcc), 639 ('gcclibs', build_gcc_libs), 640 ('gdbserver', build_gdbserver), 641 ('gnustl', build_gnustl), 642 ('gtest', build_gtest), 643 ('host-tools', build_host_tools), 644 ('libandroid_support', build_libandroid_support), 645 ('libc++', build_libcxx), 646 ('libc++abi', build_libcxxabi), 647 ('native_app_glue', build_native_app_glue), 648 ('ndk_helper', build_ndk_helper), 649 ('platforms', build_platforms), 650 ('python-packages', build_python_packages), 651 ('stlport', build_stlport), 652 ('system-stl', build_system_stl), 653 ]) 654 655 print('Building modules: {}'.format(' '.join(modules))) 656 for module in modules: 657 module_builds[module](out_dir, dist_dir, args) 658 659 if do_package: 660 package_ndk(out_dir, dist_dir, args) 661 662 if args.test: 663 result = test_ndk(out_dir, args) 664 sys.exit(0 if result else 1) 665 666 667if __name__ == '__main__': 668 main() 669