1#!/usr/bin/env python 2# 3# Copyright (C) 2015 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17"""Makes an old style monolithic NDK package out of individual modules.""" 18from __future__ import print_function 19 20import argparse 21import datetime 22import os 23import shutil 24import site 25import stat 26import subprocess 27import sys 28import tempfile 29import zipfile 30 31 32site.addsitedir(os.path.join(os.path.dirname(__file__), '../lib')) 33import build_support # pylint: disable=import-error 34 35 36THIS_DIR = os.path.dirname(__file__) 37ANDROID_TOP = os.path.realpath(os.path.join(THIS_DIR, '../../..')) 38 39# Note that we can't actually support creating both layouts from the same 40# sources because changes are needed in gnustl's Android.mk and in the build 41# system. The various "USE_NEW_LAYOUT" blocks are so we can more easily undo 42# this change when we finalize a new layout. 43USE_NEW_LAYOUT = False 44 45 46def expand_packages(package, host, arches): 47 """Expands package definition tuple into list of full package names. 48 49 >>> expand_packages('gcc-{arch}-{host}', 'linux', ['arm64', 'x86_64']) 50 ['gcc-arm64-linux-x86_64', 'gcc-x86_64-linux-x86_64'] 51 52 >>> expand_packages('gcclibs-{arch}', 'linux', ['arm64', 'x86_64']) 53 ['gcclibs-arm64', 'gcclibs-x86_64'] 54 55 >>> expand_packages('llvm-{host}', 'linux', ['arm']) 56 ['llvm-linux-x86_64'] 57 58 >>> expand_packages('platforms', 'linux', ['arm']) 59 ['platforms'] 60 61 >>> expand_packages('libc++-{abi}', 'linux', ['arm']) 62 ['libc++-armeabi', 'libc++-armeabi-v7a', 'libc++-armeabi-v7a-hard'] 63 64 >>> expand_packages('binutils/{triple}', 'linux', ['arm', 'x86_64']) 65 ['binutils/arm-linux-androideabi', 'binutils/x86_64-linux-android'] 66 67 >> expand_packages('toolchains/{toolchain}-4.9', 'linux', ['arm', 'x86']) 68 ['toolchains/arm-linux-androideabi-4.9', 'toolchains/x86-4.9'] 69 """ 70 host_tag = build_support.host_to_tag(host) 71 seen_packages = set() 72 packages = [] 73 for arch in arches: 74 triple = build_support.arch_to_triple(arch) 75 toolchain = build_support.arch_to_toolchain(arch) 76 for abi in build_support.arch_to_abis(arch): 77 expanded = package.format( 78 abi=abi, arch=arch, host=host_tag, triple=triple, 79 toolchain=toolchain) 80 if expanded not in seen_packages: 81 packages.append(expanded) 82 seen_packages.add(expanded) 83 return packages 84 85 86def get_all_packages(host, arches): 87 new_layout = [ 88 ('binutils-{arch}-{host}', 'binutils/{triple}'), 89 ('build', 'build'), 90 ('cpufeatures', 'sources/android/cpufeatures'), 91 ('gabixx', 'sources/cxx-stl/gabi++'), 92 ('gcc-{arch}-{host}', 'toolchains/{toolchain}-4.9'), 93 ('gcclibs-{arch}', 'gcclibs/{triple}'), 94 ('gdbserver-{arch}', 'gdbserver/{arch}'), 95 ('gnustl-4.9', 'sources/cxx-stl/gnu-libstdc++'), 96 ('gtest', 'sources/third_party/googletest'), 97 ('host-tools-{host}', 'host-tools'), 98 ('libandroid_support', 'sources/android/support'), 99 ('libcxx', 'sources/cxx-stl/llvm-libc++'), 100 ('libcxxabi', 'sources/cxx-stl/llvm-libc++abi'), 101 ('llvm-{host}', 'toolchains/llvm'), 102 ('native_app_glue', 'sources/android/native_app_glue'), 103 ('ndk_helper', 'sources/android/ndk_helper'), 104 ('python-packages', 'python-packages'), 105 ('stlport', 'sources/cxx-stl/stlport'), 106 ('system-stl', 'sources/cxx-stl/system'), 107 ] 108 109 old_layout = [ 110 ('build', 'build'), 111 ('cpufeatures', 'sources/android/cpufeatures'), 112 ('gabixx', 'sources/cxx-stl/gabi++'), 113 ('gcc-{arch}-{host}', 'toolchains/{toolchain}-4.9/prebuilt/{host}'), 114 ('gdbserver-{arch}', 'prebuilt/android-{arch}/gdbserver'), 115 ('gnustl-4.9', 'sources/cxx-stl/gnu-libstdc++/4.9'), 116 ('gtest', 'sources/third_party/googletest'), 117 ('host-tools-{host}', 'prebuilt/{host}'), 118 ('libandroid_support', 'sources/android/support'), 119 ('libcxx', 'sources/cxx-stl/llvm-libc++'), 120 ('libcxxabi', 'sources/cxx-stl/llvm-libc++abi'), 121 ('llvm-{host}', 'toolchains/llvm/prebuilt/{host}'), 122 ('native_app_glue', 'sources/android/native_app_glue'), 123 ('ndk_helper', 'sources/android/ndk_helper'), 124 ('python-packages', 'python-packages'), 125 ('stlport', 'sources/cxx-stl/stlport'), 126 ('system-stl', 'sources/cxx-stl/system'), 127 ] 128 129 if USE_NEW_LAYOUT: 130 packages = new_layout 131 else: 132 packages = old_layout 133 134 platforms_path = 'development/ndk/platforms' 135 for platform_dir in os.listdir(build_support.android_path(platforms_path)): 136 if not platform_dir.startswith('android-'): 137 continue 138 _, platform_str = platform_dir.split('-') 139 package_name = 'platform-' + platform_str 140 install_path = 'platforms/android-' + platform_str 141 packages.append((package_name, install_path)) 142 143 expanded = [] 144 for package, extract_path in packages: 145 package_names = expand_packages(package, host, arches) 146 extract_paths = expand_packages(extract_path, host, arches) 147 expanded.extend(zip(package_names, extract_paths)) 148 return expanded 149 150 151def check_packages(path, packages): 152 for package, _ in packages: 153 print('Checking ' + package) 154 package_path = os.path.join(path, package + '.zip') 155 if not os.path.exists(package_path): 156 raise RuntimeError('Missing package: ' + package_path) 157 158 top_level_files = [] 159 with zipfile.ZipFile(package_path, 'r') as zip_file: 160 for f in zip_file.namelist(): 161 components = os.path.split(f) 162 if len(components) == 2: 163 top_level_files.append(components[1]) 164 165 if 'repo.prop' not in top_level_files: 166 msg = 'Package does not contain a repo.prop: ' + package_path 167 raise RuntimeError(msg) 168 169 if 'NOTICE' not in top_level_files: 170 msg = 'Package does not contain a NOTICE: ' + package_path 171 raise RuntimeError(msg) 172 173 174def extract_all(path, packages, out_dir): 175 os.makedirs(out_dir) 176 for package, extract_path in packages: 177 print('Unpacking ' + package) 178 package_path = os.path.join(path, package + '.zip') 179 install_dir = os.path.join(out_dir, extract_path) 180 181 if os.path.exists(install_dir): 182 raise RuntimeError('Install path already exists: ' + install_dir) 183 184 if extract_path == '.': 185 raise RuntimeError('Found old style package: ' + package) 186 187 extract_dir = tempfile.mkdtemp() 188 try: 189 subprocess.check_call( 190 ['unzip', '-q', package_path, '-d', extract_dir]) 191 dirs = os.listdir(extract_dir) 192 if len(dirs) > 1: 193 msg = 'Package has more than one root directory: ' + package 194 raise RuntimeError(msg) 195 elif len(dirs) == 0: 196 raise RuntimeError('Package was empty: ' + package) 197 parent_dir = os.path.dirname(install_dir) 198 if not os.path.exists(parent_dir): 199 os.makedirs(parent_dir) 200 shutil.move(os.path.join(extract_dir, dirs[0]), install_dir) 201 finally: 202 shutil.rmtree(extract_dir) 203 204 if not USE_NEW_LAYOUT: 205 # FIXME(danalbert): OMG HACK 206 # The old package layout had libstdc++'s Android.mk at 207 # sources/cxx-stl/gnu-libstdc++/Android.mk. The gnustl package doesn't 208 # include the version in the path. To mimic the old package layout, we 209 # extract the gnustl package to sources/cxx-stl/gnu-libstdc++/4.9. As 210 # such, the Android.mk ends up in the 4.9 directory. We need to pull it 211 # up a directory. 212 gnustl_path = os.path.join(out_dir, 'sources/cxx-stl/gnu-libstdc++') 213 shutil.move(os.path.join(gnustl_path, '4.9/Android.mk'), 214 os.path.join(gnustl_path, 'Android.mk')) 215 216 217def make_ndk_build_shortcut(out_dir, host): 218 if host.startswith('windows'): 219 make_ndk_build_cmd_helper(out_dir) 220 else: 221 make_ndk_build_sh_helper(out_dir) 222 223 224def make_ndk_build_cmd_helper(out_dir): 225 with open(os.path.join(out_dir, 'ndk-build.cmd'), 'w') as helper: 226 helper.writelines([ 227 '@echo off\n', 228 r'%~dp0\build\ndk-build.cmd %*', 229 ]) 230 231 232def make_ndk_build_sh_helper(out_dir): 233 file_path = os.path.join(out_dir, 'ndk-build') 234 with open(file_path, 'w') as helper: 235 helper.writelines([ 236 '#!/bin/sh\n', 237 'DIR="$(cd "$(dirname "$0")" && pwd)"\n', 238 '$DIR/build/ndk-build "$@"', 239 ]) 240 mode = os.stat(file_path).st_mode 241 os.chmod(file_path, mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) 242 243 244def make_source_properties(out_dir): 245 path = os.path.join(out_dir, 'source.properties') 246 with open(path, 'w') as source_properties: 247 source_properties.write('\n'.join([ 248 'Pkg.Desc = Android NDK', 249 'Pkg.Revision = 11.0.0', 250 ])) 251 252 253def copy_changelog(out_dir): 254 changelog_path = build_support.ndk_path('CHANGELOG.md') 255 shutil.copy2(changelog_path, out_dir) 256 257 258def make_package(release, package_dir, packages, host, out_dir, temp_dir): 259 release_name = 'android-ndk-{}'.format(release) 260 extract_dir = os.path.join(temp_dir, release_name) 261 if os.path.exists(extract_dir): 262 shutil.rmtree(extract_dir) 263 extract_all(package_dir, packages, extract_dir) 264 265 make_ndk_build_shortcut(extract_dir, host) 266 make_source_properties(extract_dir) 267 copy_changelog(extract_dir) 268 269 host_tag = build_support.host_to_tag(host) 270 package_name = '{}-{}'.format(release_name, host_tag) 271 package_path = os.path.join(out_dir, package_name) 272 print('Packaging ' + package_name) 273 files = os.path.relpath(extract_dir, temp_dir) 274 275 if host.startswith('windows'): 276 _make_zip_package(package_path, temp_dir, files) 277 else: 278 _make_tar_package(package_path, temp_dir, files) 279 280 281def _make_tar_package(package_path, base_dir, files): 282 subprocess.check_call( 283 ['tar', 'cjf', package_path + '.tar.bz2', '-C', base_dir, files]) 284 285 286def _make_zip_package(package_path, base_dir, files): 287 cwd = os.getcwd() 288 package_path = os.path.realpath(package_path) 289 os.chdir(base_dir) 290 try: 291 subprocess.check_call(['zip', '-9qr', package_path + '.zip', files]) 292 finally: 293 os.chdir(cwd) 294 295 296class ArgParser(argparse.ArgumentParser): 297 def __init__(self): 298 super(ArgParser, self).__init__( 299 description='Repackages NDK modules as a monolithic package. If ' 300 '--unpack is used, instead installs the monolithic ' 301 'package to a directory.') 302 303 self.add_argument( 304 '--arch', choices=build_support.ALL_ARCHITECTURES, 305 help='Bundle only the given architecture.') 306 self.add_argument( 307 '--host', choices=('darwin', 'linux', 'windows', 'windows64'), 308 default=build_support.get_default_host(), 309 help='Package binaries for given OS (e.g. linux).') 310 self.add_argument( 311 '--release', default=datetime.date.today().strftime('%Y%m%d'), 312 help='Release name for the package.') 313 314 self.add_argument( 315 '-f', '--force', dest='force', action='store_true', 316 help='Clobber out directory if it exists.') 317 self.add_argument( 318 '--dist-dir', type=os.path.realpath, 319 default=build_support.get_dist_dir(build_support.get_out_dir()), 320 help='Directory containing NDK modules.') 321 self.add_argument( 322 '--unpack', action='store_true', 323 help='Unpack the NDK to a directory rather than make a tarball.') 324 self.add_argument( 325 'out_dir', metavar='OUT_DIR', 326 help='Directory to install bundle or tarball to.') 327 328 329def main(): 330 if 'ANDROID_BUILD_TOP' not in os.environ: 331 os.environ['ANDROID_BUILD_TOP'] = ANDROID_TOP 332 333 args = ArgParser().parse_args() 334 arches = build_support.ALL_ARCHITECTURES 335 if args.arch is not None: 336 arches = [args.arch] 337 338 if os.path.exists(args.out_dir) and args.unpack: 339 if args.force: 340 shutil.rmtree(args.out_dir) 341 else: 342 sys.exit(args.out_dir + ' already exists. Use -f to overwrite.') 343 344 packages = get_all_packages(args.host, arches) 345 check_packages(args.dist_dir, packages) 346 347 if args.unpack: 348 extract_all(args.dist_dir, packages, args.out_dir) 349 make_ndk_build_shortcut(args.out_dir, args.host) 350 else: 351 make_package(args.release, args.dist_dir, packages, args.host, 352 args.out_dir, build_support.get_out_dir()) 353 354 355if __name__ == '__main__': 356 main() 357