1#!/usr/bin/env python3 2 3# Copyright 2018 The SwiftShader Authors. All Rights Reserved. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import argparse 18import contextlib 19import multiprocessing 20import os 21import platform 22import re 23import shutil 24import subprocess 25import sys 26import tempfile 27from os import path 28 29# LLVM_BRANCH must match the value of the same variable in third_party/update-llvm-10.sh 30LLVM_BRANCH = "release/10.x" 31 32# LLVM_COMMIT must be set to the commit hash that we last updated to when running third_party/update-llvm-10.sh. 33# Run 'git show -s origin/llvm10-clean' and look for 'llvm-10-update: <hash>' to retrieve it. 34LLVM_COMMIT = "d32170dbd5b0d54436537b6b75beaf44324e0c28" 35 36SCRIPT_DIR = path.dirname(path.realpath(sys.argv[0])) 37LLVM_STAGING_DIR = path.abspath(path.join(tempfile.gettempdir(), "llvm-10")) 38LLVM_DIR = path.abspath(path.join(LLVM_STAGING_DIR, "llvm")) 39LLVM_OBJS = path.join(LLVM_STAGING_DIR, "build") 40LLVM_CONFIGS = path.abspath(path.join(SCRIPT_DIR, '..', 'configs')) 41 42# List of all arches SwiftShader supports 43LLVM_TARGETS = [ 44 ('AArch64', ('__aarch64__',)), 45 ('ARM', ('__arm__',)), 46 ('X86', ('__i386__', '__x86_64__')), 47 ('Mips', ('__mips__',)), 48 ('PowerPC', ('__powerpc64__',)), 49 ('RISCV', ('__riscv',)), 50] 51 52# Per-platform arches 53LLVM_TRIPLES = { 54 'android': [ 55 ('__x86_64__', 'x86_64-linux-android'), 56 ('__i386__', 'i686-linux-android'), 57 ('__arm__', 'armv7-linux-androideabi'), 58 ('__aarch64__', 'aarch64-linux-android'), 59 ], 60 'linux': [ 61 ('__x86_64__', 'x86_64-unknown-linux-gnu'), 62 ('__i386__', 'i686-pc-linux-gnu'), 63 ('__arm__', 'armv7-linux-gnueabihf'), 64 ('__aarch64__', 'aarch64-linux-gnu'), 65 ('__mips__', 'mipsel-linux-gnu'), 66 ('__mips64', 'mips64el-linux-gnuabi64'), 67 ('__powerpc64__', 'powerpc64le-unknown-linux-gnu'), 68 ('__riscv', 'riscv64-unknown-linux-gnu'), 69 ], 70 'darwin': [ 71 ('__x86_64__', 'x86_64-apple-darwin'), 72 ('__aarch64__', 'arm64-apple-darwin'), 73 ], 74 'windows': [ 75 ('__x86_64__', 'x86_64-pc-win32'), 76 ('__i386__', 'i686-pc-win32'), 77 ('__arm__', 'armv7-pc-win32'), 78 ('__aarch64__', 'aarch64-pc-win32'), 79 ('__mips__', 'mipsel-pc-win32'), 80 ('__mips64', 'mips64el-pc-win32'), 81 ], 82 'fuchsia': [ 83 ('__x86_64__', 'x86_64-unknown-fuchsia'), 84 ('__aarch64__', 'aarch64-unknown-fuchsia'), 85 ] 86} 87 88# Mapping of target platform to the host it must be built on 89LLVM_PLATFORM_TO_HOST_SYSTEM = { 90 'android': 'Linux', 91 'darwin': 'Darwin', 92 'linux': 'Linux', 93 'windows': 'Windows', 94 'fuchsia': 'Linux' 95} 96 97# LLVM configurations to be undefined. 98LLVM_UNDEF_MACROS = [ 99 'BACKTRACE_HEADER', 100 'ENABLE_BACKTRACES', 101 'ENABLE_CRASH_OVERRIDES', 102 'HAVE_BACKTRACE', 103 'HAVE_POSIX_SPAWN', 104 'HAVE_PTHREAD_GETNAME_NP', 105 'HAVE_PTHREAD_SETNAME_NP', 106 'HAVE_TERMIOS_H', 107 'HAVE_ZLIB_H', 108 'HAVE__UNWIND_BACKTRACE', 109] 110 111# General CMake options for building LLVM 112LLVM_CMAKE_OPTIONS = [ 113 '-DCMAKE_BUILD_TYPE=Release', 114 '-DLLVM_ENABLE_THREADS=ON', 115 '-DLLVM_ENABLE_TERMINFO=OFF', 116 '-DLLVM_ENABLE_LIBXML2=OFF', 117 '-DLLVM_ENABLE_LIBEDIT=OFF', 118 '-DLLVM_ENABLE_LIBPFM=OFF', 119 '-DLLVM_ENABLE_ZLIB=OFF', 120 '-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON' 121] 122 123# Used when building LLVM for darwin. Affects values set in the generated config files. 124MIN_MACOS_VERSION = '10.10' 125 126@contextlib.contextmanager 127def pushd(new_dir): 128 previous_dir = os.getcwd() 129 os.chdir(new_dir) 130 try: 131 yield 132 finally: 133 os.chdir(previous_dir) 134 135 136def log(message, level=1): 137 print(' ' * level + '> ' + message) 138 139 140def run_command(command, log_level=1): 141 log(command, log_level) 142 os.system(command) 143 144 145def run_subprocess(*popenargs, log_level=1, cwd=None): 146 log([' '.join(t) for t in popenargs][0], log_level) 147 return subprocess.run(*popenargs, cwd=cwd) 148 149 150def _parse_args(): 151 parser = argparse.ArgumentParser() 152 parser.add_argument('name', help='destination name', 153 choices=['android', 'linux', 'darwin', 'windows', 'fuchsia']) 154 parser.add_argument('-j', '--jobs', help='parallel compilation', type=int) 155 return parser.parse_args() 156 157 158def validate_args(args): 159 host = platform.system() 160 if host not in LLVM_PLATFORM_TO_HOST_SYSTEM.values(): 161 raise Exception(f"Host system not supported: '{host}'") 162 163 if args.name not in LLVM_PLATFORM_TO_HOST_SYSTEM.keys(): 164 raise Exception(f"Unknown target platform: '{args.name}'") 165 166 expected_host = LLVM_PLATFORM_TO_HOST_SYSTEM[args.name] 167 if LLVM_PLATFORM_TO_HOST_SYSTEM[args.name] != host: 168 raise Exception( 169 f"Target platform '{args.name}' must be built on '{expected_host}', not on '{host}'") 170 171 172def get_cmake_targets_to_build(platform): 173 """Returns list of LLVM targets to build for the input platform""" 174 targets = set() 175 for arch_def, triplet in LLVM_TRIPLES[platform]: 176 for arch, defs in LLVM_TARGETS: 177 if arch_def in defs: 178 targets.add(arch) 179 180 # Maintain the sort order of LLVM_TARGETS as this affects how config 181 # headers are generated 182 return [t[0] for t in LLVM_TARGETS if t[0] in targets] 183 184 185def clone_llvm(): 186 log("Cloning/Updating LLVM", 1) 187 # Clone or update staging directory 188 if not path.exists(LLVM_STAGING_DIR): 189 os.mkdir(LLVM_STAGING_DIR) 190 with pushd(LLVM_STAGING_DIR): 191 run_command('git init', 2) 192 run_command( 193 'git remote add origin https://github.com/llvm/llvm-project.git', 2) 194 run_command('git config core.sparsecheckout true', 2) 195 run_command('echo /llvm > .git/info/sparse-checkout', 2) 196 197 with pushd(LLVM_STAGING_DIR): 198 run_command('echo /llvm > .git/info/sparse-checkout', 2) 199 run_command('git fetch origin {}'.format(LLVM_BRANCH), 2) 200 run_command('git checkout {}'.format(LLVM_COMMIT), 2) 201 return 202 203 204def build_llvm(name, num_jobs): 205 """Build LLVM and get all generated files.""" 206 log("Building LLVM", 1) 207 if num_jobs is None: 208 num_jobs = multiprocessing.cpu_count() 209 210 """On Windows we need to have CMake generate build files for the 64-bit 211 Visual Studio host toolchain.""" 212 host = '-Thost=x64' if name == 'windows' else '' 213 214 cmake_options = LLVM_CMAKE_OPTIONS + ['-DLLVM_TARGETS_TO_BUILD=' + 215 ';'.join(t for t in get_cmake_targets_to_build(name))] 216 217 if name == 'darwin': 218 cmake_options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET={}'.format(MIN_MACOS_VERSION)) 219 220 os.makedirs(LLVM_OBJS, exist_ok=True) 221 run_subprocess(['cmake', host, LLVM_DIR] + 222 cmake_options, log_level=2, cwd=LLVM_OBJS) 223 run_subprocess(['cmake', '--build', '.', '-j', 224 str(num_jobs)], log_level=2, cwd=LLVM_OBJS) 225 226 227def list_files(src_base, src, dst_base, suffixes): 228 """Enumerate the files that are under `src` and end with one of the 229 `suffixes` and yield the source path and the destination path.""" 230 src_base = path.abspath(src_base) 231 src = path.join(src_base, src) 232 for base_dir, dirnames, filenames in os.walk(src): 233 for filename in filenames: 234 if path.splitext(filename)[1] in suffixes: 235 relative = path.relpath(base_dir, src_base) 236 yield (path.join(base_dir, filename), 237 path.join(dst_base, relative, filename)) 238 239 240def copy_common_generated_files(dst_base): 241 """Copy platform-independent generated files.""" 242 log("Copying platform-independent generated files", 1) 243 suffixes = {'.inc', '.h', '.def'} 244 subdirs = [ 245 path.join('include', 'llvm', 'IR'), 246 path.join('include', 'llvm', 'Support'), 247 path.join('lib', 'IR'), 248 path.join('lib', 'Transforms', 'InstCombine'), 249 ] + [path.join('lib', 'Target', arch) for arch, defs in LLVM_TARGETS] 250 for subdir in subdirs: 251 for src, dst in list_files(LLVM_OBJS, subdir, dst_base, suffixes): 252 log('{} -> {}'.format(src, dst), 2) 253 os.makedirs(path.dirname(dst), exist_ok=True) 254 shutil.copyfile(src, dst) 255 256 257def copy_platform_file(platform, src, dst): 258 """Copy platform-dependent generated files and add platform-specific 259 modifications.""" 260 261 # LLVM configuration patterns to be post-processed. 262 llvm_target_pattern = re.compile('^LLVM_[A-Z_]+\\(([A-Za-z0-9_]+)\\)$') 263 llvm_native_pattern = re.compile( 264 '^#define LLVM_NATIVE_([A-Z]+) (LLVMInitialize)?(.*)$') 265 llvm_triple_pattern = re.compile('^#define (LLVM_[A-Z_]+_TRIPLE) "(.*)"$') 266 llvm_define_pattern = re.compile('^#define ([A-Za-z0-9_]+) (.*)$') 267 268 # Build architecture-specific conditions. 269 conds = {} 270 for arch, defs in LLVM_TARGETS: 271 conds[arch] = ' || '.join('defined(' + v + ')' for v in defs) 272 273 # Get a set of platform-specific triples. 274 triples = LLVM_TRIPLES[platform] 275 276 with open(src, 'r') as src_file: 277 os.makedirs(path.dirname(dst), exist_ok=True) 278 with open(dst, 'w') as dst_file: 279 for line in src_file: 280 if line == '#define LLVM_CONFIG_H\n': 281 print(line, file=dst_file, end='') 282 print('', file=dst_file) 283 print('#if !defined(__i386__) && defined(_M_IX86)', 284 file=dst_file) 285 print('#define __i386__ 1', file=dst_file) 286 print('#endif', file=dst_file) 287 print('', file=dst_file) 288 print( 289 '#if !defined(__x86_64__) && (defined(_M_AMD64) || defined (_M_X64))', file=dst_file) 290 print('#define __x86_64__ 1', file=dst_file) 291 print('#endif', file=dst_file) 292 print('', file=dst_file) 293 294 match = llvm_target_pattern.match(line) 295 if match: 296 arch = match.group(1) 297 print('#if ' + conds[arch], file=dst_file) 298 print(line, file=dst_file, end='') 299 print('#endif', file=dst_file) 300 continue 301 302 match = llvm_native_pattern.match(line) 303 if match: 304 name = match.group(1) 305 init = match.group(2) or '' 306 arch = match.group(3) 307 end = '' 308 if arch.lower().endswith(name.lower()): 309 end = arch[-len(name):] 310 directive = '#if ' 311 for arch, defs in LLVM_TARGETS: 312 print(directive + conds[arch], file=dst_file) 313 print('#define LLVM_NATIVE_' + name + ' ' + 314 init + arch + end, file=dst_file) 315 directive = '#elif ' 316 print('#else', file=dst_file) 317 print('#error "unknown architecture"', file=dst_file) 318 print('#endif', file=dst_file) 319 continue 320 321 match = llvm_triple_pattern.match(line) 322 if match: 323 name = match.group(1) 324 directive = '#if' 325 for defs, triple in triples: 326 print(directive + ' defined(' + defs + ')', 327 file=dst_file) 328 print('#define ' + name + ' "' + triple + '"', 329 file=dst_file) 330 directive = '#elif' 331 print('#else', file=dst_file) 332 print('#error "unknown architecture"', file=dst_file) 333 print('#endif', file=dst_file) 334 continue 335 336 match = llvm_define_pattern.match(line) 337 if match and match.group(1) in LLVM_UNDEF_MACROS: 338 print('/* #undef ' + match.group(1) + ' */', file=dst_file) 339 continue 340 341 print(line, file=dst_file, end='') 342 343 344def copy_platform_generated_files(platform, dst_base): 345 """Copy platform-specific generated files.""" 346 log("Copying platform-specific generated files", 1) 347 suffixes = {'.inc', '.h', '.def'} 348 src_dir = path.join('include', 'llvm', 'Config') 349 for src, dst in list_files(LLVM_OBJS, src_dir, dst_base, suffixes): 350 log('{}, {} -> {}'.format(platform, src, dst), 2) 351 copy_platform_file(platform, src, dst) 352 353 354def main(): 355 args = _parse_args() 356 validate_args(args) 357 clone_llvm() 358 build_llvm(args.name, args.jobs) 359 copy_common_generated_files(path.join(LLVM_CONFIGS, 'common')) 360 copy_platform_generated_files( 361 args.name, path.join(LLVM_CONFIGS, args.name)) 362 363 364if __name__ == '__main__': 365 main() 366