1# Copyright 2018 The Chromium Authors. All rights reserved. 2# Use of this source code is governed under the Apache License, Version 2.0 3# that can be found in the LICENSE file. 4"""Recipe for building GN.""" 5 6from recipe_engine.recipe_api import Property 7 8PYTHON_VERSION_COMPATIBILITY = 'PY3' 9 10DEPS = [ 11 'recipe_engine/buildbucket', 12 'recipe_engine/cas', 13 'recipe_engine/cipd', 14 'recipe_engine/context', 15 'recipe_engine/file', 16 'recipe_engine/json', 17 'recipe_engine/path', 18 'recipe_engine/platform', 19 'recipe_engine/properties', 20 'recipe_engine/python', 21 'recipe_engine/raw_io', 22 'recipe_engine/step', 23 'target', 24 'macos_sdk', 25 'windows_sdk', 26] 27 28PROPERTIES = { 29 'repository': Property(kind=str, default='https://gn.googlesource.com/gn'), 30} 31 32# On select platforms, link the GN executable against rpmalloc for a small 10% speed boost. 33RPMALLOC_GIT_URL = 'https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc' 34RPMALLOC_BRANCH = '+upstream/develop' 35RPMALLOC_REVISION = 'b097fd0916500439721a114bb9cd8d14bd998683' 36 37# Used to convert os and arch strings to rpmalloc format 38RPMALLOC_MAP = { 39 'amd64': 'x86-64', 40 'mac': 'macos', 41} 42 43 44def _get_libcxx_include_path(api): 45 # Run the preprocessor with an empty input and print all include paths. 46 lines = api.step( 47 'xcrun toolchain', [ 48 'xcrun', '--toolchain', 'clang', 'clang++', '-xc++', '-fsyntax-only', 49 '-Wp,-v', '/dev/null' 50 ], 51 stderr=api.raw_io.output_text(name='toolchain', add_output_log=True), 52 step_test_data=lambda: api.raw_io.test_api.stream_output_text( 53 str(api.macos_sdk.sdk_dir.join('include', 'c++', 'v1')), 54 stream='stderr')).stderr.splitlines() 55 # Iterate over all include paths and look for the SDK libc++ one. 56 sdk_dir = str(api.macos_sdk.sdk_dir) 57 for line in lines: 58 line = line.strip() 59 if line.startswith(sdk_dir) and 'include/c++/v1' in line: 60 return line 61 return None # pragma: no cover 62 63 64def _get_compilation_environment(api, target, cipd_dir): 65 if target.is_linux: 66 triple = '--target=%s' % target.triple 67 sysroot = '--sysroot=%s' % cipd_dir.join('sysroot') 68 env = { 69 'CC': cipd_dir.join('bin', 'clang'), 70 'CXX': cipd_dir.join('bin', 'clang++'), 71 'AR': cipd_dir.join('bin', 'llvm-ar'), 72 'CFLAGS': '%s %s' % (triple, sysroot), 73 'LDFLAGS': '%s %s -static-libstdc++' % (triple, sysroot), 74 } 75 elif target.is_mac: 76 triple = '--target=%s' % target.triple 77 sysroot = '--sysroot=%s' % api.step( 78 'xcrun sdk-path', ['xcrun', '--show-sdk-path'], 79 stdout=api.raw_io.output_text(name='sdk-path', add_output_log=True), 80 step_test_data=lambda: api.raw_io.test_api.stream_output_text( 81 '/some/xcode/path')).stdout.strip() 82 stdlib = cipd_dir.join('lib', 'libc++.a') 83 cxx_include = _get_libcxx_include_path(api) 84 env = { 85 'CC': 86 cipd_dir.join('bin', 'clang'), 87 'CXX': 88 cipd_dir.join('bin', 'clang++'), 89 'AR': 90 cipd_dir.join('bin', 'llvm-ar'), 91 'CFLAGS': 92 '%s %s -nostdinc++ -cxx-isystem %s' % 93 (triple, sysroot, cxx_include), 94 # TODO(phosek): Use the system libc++ temporarily until we 95 # have universal libc++.a for macOS that supports both x86_64 96 # and arm64. 97 'LDFLAGS': 98 '%s %s' % (triple, sysroot), 99 } 100 else: 101 env = {} 102 103 return env 104 105 106def RunSteps(api, repository): 107 src_dir = api.path['start_dir'].join('gn') 108 109 # TODO: Verify that building and linking rpmalloc works on OS X and Windows as 110 # well. 111 with api.step.nest('git'), api.context(infra_steps=True): 112 api.step('init', ['git', 'init', src_dir]) 113 114 with api.context(cwd=src_dir): 115 build_input = api.buildbucket.build_input 116 ref = ( 117 build_input.gitiles_commit.id 118 if build_input.gitiles_commit else 'refs/heads/master') 119 # Fetch tags so `git describe` works. 120 api.step('fetch', ['git', 'fetch', '--tags', repository, ref]) 121 api.step('checkout', ['git', 'checkout', 'FETCH_HEAD']) 122 revision = api.step( 123 'rev-parse', ['git', 'rev-parse', 'HEAD'], 124 stdout=api.raw_io.output_text()).stdout.strip() 125 for change in build_input.gerrit_changes: 126 api.step('fetch %s/%s' % (change.change, change.patchset), [ 127 'git', 'fetch', repository, 128 'refs/changes/%s/%s/%s' % 129 (str(change.change)[-2:], change.change, change.patchset) 130 ]) 131 api.step('checkout %s/%s' % (change.change, change.patchset), 132 ['git', 'checkout', 'FETCH_HEAD']) 133 134 with api.context(infra_steps=True): 135 cipd_dir = api.path['start_dir'].join('cipd') 136 pkgs = api.cipd.EnsureFile() 137 pkgs.add_package('infra/ninja/${platform}', 'version:1.8.2') 138 if api.platform.is_linux or api.platform.is_mac: 139 pkgs.add_package('fuchsia/third_party/clang/${platform}', 'integration') 140 if api.platform.is_linux: 141 pkgs.add_package('fuchsia/third_party/sysroot/linux', 142 'git_revision:c912d089c3d46d8982fdef76a50514cca79b6132', 143 'sysroot') 144 api.cipd.ensure(cipd_dir, pkgs) 145 146 def release_targets(): 147 if api.platform.is_linux: 148 return [api.target('linux-amd64'), api.target('linux-arm64')] 149 elif api.platform.is_mac: 150 return [api.target('mac-amd64'), api.target('mac-arm64')] 151 else: 152 return [api.target.host] 153 154 # The order is important since release build will get uploaded to CIPD. 155 configs = [ 156 { 157 'name': 'debug', 158 'args': ['-d'], 159 'targets': [api.target.host], 160 }, 161 { 162 'name': 'release', 163 'args': ['--use-lto', '--use-icf'], 164 'targets': release_targets(), 165 # TODO: Enable this for OS X and Windows. 166 'use_rpmalloc': api.platform.is_linux, 167 }, 168 ] 169 170 # True if any config uses rpmalloc. 171 use_rpmalloc = any(c.get('use_rpmalloc', False) for c in configs) 172 173 with api.macos_sdk(), api.windows_sdk(): 174 # Build the rpmalloc static libraries if needed. 175 if use_rpmalloc: 176 # Maps a target.platform string to the location of the corresponding 177 # rpmalloc static library. 178 rpmalloc_static_libs = {} 179 180 # Get the list of all target platforms that are listed in `configs` 181 # above. Note that this is a list of Target instances, some of them 182 # may refer to the same platform string (e.g. linux-amd64). 183 # 184 # For each platform, a version of rpmalloc will be built if necessary, 185 # but doing this properly requires having a valid target instance to 186 # call _get_compilation_environment. So create a { platform -> Target } 187 # map to do that later. 188 all_config_platforms = {} 189 for c in configs: 190 if not c.get('use_rpmalloc', False): 191 continue 192 for t in c['targets']: 193 if t.platform not in all_config_platforms: 194 all_config_platforms[t.platform] = t 195 196 rpmalloc_src_dir = api.path['start_dir'].join('rpmalloc') 197 with api.step.nest('rpmalloc'): 198 api.step('init', ['git', 'init', rpmalloc_src_dir]) 199 with api.context(cwd=rpmalloc_src_dir, infra_steps=True): 200 api.step( 201 'fetch', 202 ['git', 'fetch', '--tags', RPMALLOC_GIT_URL, RPMALLOC_BRANCH]) 203 api.step('checkout', ['git', 'checkout', RPMALLOC_REVISION]) 204 205 # Patch configure.py since to add -Wno-ignored-optimization-flag since 206 # Clang will now complain when `-funit-at-a-time` is being used. 207 build_ninja_clang_path = api.path.join(rpmalloc_src_dir, 'build/ninja/clang.py') 208 build_ninja_clang_py = api.file.read_text('read %s' % build_ninja_clang_path, 209 build_ninja_clang_path, 210 "CXXFLAGS = ['-Wall', '-Weverything', '-Wfoo']") 211 build_ninja_clang_py = build_ninja_clang_py.replace( 212 "'-Wno-disabled-macro-expansion'", 213 "'-Wno-disabled-macro-expansion', '-Wno-ignored-optimization-argument'") 214 api.file.write_text('write %s' % build_ninja_clang_path, 215 build_ninja_clang_path, 216 build_ninja_clang_py) 217 218 for platform in all_config_platforms: 219 # Convert target architecture and os to rpmalloc format. 220 rpmalloc_os, rpmalloc_arch = platform.split('-') 221 rpmalloc_os = RPMALLOC_MAP.get(rpmalloc_os, rpmalloc_os) 222 rpmalloc_arch = RPMALLOC_MAP.get(rpmalloc_arch, rpmalloc_arch) 223 224 env = _get_compilation_environment(api, 225 all_config_platforms[platform], 226 cipd_dir) 227 with api.step.nest('build rpmalloc-' + platform), api.context( 228 env=env, cwd=rpmalloc_src_dir): 229 api.python( 230 'configure', 231 rpmalloc_src_dir.join('configure.py'), 232 args=['-c', 'release', '-a', rpmalloc_arch, '--lto']) 233 234 # NOTE: Only build the static library. 235 rpmalloc_static_lib = api.path.join('lib', rpmalloc_os, 'release', 236 rpmalloc_arch, 237 'librpmallocwrap.a') 238 api.step('ninja', [cipd_dir.join('ninja'), rpmalloc_static_lib]) 239 240 rpmalloc_static_libs[platform] = rpmalloc_src_dir.join( 241 rpmalloc_static_lib) 242 243 for config in configs: 244 with api.step.nest(config['name']): 245 for target in config['targets']: 246 env = _get_compilation_environment(api, target, cipd_dir) 247 with api.step.nest(target.platform), api.context( 248 env=env, cwd=src_dir): 249 args = config['args'] 250 if config.get('use_rpmalloc', False): 251 args = args[:] + [ 252 '--link-lib=%s' % rpmalloc_static_libs[target.platform] 253 ] 254 255 api.python('generate', src_dir.join('build', 'gen.py'), args=args) 256 257 # Windows requires the environment modifications when building too. 258 api.step('build', 259 [cipd_dir.join('ninja'), '-C', 260 src_dir.join('out')]) 261 262 if target.is_host: 263 api.step('test', [src_dir.join('out', 'gn_unittests')]) 264 265 if config['name'] != 'release': 266 continue 267 268 with api.step.nest('upload'): 269 gn = 'gn' + ('.exe' if target.is_win else '') 270 271 if build_input.gerrit_changes: 272 # Upload to CAS from CQ. 273 api.cas.archive('upload binary to CAS', src_dir.join('out'), 274 src_dir.join('out', gn)) 275 continue 276 277 cipd_pkg_name = 'gn/gn/%s' % target.platform 278 279 pkg_def = api.cipd.PackageDefinition( 280 package_name=cipd_pkg_name, 281 package_root=src_dir.join('out'), 282 install_mode='copy') 283 pkg_def.add_file(src_dir.join('out', gn)) 284 pkg_def.add_version_file('.versions/%s.cipd_version' % gn) 285 286 cipd_pkg_file = api.path['cleanup'].join('gn.cipd') 287 288 api.cipd.build_from_pkg( 289 pkg_def=pkg_def, 290 output_package=cipd_pkg_file, 291 ) 292 293 if api.buildbucket.builder_id.project == 'infra-internal': 294 cipd_pin = api.cipd.search(cipd_pkg_name, 295 'git_revision:' + revision) 296 if cipd_pin: 297 api.step('Package is up-to-date', cmd=None) 298 continue 299 300 api.cipd.register( 301 package_name=cipd_pkg_name, 302 package_path=cipd_pkg_file, 303 refs=['latest'], 304 tags={ 305 'git_repository': repository, 306 'git_revision': revision, 307 }, 308 ) 309 310 311def GenTests(api): 312 for platform in ('linux', 'mac', 'win'): 313 yield (api.test('ci_' + platform) + api.platform.name(platform) + 314 api.buildbucket.ci_build( 315 project='gn', 316 git_repo='gn.googlesource.com/gn', 317 )) 318 319 yield (api.test('cq_' + platform) + api.platform.name(platform) + 320 api.buildbucket.try_build( 321 project='gn', 322 git_repo='gn.googlesource.com/gn', 323 )) 324 325 yield (api.test('cipd_exists') + api.buildbucket.ci_build( 326 project='infra-internal', 327 git_repo='gn.googlesource.com/gn', 328 revision='a' * 40, 329 ) + api.step_data( 330 'git.rev-parse', api.raw_io.stream_output_text('a' * 40) 331 ) + api.step_data( 332 'release.linux-amd64.upload.cipd search gn/gn/linux-amd64 git_revision:' + 333 'a' * 40, 334 api.cipd.example_search('gn/gn/linux-amd64', 335 ['git_revision:' + 'a' * 40]))) 336 337 yield (api.test('cipd_register') + api.buildbucket.ci_build( 338 project='infra-internal', 339 git_repo='gn.googlesource.com/gn', 340 revision='a' * 40, 341 ) + api.step_data( 342 'git.rev-parse', api.raw_io.stream_output_text('a' * 40) 343 ) + api.step_data( 344 'release.linux-amd64.upload.cipd search gn/gn/linux-amd64 git_revision:' + 345 'a' * 40, api.cipd.example_search('gn/gn/linux-amd64', []))) 346