1# Copyright 2018 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5 6from . import util 7 8 9def compile_swiftshader(api, extra_tokens, swiftshader_root, cc, cxx, out): 10 """Build SwiftShader with CMake. 11 12 Building SwiftShader works differently from any other Skia third_party lib. 13 See discussion in skia:7671 for more detail. 14 15 Args: 16 swiftshader_root: root of the SwiftShader checkout. 17 cc, cxx: compiler binaries to use 18 out: target directory for libvk_swiftshader.so 19 """ 20 swiftshader_opts = [ 21 '-DSWIFTSHADER_BUILD_TESTS=OFF', 22 '-DSWIFTSHADER_WARNINGS_AS_ERRORS=OFF', 23 '-DREACTOR_ENABLE_MEMORY_SANITIZER_INSTRUMENTATION=OFF', # Way too slow. 24 ] 25 cmake_bin = str(api.vars.workdir.join('cmake_linux', 'bin')) 26 env = { 27 'CC': cc, 28 'CXX': cxx, 29 'PATH': '%%(PATH)s:%s' % cmake_bin, 30 # We arrange our MSAN/TSAN prebuilts a little differently than 31 # SwiftShader's CMakeLists.txt expects, so we'll just keep our custom 32 # setup (everything mentioning libcxx below) and point SwiftShader's 33 # CMakeLists.txt at a harmless non-existent path. 34 'SWIFTSHADER_MSAN_INSTRUMENTED_LIBCXX_PATH': '/totally/phony/path', 35 } 36 37 # Extra flags for MSAN/TSAN, if necessary. 38 san = None 39 if 'MSAN' in extra_tokens: 40 san = ('msan','memory') 41 42 if san: 43 short,full = san 44 clang_linux = str(api.vars.workdir.join('clang_linux')) 45 libcxx = clang_linux + '/' + short 46 cflags = ' '.join([ 47 '-fsanitize=' + full, 48 '-stdlib=libc++', 49 '-L%s/lib' % libcxx, 50 '-lc++abi', 51 '-I%s/include' % libcxx, 52 '-I%s/include/c++/v1' % libcxx, 53 '-Wno-unused-command-line-argument' # Are -lc++abi and -Llibcxx/lib always unused? 54 ]) 55 swiftshader_opts.extend([ 56 '-DSWIFTSHADER_{}=ON'.format(short.upper()), 57 '-DCMAKE_C_FLAGS=%s' % cflags, 58 '-DCMAKE_CXX_FLAGS=%s' % cflags, 59 ]) 60 61 # Build SwiftShader. 62 api.file.ensure_directory('makedirs swiftshader_out', out) 63 with api.context(cwd=out, env=env): 64 api.run(api.step, 'swiftshader cmake', 65 cmd=['cmake'] + swiftshader_opts + [swiftshader_root, '-GNinja']) 66 # See https://swiftshader-review.googlesource.com/c/SwiftShader/+/56452 for when the 67 # deprecated targets were added. See skbug.com/12386 for longer-term plans. 68 api.run(api.step, 'swiftshader ninja', cmd=['ninja', '-C', out, 'vk_swiftshader']) 69 70 71def compile_fn(api, checkout_root, out_dir): 72 skia_dir = checkout_root.join('skia') 73 compiler = api.vars.builder_cfg.get('compiler', '') 74 configuration = api.vars.builder_cfg.get('configuration', '') 75 extra_tokens = api.vars.extra_tokens 76 os = api.vars.builder_cfg.get('os', '') 77 target_arch = api.vars.builder_cfg.get('target_arch', '') 78 79 clang_linux = str(api.vars.workdir.join('clang_linux')) 80 win_toolchain = str(api.vars.workdir.join('win_toolchain')) 81 82 cc, cxx, ccache = None, None, None 83 extra_cflags = [] 84 extra_ldflags = [] 85 args = {'werror': 'true'} 86 env = {} 87 88 if os == 'Mac': 89 # XCode build is listed in parentheses after the version at 90 # https://developer.apple.com/news/releases/, or on Wikipedia here: 91 # https://en.wikipedia.org/wiki/Xcode#Version_comparison_table 92 # Use lowercase letters. 93 # https://chrome-infra-packages.appspot.com/p/infra_internal/ios/xcode 94 XCODE_BUILD_VERSION = '12c33' 95 if compiler == 'Xcode11.4.1': 96 XCODE_BUILD_VERSION = '11e503a' 97 extra_cflags.append( 98 '-DREBUILD_IF_CHANGED_xcode_build_version=%s' % XCODE_BUILD_VERSION) 99 mac_toolchain_cmd = api.vars.workdir.join( 100 'mac_toolchain', 'mac_toolchain') 101 xcode_app_path = api.vars.cache_dir.join('Xcode.app') 102 # Copied from 103 # https://chromium.googlesource.com/chromium/tools/build/+/e19b7d9390e2bb438b566515b141ed2b9ed2c7c2/scripts/slave/recipe_modules/ios/api.py#322 104 with api.step.nest('ensure xcode') as step_result: 105 step_result.presentation.step_text = ( 106 'Ensuring Xcode version %s in %s' % ( 107 XCODE_BUILD_VERSION, xcode_app_path)) 108 install_xcode_cmd = [ 109 mac_toolchain_cmd, 'install', 110 # "ios" is needed for simulator builds 111 # (Build-Mac-Clang-x64-Release-iOS). 112 '-kind', 'ios', 113 '-xcode-version', XCODE_BUILD_VERSION, 114 '-output-dir', xcode_app_path, 115 ] 116 api.step('install xcode', install_xcode_cmd) 117 api.step('select xcode', [ 118 'sudo', 'xcode-select', '-switch', xcode_app_path]) 119 if 'iOS' in extra_tokens: 120 # Our current min-spec for Skia is iOS 11 121 env['IPHONEOS_DEPLOYMENT_TARGET'] = '11.0' 122 args['ios_min_target'] = '"11.0"' 123 else: 124 # We have some bots on 10.13. 125 env['MACOSX_DEPLOYMENT_TARGET'] = '10.13' 126 127 # ccache + clang-tidy.sh chokes on the argument list. 128 if (api.vars.is_linux or os == 'Mac' or os == 'Mac10.15.5' or os == 'Mac10.15.7') and 'Tidy' not in extra_tokens: 129 if api.vars.is_linux: 130 ccache = api.vars.workdir.join('ccache_linux', 'bin', 'ccache') 131 # As of 2020-02-07, the sum of each Debian10-Clang-x86 132 # non-flutter/android/chromebook build takes less than 75G cache space. 133 env['CCACHE_MAXSIZE'] = '75G' 134 else: 135 ccache = api.vars.workdir.join('ccache_mac', 'bin', 'ccache') 136 # As of 2020-02-10, the sum of each Build-Mac-Clang- non-android build 137 # takes ~30G cache space. 138 env['CCACHE_MAXSIZE'] = '50G' 139 140 args['cc_wrapper'] = '"%s"' % ccache 141 142 env['CCACHE_DIR'] = api.vars.cache_dir.join('ccache') 143 env['CCACHE_MAXFILES'] = '0' 144 # Compilers are unpacked from cipd with bogus timestamps, only contribute 145 # compiler content to hashes. If Ninja ever uses absolute paths to changing 146 # directories we'll also need to set a CCACHE_BASEDIR. 147 env['CCACHE_COMPILERCHECK'] = 'content' 148 149 if compiler == 'Clang' and api.vars.is_linux: 150 cc = clang_linux + '/bin/clang' 151 cxx = clang_linux + '/bin/clang++' 152 extra_cflags .append('-B%s/bin' % clang_linux) 153 extra_ldflags.append('-B%s/bin' % clang_linux) 154 extra_ldflags.append('-fuse-ld=lld') 155 extra_cflags.append('-DPLACEHOLDER_clang_linux_version=%s' % 156 api.run.asset_version('clang_linux', skia_dir)) 157 if 'Static' in extra_tokens: 158 extra_ldflags.extend(['-static-libstdc++', '-static-libgcc']) 159 160 elif compiler == 'Clang': 161 cc, cxx = 'clang', 'clang++' 162 163 if 'Tidy' in extra_tokens: 164 # Swap in clang-tidy.sh for clang++, but update PATH so it can find clang++. 165 cxx = skia_dir.join("tools/clang-tidy.sh") 166 env['PATH'] = '%s:%%(PATH)s' % (clang_linux + '/bin') 167 # Increase ClangTidy code coverage by enabling features. 168 args.update({ 169 'skia_enable_fontmgr_empty': 'true', 170 'skia_enable_pdf': 'true', 171 'skia_use_expat': 'true', 172 'skia_use_freetype': 'true', 173 'skia_use_vulkan': 'true', 174 }) 175 176 if 'Coverage' in extra_tokens: 177 # See https://clang.llvm.org/docs/SourceBasedCodeCoverage.html for 178 # more info on using llvm to gather coverage information. 179 extra_cflags.append('-fprofile-instr-generate') 180 extra_cflags.append('-fcoverage-mapping') 181 extra_ldflags.append('-fprofile-instr-generate') 182 extra_ldflags.append('-fcoverage-mapping') 183 184 if compiler != 'MSVC' and configuration == 'Debug': 185 extra_cflags.append('-O1') 186 if compiler != 'MSVC' and configuration == 'OptimizeForSize': 187 # build IDs are required for Bloaty if we want to use strip to ignore debug symbols. 188 # https://github.com/google/bloaty/blob/master/doc/using.md#debugging-stripped-binaries 189 extra_ldflags.append('-Wl,--build-id=sha1') 190 args.update({ 191 'skia_use_runtime_icu': 'true', 192 'skia_enable_optimize_size': 'true', 193 'skia_use_jpeg_gainmaps': 'false', 194 }) 195 196 if 'Exceptions' in extra_tokens: 197 extra_cflags.append('/EHsc') 198 if 'Fast' in extra_tokens: 199 extra_cflags.extend(['-march=native', '-fomit-frame-pointer', '-O3', 200 '-ffp-contract=off']) 201 202 if len(extra_tokens) == 1 and extra_tokens[0].startswith('SK'): 203 extra_cflags.append('-D' + extra_tokens[0]) 204 # If we're limiting Skia at all, drop skcms to portable code. 205 if 'SK_CPU_LIMIT' in extra_tokens[0]: 206 extra_cflags.append('-DSKCMS_PORTABLE') 207 208 if 'MSAN' in extra_tokens: 209 extra_ldflags.append('-L' + clang_linux + '/msan') 210 elif 'TSAN' in extra_tokens: 211 extra_ldflags.append('-L' + clang_linux + '/tsan') 212 elif api.vars.is_linux: 213 extra_ldflags.append('-L' + clang_linux + '/lib') 214 215 if configuration != 'Debug': 216 args['is_debug'] = 'false' 217 if 'Dawn' in extra_tokens: 218 util.set_dawn_args_and_env(args, env, api, skia_dir) 219 if 'ANGLE' in extra_tokens: 220 args['skia_use_angle'] = 'true' 221 if 'SwiftShader' in extra_tokens: 222 swiftshader_root = skia_dir.join('third_party', 'externals', 'swiftshader') 223 swiftshader_out = out_dir.join('swiftshader_out') 224 compile_swiftshader(api, extra_tokens, swiftshader_root, cc, cxx, swiftshader_out) 225 args['skia_use_vulkan'] = 'true' 226 extra_cflags.extend(['-DSK_GPU_TOOLS_VK_LIBRARY_NAME=%s' % 227 api.vars.swarming_out_dir.join('swiftshader_out', 'libvk_swiftshader.so'), 228 ]) 229 if 'MSAN' in extra_tokens: 230 args['skia_use_fontconfig'] = 'false' 231 if 'ASAN' in extra_tokens: 232 args['skia_enable_spirv_validation'] = 'false' 233 if 'NoPrecompile' in extra_tokens: 234 args['skia_enable_precompile'] = 'false' 235 if 'Graphite' in extra_tokens: 236 args['skia_enable_graphite'] = 'true' 237 if 'NoGpu' in extra_tokens: 238 args['skia_enable_ganesh'] = 'false' 239 if 'NoDEPS' in extra_tokens: 240 args.update({ 241 'is_official_build': 'true', 242 'skia_enable_fontmgr_empty': 'true', 243 'skia_enable_ganesh': 'true', 244 245 'skia_enable_pdf': 'false', 246 'skia_use_expat': 'false', 247 'skia_use_freetype': 'false', 248 'skia_use_harfbuzz': 'false', 249 'skia_use_icu': 'false', 250 'skia_use_libjpeg_turbo_decode': 'false', 251 'skia_use_libjpeg_turbo_encode': 'false', 252 'skia_use_libpng_decode': 'false', 253 'skia_use_libpng_encode': 'false', 254 'skia_use_libwebp_decode': 'false', 255 'skia_use_libwebp_encode': 'false', 256 'skia_use_vulkan': 'false', 257 'skia_use_wuffs': 'false', 258 'skia_use_zlib': 'false', 259 }) 260 if 'Shared' in extra_tokens: 261 args['is_component_build'] = 'true' 262 if 'Vulkan' in extra_tokens and not 'Android' in extra_tokens: 263 args['skia_use_vulkan'] = 'true' 264 args['skia_enable_vulkan_debug_layers'] = 'true' 265 # When running TSAN with Vulkan on NVidia, we experienced some timeouts. We found 266 # a workaround (in GrContextFactory) that requires GL (in addition to Vulkan). 267 if 'TSAN' in extra_tokens: 268 args['skia_use_gl'] = 'true' 269 else: 270 args['skia_use_gl'] = 'false' 271 if 'Direct3D' in extra_tokens: 272 args['skia_use_direct3d'] = 'true' 273 args['skia_use_gl'] = 'false' 274 if 'Metal' in extra_tokens: 275 args['skia_use_metal'] = 'true' 276 args['skia_use_gl'] = 'false' 277 if 'iOS' in extra_tokens: 278 # Bots use Chromium signing cert. 279 args['skia_ios_identity'] = '".*83FNP.*"' 280 # Get mobileprovision via the CIPD package. 281 args['skia_ios_profile'] = '"%s"' % api.vars.workdir.join( 282 'provisioning_profile_ios', 283 'Upstream_Testing_Provisioning_Profile.mobileprovision') 284 if compiler == 'Clang' and 'Win' in os: 285 args['clang_win'] = '"%s"' % api.vars.workdir.join('clang_win') 286 extra_cflags.append('-DPLACEHOLDER_clang_win_version=%s' % 287 api.run.asset_version('clang_win', skia_dir)) 288 289 sanitize = '' 290 for t in extra_tokens: 291 if t.endswith('SAN'): 292 sanitize = t 293 if api.vars.is_linux and t == 'ASAN': 294 # skia:8712 and skia:8713 295 extra_cflags.append('-DSK_ENABLE_SCOPED_LSAN_SUPPRESSIONS') 296 if 'SafeStack' in extra_tokens: 297 assert sanitize == '' 298 sanitize = 'safe-stack' 299 300 if 'Wuffs' in extra_tokens: 301 args['skia_use_wuffs'] = 'true' 302 303 if 'AVIF' in extra_tokens: 304 args['skia_use_libavif'] = 'true' 305 306 for (k,v) in { 307 'cc': cc, 308 'cxx': cxx, 309 'sanitize': sanitize, 310 'target_cpu': target_arch, 311 'target_os': 'ios' if 'iOS' in extra_tokens else '', 312 'win_sdk': win_toolchain + '/win_sdk' if 'Win' in os else '', 313 'win_vc': win_toolchain + '/VC' if 'Win' in os else '', 314 }.items(): 315 if v: 316 args[k] = '"%s"' % v 317 if extra_cflags: 318 args['extra_cflags'] = repr(extra_cflags).replace("'", '"') 319 if extra_ldflags: 320 args['extra_ldflags'] = repr(extra_ldflags).replace("'", '"') 321 322 gn_args = ' '.join('%s=%s' % (k,v) for (k,v) in sorted(args.items())) 323 gn = skia_dir.join('bin', 'gn') 324 325 with api.context(cwd=skia_dir): 326 api.run(api.python, 327 'fetch-gn', 328 script=skia_dir.join('bin', 'fetch-gn'), 329 infra_step=True) 330 331 with api.env(env): 332 if ccache: 333 api.run(api.step, 'ccache stats-start', cmd=[ccache, '-s']) 334 api.run(api.step, 'gn gen', 335 cmd=[gn, 'gen', out_dir, '--args=' + gn_args]) 336 api.run(api.step, 'ninja', cmd=['ninja', '-C', out_dir]) 337 if ccache: 338 api.run(api.step, 'ccache stats-end', cmd=[ccache, '-s']) 339 340 341def copy_build_products(api, src, dst): 342 util.copy_listed_files(api, src, dst, util.DEFAULT_BUILD_PRODUCTS) 343 extra_tokens = api.vars.extra_tokens 344 os = api.vars.builder_cfg.get('os', '') 345 configuration = api.vars.builder_cfg.get('configuration', '') 346 347 if 'SwiftShader' in extra_tokens: 348 util.copy_listed_files(api, 349 src.join('swiftshader_out'), 350 api.vars.swarming_out_dir.join('swiftshader_out'), 351 util.DEFAULT_BUILD_PRODUCTS) 352 353 if configuration == 'OptimizeForSize': 354 util.copy_listed_files(api, src, dst, ['skottie_tool_cpu', 'skottie_tool_gpu']) 355 356 if os == 'Mac' and any('SAN' in t for t in extra_tokens): 357 # The XSAN dylibs are in 358 # Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib 359 # /clang/11.0.0/lib/darwin, where 11.0.0 could change in future versions. 360 xcode_clang_ver_dirs = api.file.listdir( 361 'find XCode Clang version', 362 api.vars.cache_dir.join( 363 'Xcode.app', 'Contents', 'Developer', 'Toolchains', 364 'XcodeDefault.xctoolchain', 'usr', 'lib', 'clang'), 365 test_data=['11.0.0']) 366 assert len(xcode_clang_ver_dirs) == 1 367 dylib_dir = xcode_clang_ver_dirs[0].join('lib', 'darwin') 368 dylibs = api.file.glob_paths('find xSAN dylibs', dylib_dir, 369 'libclang_rt.*san_osx_dynamic.dylib', 370 test_data=[ 371 'libclang_rt.asan_osx_dynamic.dylib', 372 'libclang_rt.tsan_osx_dynamic.dylib', 373 'libclang_rt.ubsan_osx_dynamic.dylib', 374 ]) 375 for f in dylibs: 376 api.file.copy('copy %s' % api.path.basename(f), f, dst) 377