1# Copyright 2018 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5# Recipe which analyzes a compiled binary for information (e.g. file size) 6 7import ast 8import json 9 10PYTHON_VERSION_COMPATIBILITY = "PY2+3" 11 12DEPS = [ 13 'checkout', 14 'env', 15 'recipe_engine/context', 16 'recipe_engine/file', 17 'recipe_engine/path', 18 'recipe_engine/properties', 19 'recipe_engine/python', 20 'recipe_engine/raw_io', 21 'recipe_engine/step', 22 'run', 23 'vars', 24] 25 26 27MAGIC_SEPERATOR = '#$%^&*' 28TOTAL_SIZE_BYTES_KEY = "total_size_bytes" 29 30 31def add_binary_size_output_property(result, source, binary_size): 32 result.presentation.properties['binary_size_%s' % source] = binary_size 33 34 35def RunSteps(api): 36 api.vars.setup() 37 38 checkout_root = api.checkout.default_checkout_root 39 api.checkout.bot_update(checkout_root=checkout_root) 40 41 out_dir = api.vars.swarming_out_dir 42 # Any binaries to scan should be here. 43 bin_dir = api.vars.build_dir 44 45 api.file.ensure_directory('mkdirs out_dir', out_dir, mode=0o777) 46 47 analyzed = 0 48 with api.context(cwd=bin_dir): 49 files = api.file.glob_paths( 50 'find WASM binaries', 51 bin_dir, 52 '*.wasm', 53 test_data=['pathkit.wasm']) 54 analyzed += len(files) 55 if files: 56 analyze_wasm_file(api, checkout_root, out_dir, files) 57 58 files = api.file.glob_paths( 59 'find JS files', 60 bin_dir, 61 '*.js', 62 test_data=['pathkit.js']) 63 analyzed += len(files) 64 if files: 65 analyze_web_file(api, checkout_root, out_dir, files) 66 67 files = api.file.glob_paths( 68 'find JS mem files', 69 bin_dir, 70 '*.js.mem', 71 test_data=['pathkit.js.mem']) 72 analyzed += len(files) 73 if files: 74 analyze_web_file(api, checkout_root, out_dir, files) 75 76 files = api.file.glob_paths( 77 'find flutter library', 78 bin_dir, 79 'libflutter.so', 80 test_data=['libflutter.so']) 81 analyzed += len(files) 82 if files: 83 analyze_flutter_lib(api, checkout_root, out_dir, files) 84 85 files = api.file.glob_paths( 86 'find skia library', 87 bin_dir, 88 'libskia.so', 89 test_data=['libskia.so']) 90 analyzed += len(files) 91 if files: 92 analyze_cpp_lib(api, checkout_root, out_dir, files) 93 94 files = api.file.glob_paths( 95 'find skottie_tool', 96 bin_dir, 97 'skottie_tool', 98 test_data=['skottie_tool']) 99 analyzed += len(files) 100 if files: 101 make_treemap(api, checkout_root, out_dir, files) 102 103 files = api.file.glob_paths( 104 'find dm', 105 bin_dir, 106 'dm', 107 test_data=['dm']) 108 analyzed += len(files) 109 if files: 110 make_treemap(api, checkout_root, out_dir, files) 111 112 if not analyzed: # pragma: nocover 113 raise Exception('No files were analyzed!') 114 115 116def keys_and_props(api): 117 keys = [] 118 for k in sorted(api.vars.builder_cfg.keys()): 119 if not k in ['role']: 120 keys.extend([k, api.vars.builder_cfg[k]]) 121 keystr = ' '.join(keys) 122 123 props = [ 124 'gitHash', api.properties['revision'], 125 'swarming_bot_id', api.vars.swarming_bot_id, 126 'swarming_task_id', api.vars.swarming_task_id, 127 ] 128 129 if api.vars.is_trybot: 130 props.extend([ 131 'issue', api.vars.issue, 132 'patchset', api.vars.patchset, 133 'patch_storage', api.vars.patch_storage, 134 ]) 135 propstr = ' '.join(str(prop) for prop in props) 136 return (keystr, propstr) 137 138 139# Get the raw and gzipped size of the given file 140def analyze_web_file(api, checkout_root, out_dir, files): 141 (keystr, propstr) = keys_and_props(api) 142 143 for f in files: 144 skia_dir = checkout_root.join('skia') 145 with api.context(cwd=skia_dir): 146 script = skia_dir.join('infra', 'bots', 'buildstats', 147 'buildstats_web.py') 148 step_data = api.run(api.python, 'Analyze %s' % f, script=script, 149 args=[f, out_dir, keystr, propstr, TOTAL_SIZE_BYTES_KEY, 150 MAGIC_SEPERATOR], 151 stdout=api.raw_io.output()) 152 if step_data and step_data.stdout: 153 sections = step_data.stdout.decode('utf-8').split(MAGIC_SEPERATOR) 154 result = api.step.active_result 155 logs = result.presentation.logs 156 logs['perf_json'] = sections[1].split('\n') 157 158 add_binary_size_output_property(result, api.path.basename(f), ( 159 ast.literal_eval(sections[1]) 160 .get('results', {}) 161 .get(api.path.basename(f), {}) 162 .get('default', {}) 163 .get(TOTAL_SIZE_BYTES_KEY, {}))) 164 165 166# Get the raw size and a few metrics from bloaty 167def analyze_cpp_lib(api, checkout_root, out_dir, files): 168 (keystr, propstr) = keys_and_props(api) 169 bloaty_exe = api.path['start_dir'].join('bloaty', 'bloaty') 170 171 for f in files: 172 skia_dir = checkout_root.join('skia') 173 with api.context(cwd=skia_dir): 174 script = skia_dir.join('infra', 'bots', 'buildstats', 175 'buildstats_cpp.py') 176 step_data = api.run(api.python, 'Analyze %s' % f, script=script, 177 args=[f, out_dir, keystr, propstr, bloaty_exe, TOTAL_SIZE_BYTES_KEY, 178 MAGIC_SEPERATOR], 179 stdout=api.raw_io.output()) 180 if step_data and step_data.stdout: 181 sections = step_data.stdout.decode('utf-8').split(MAGIC_SEPERATOR) 182 result = api.step.active_result 183 logs = result.presentation.logs 184 logs['perf_json'] = sections[2].split('\n') 185 186 add_binary_size_output_property(result, api.path.basename(f), ( 187 ast.literal_eval(sections[2]) 188 .get('results', {}) 189 .get(api.path.basename(f), {}) 190 .get('default', {}) 191 .get(TOTAL_SIZE_BYTES_KEY, {}))) 192 193 194# Get the size of skia in flutter and a few metrics from bloaty 195def analyze_flutter_lib(api, checkout_root, out_dir, files): 196 (keystr, propstr) = keys_and_props(api) 197 bloaty_exe = api.path['start_dir'].join('bloaty', 'bloaty') 198 199 for f in files: 200 201 skia_dir = checkout_root.join('skia') 202 with api.context(cwd=skia_dir): 203 stripped = api.vars.build_dir.join('libflutter_stripped.so') 204 script = skia_dir.join('infra', 'bots', 'buildstats', 205 'buildstats_flutter.py') 206 config = "skia_in_flutter" 207 lib_name = "libflutter.so" 208 step_data = api.run(api.python, 'Analyze flutter', script=script, 209 args=[stripped, out_dir, keystr, propstr, bloaty_exe, 210 f, config, TOTAL_SIZE_BYTES_KEY, lib_name, 211 MAGIC_SEPERATOR], 212 stdout=api.raw_io.output()) 213 if step_data and step_data.stdout: 214 sections = step_data.stdout.decode('utf-8').split(MAGIC_SEPERATOR) 215 result = api.step.active_result 216 logs = result.presentation.logs 217 # Skip section 0 because it's everything before first print, 218 # which is probably the empty string. 219 logs['bloaty_file_symbol_short'] = sections[1].split('\n') 220 logs['bloaty_file_symbol_full'] = sections[2].split('\n') 221 logs['bloaty_symbol_file_short'] = sections[3].split('\n') 222 logs['bloaty_symbol_file_full'] = sections[4].split('\n') 223 logs['perf_json'] = sections[5].split('\n') 224 225 add_binary_size_output_property(result, lib_name, ( 226 ast.literal_eval(sections[5]) 227 .get('results', {}) 228 .get(lib_name, {}) 229 .get(config, {}) 230 .get(TOTAL_SIZE_BYTES_KEY, {}))) 231 232 233# Get the size of skia in flutter and a few metrics from bloaty 234def analyze_wasm_file(api, checkout_root, out_dir, files): 235 (keystr, propstr) = keys_and_props(api) 236 bloaty_exe = api.path['start_dir'].join('bloaty', 'bloaty') 237 238 for f in files: 239 240 skia_dir = checkout_root.join('skia') 241 with api.context(cwd=skia_dir): 242 script = skia_dir.join('infra', 'bots', 'buildstats', 243 'buildstats_wasm.py') 244 step_data = api.run(api.python, 'Analyze wasm', script=script, 245 args=[f, out_dir, keystr, propstr, bloaty_exe, 246 TOTAL_SIZE_BYTES_KEY, MAGIC_SEPERATOR], 247 stdout=api.raw_io.output()) 248 if step_data and step_data.stdout: 249 sections = step_data.stdout.decode('utf-8').split(MAGIC_SEPERATOR) 250 result = api.step.active_result 251 logs = result.presentation.logs 252 # Skip section 0 because it's everything before first print, 253 # which is probably the empty string. 254 logs['bloaty_symbol_short'] = sections[1].split('\n') 255 logs['bloaty_symbol_full'] = sections[2].split('\n') 256 logs['perf_json'] = sections[3].split('\n') 257 add_binary_size_output_property(result, api.path.basename(f), ( 258 ast.literal_eval(str(sections[3])) 259 .get('results', {}) 260 .get(api.path.basename(f), {}) 261 .get('default', {}) 262 .get(TOTAL_SIZE_BYTES_KEY, {}))) 263 264 265# make a zip file containing an HTML treemap of the files 266def make_treemap(api, checkout_root, out_dir, files): 267 for f in files: 268 env = {'DOCKER_CONFIG': '/home/chrome-bot/.docker'} 269 with api.env(env): 270 skia_dir = checkout_root.join('skia') 271 with api.context(cwd=skia_dir): 272 script = skia_dir.join('infra', 'bots', 'buildstats', 273 'make_treemap.py') 274 api.run(api.python, 'Make code size treemap %s' % f, 275 script=script, 276 args=[f, out_dir], 277 stdout=api.raw_io.output()) 278 279 280def GenTests(api): 281 builder = 'BuildStats-Debian10-EMCC-wasm-Release-PathKit' 282 yield ( 283 api.test('normal_bot') + 284 api.properties(buildername=builder, 285 repository='https://skia.googlesource.com/skia.git', 286 revision='abc123', 287 swarm_out_dir='[SWARM_OUT_DIR]', 288 path_config='kitchen') + 289 api.step_data('get swarming bot id', 290 stdout=api.raw_io.output('skia-bot-123')) + 291 api.step_data('get swarming task id', 292 stdout=api.raw_io.output('123456abc')) + 293 api.step_data('Analyze [START_DIR]/build/pathkit.js.mem', 294 stdout=api.raw_io.output(sample_web)) + 295 api.step_data('Analyze [START_DIR]/build/libskia.so', 296 stdout=api.raw_io.output(sample_cpp)) + 297 api.step_data('Analyze wasm', 298 stdout=api.raw_io.output(sample_wasm)) + 299 api.step_data('Analyze flutter', 300 stdout=api.raw_io.output(sample_flutter)) 301 ) 302 303 yield ( 304 api.test('trybot') + 305 api.properties(buildername=builder, 306 repository='https://skia.googlesource.com/skia.git', 307 revision='abc123', 308 swarm_out_dir='[SWARM_OUT_DIR]', 309 patch_repo='https://skia.googlesource.com/skia.git', 310 path_config='kitchen') + 311 api.step_data('get swarming bot id', 312 stdout=api.raw_io.output('skia-bot-123')) + 313 api.step_data('get swarming task id', 314 stdout=api.raw_io.output('123456abc')) + 315 api.properties(patch_storage='gerrit') + 316 api.properties.tryserver( 317 buildername=builder, 318 gerrit_project='skia', 319 gerrit_url='https://skia-review.googlesource.com/', 320 ) + 321 api.step_data('Analyze [START_DIR]/build/pathkit.js.mem', 322 stdout=api.raw_io.output(sample_web)) + 323 api.step_data('Analyze [START_DIR]/build/libskia.so', 324 stdout=api.raw_io.output(sample_cpp)) + 325 api.step_data('Analyze wasm', 326 stdout=api.raw_io.output(sample_wasm)) + 327 api.step_data('Analyze flutter', 328 stdout=api.raw_io.output(sample_flutter)) 329 ) 330 331sample_web = """ 332Report A 333 Total size: 50 bytes 334#$%^&* 335{ 336 "some": "json", 337 "results": { 338 "pathkit.js.mem": { 339 "default": { 340 "total_size_bytes": 7391117, 341 "gzip_size_bytes": 2884841 342 } 343 } 344 } 345} 346""" 347 348sample_cpp = """ 349#$%^&* 350Report A 351 Total size: 50 bytes 352#$%^&* 353{ 354 "some": "json", 355 "results": { 356 "libskia.so": { 357 "default": { 358 "total_size_bytes": 7391117, 359 "gzip_size_bytes": 2884841 360 } 361 } 362 } 363} 364""" 365 366sample_wasm = """ 367#$%^&* 368Report A 369 Total size: 50 bytes 370#$%^&* 371Report B 372 Total size: 60 bytes 373#$%^&* 374{ 375 "some": "json", 376 "results": { 377 "pathkit.wasm": { 378 "default": { 379 "total_size_bytes": 7391117, 380 "gzip_size_bytes": 2884841 381 } 382 } 383 } 384} 385""" 386 387sample_flutter = """ 388#$%^&* 389Report A 390 Total size: 50 bytes 391#$%^&* 392Report B 393 Total size: 60 bytes 394#$%^&* 395Report C 396 Total size: 70 bytes 397#$%^&* 398Report D 399 Total size: 80 bytes 400#$%^&* 401{ 402 "some": "json", 403 "results": { 404 "libflutter.so": { 405 "skia_in_flutter": { 406 "total_size_bytes": 1256676 407 } 408 } 409 } 410} 411""" 412