1# Copyright 2019 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5# Recipe which runs Skottie-WASM and Lottie-Web perf. 6 7import calendar 8import json 9import re 10 11PYTHON_VERSION_COMPATIBILITY = "PY3" 12 13# trim 14DEPS = [ 15 'flavor', 16 'checkout', 17 'env', 18 'infra', 19 'recipe_engine/context', 20 'recipe_engine/file', 21 'recipe_engine/json', 22 'recipe_engine/path', 23 'recipe_engine/properties', 24 'recipe_engine/step', 25 'recipe_engine/time', 26 'run', 27 'vars', 28] 29 30LOTTIE_WEB_EXCLUDE = [ 31 # See https://bugs.chromium.org/p/skia/issues/detail?id=9187#c4 32 'lottiefiles.com - Progress Success.json', 33 # Fails with "val2 is not defined". 34 'lottiefiles.com - VR.json', 35 'vr_animation.json', 36 # Times out. 37 'lottiefiles.com - Nudge.json', 38 'lottiefiles.com - Retweet.json', 39 'regress-perspective-blur-01.json', 40 'regress-perspective-blur-02.json', 41 # Trace file has majority main_frame_aborted terminations in it and < 25 42 # occurrences of submitted_frame + missed_frame. 43 # Static scenes (nothing animating) 44 'mask1.json', 45 'mask2.json', 46 'stacking.json', 47] 48 49SKOTTIE_WASM_EXCLUDE = [ 50 # Trace file has majority main_frame_aborted terminations in it and < 25 51 # occurrences of submitted_frame + missed_frame. 52 # Below descriptions are added from fmalita@'s comments in 53 # https://skia-review.googlesource.com/c/skia/+/229419 54 55 # Static scenes (nothing animating) 56 'mask1.json', 57 'mask2.json', 58 'stacking.json', 59 # Static in Skottie only due to unsupported feature (expressions). 60 'dna.json', 61 'elephant_trunk_swing.json', 62 # Looks all static in both skottie/lottie, not sure why lottie doesn't abort 63 # as many frames. 64 'hexadots.json', 65 # Very short transition, mostly static. 66 'screenhole.json', 67 # Broken in Skottie due to unidentified missing feature. 68 'interleague_golf_logo.json', 69 'loading.json', 70 'lottiefiles.com - Loading 2.json', 71 'streetby_loading.json', 72 'streetby_test_loading.json', 73 # Times out 74 'beetle.json', 75 # Too slow? Doesn't provide enough frames for analysis b/325452373 76 'Name.json', 77] 78 79# These files work in SVG but not in Canvas. 80LOTTIE_WEB_CANVAS_EXCLUDE = LOTTIE_WEB_EXCLUDE + [ 81 'Hello World.json', 82 'interactive_menu.json', 83 'Name.json', 84] 85 86 87def RunSteps(api): 88 api.vars.setup() 89 api.flavor.setup(None) 90 checkout_root = api.path.start_dir 91 buildername = api.properties['buildername'] 92 node_path = api.path.start_dir.joinpath('node', 'node', 'bin', 'node') 93 lottie_files = api.file.listdir( 94 'list lottie files', api.flavor.host_dirs.lotties_dir, 95 test_data=['lottie1.json', 'lottie2.json', 'lottie3.json', 'LICENSE']) 96 97 if 'SkottieWASM' in buildername: 98 source_type = 'skottie' 99 renderer = 'skottie-wasm' 100 101 perf_app_dir = checkout_root.joinpath('skia', 'tools', 'skottie-wasm-perf') 102 canvaskit_js_path = api.vars.build_dir.joinpath('canvaskit.js') 103 canvaskit_wasm_path = api.vars.build_dir.joinpath('canvaskit.wasm') 104 skottie_wasm_js_path = perf_app_dir.joinpath('skottie-wasm-perf.js') 105 perf_app_cmd = [ 106 node_path, skottie_wasm_js_path, 107 '--canvaskit_js', canvaskit_js_path, 108 '--canvaskit_wasm', canvaskit_wasm_path, 109 ] 110 lottie_files = [x for x in lottie_files 111 if api.path.basename(x) not in SKOTTIE_WASM_EXCLUDE] 112 elif 'LottieWeb' in buildername: 113 source_type = 'lottie-web' 114 renderer = 'lottie-web' 115 if 'Canvas' in buildername: 116 backend = 'canvas' 117 lottie_files = [ 118 x for x in lottie_files 119 if api.path.basename(x) not in LOTTIE_WEB_CANVAS_EXCLUDE] 120 else: 121 backend = 'svg' 122 lottie_files = [x for x in lottie_files 123 if api.path.basename(x) not in LOTTIE_WEB_EXCLUDE] 124 125 perf_app_dir = checkout_root.joinpath('skia', 'tools', 'lottie-web-perf') 126 lottie_web_js_path = perf_app_dir.joinpath('lottie-web-perf.js') 127 perf_app_cmd = [ 128 node_path, lottie_web_js_path, 129 '--backend', backend, 130 ] 131 else: 132 raise Exception('Could not recognize the buildername %s' % buildername) 133 134 if api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': 135 perf_app_cmd.append('--use_gpu') 136 137 # Install prerequisites. 138 env_prefixes = {'PATH': [api.path.start_dir.joinpath('node', 'node', 'bin')]} 139 with api.context(cwd=perf_app_dir, env_prefixes=env_prefixes): 140 api.step('npm install', cmd=['npm', 'install']) 141 142 perf_results = {} 143 output_dir = api.path.mkdtemp('g3_try') 144 # Run the perf_app_cmd on each lottie file and parse the trace files. 145 for _, lottie_file in enumerate(lottie_files): 146 lottie_filename = api.path.basename(lottie_file) 147 if not lottie_filename.endswith('.json'): 148 continue 149 output_file = output_dir.joinpath(lottie_filename) 150 with api.context(cwd=perf_app_dir, env={'DISPLAY': ':0'}): 151 # This is occasionally flaky due to skbug.com/9207, adding retries. 152 attempts = 3 153 # Add output and input arguments to the cmd. 154 api.run.with_retry(api.step, 'Run perf cmd line app', attempts, 155 cmd=perf_app_cmd + [ 156 '--input', lottie_file, 157 '--output', output_file, 158 ], infra_step=True) 159 160 perf_results[lottie_filename] = { 161 'gl': parse_trace(output_file, lottie_filename, api, renderer), 162 } 163 164 # Construct contents of the output JSON. 165 perf_json = { 166 'gitHash': api.properties['revision'], 167 'swarming_bot_id': api.vars.swarming_bot_id, 168 'swarming_task_id': api.vars.swarming_task_id, 169 'key': { 170 'bench_type': 'tracing', 171 'source_type': source_type, 172 }, 173 'renderer': renderer, 174 'results': perf_results, 175 } 176 if api.vars.is_trybot: 177 perf_json['issue'] = api.vars.issue 178 perf_json['patchset'] = api.vars.patchset 179 perf_json['patch_storage'] = api.vars.patch_storage 180 # Add tokens from the builder name to the key. 181 reg = re.compile('Perf-(?P<os>[A-Za-z0-9_]+)-' 182 '(?P<compiler>[A-Za-z0-9_]+)-' 183 '(?P<model>[A-Za-z0-9_]+)-' 184 '(?P<cpu_or_gpu>[A-Z]+)-' 185 '(?P<cpu_or_gpu_value>[A-Za-z0-9_]+)-' 186 '(?P<arch>[A-Za-z0-9_]+)-' 187 '(?P<configuration>[A-Za-z0-9_]+)-' 188 'All(-(?P<extra_config>[A-Za-z0-9_]+)|)') 189 m = reg.match(api.properties['buildername']) 190 keys = ['os', 'compiler', 'model', 'cpu_or_gpu', 'cpu_or_gpu_value', 'arch', 191 'configuration', 'extra_config'] 192 for k in keys: 193 perf_json['key'][k] = m.group(k) 194 195 # Create the output JSON file in perf_data_dir for the Upload task to upload. 196 api.file.ensure_directory( 197 'makedirs perf_dir', 198 api.flavor.host_dirs.perf_data_dir) 199 now = api.time.utcnow() 200 ts = int(calendar.timegm(now.utctimetuple())) 201 json_path = api.flavor.host_dirs.perf_data_dir.joinpath( 202 'perf_%s_%d.json' % (api.properties['revision'], ts)) 203 json_contents = json.dumps( 204 perf_json, indent=4, sort_keys=True, separators=(',', ': ')) 205 api.file.write_text('write output JSON', json_path, json_contents) 206 207 208def parse_trace(trace_json, lottie_filename, api, renderer): 209 """parse_trace parses the specified trace JSON. 210 211 Parses the trace JSON and calculates the time of a single frame. 212 A dictionary is returned that has the following structure: 213 { 214 'frame_max_us': 100, 215 'frame_min_us': 90, 216 'frame_avg_us': 95, 217 } 218 """ 219 script = api.infra.resource('parse_lottieweb_trace.py') 220 step_result = api.run( 221 api.step, 222 'parse %s trace' % lottie_filename, 223 cmd=['python3', script, trace_json, api.json.output(), renderer]) 224 225 # Sanitize float outputs to 2 precision points. 226 output = dict(step_result.json.output) 227 output['frame_max_us'] = float("%.2f" % output['frame_max_us']) 228 output['frame_min_us'] = float("%.2f" % output['frame_min_us']) 229 output['frame_avg_us'] = float("%.2f" % output['frame_avg_us']) 230 return output 231 232 233def GenTests(api): 234 trace_output = """ 235[{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":452,"dur":2.57,"tid":1,"pid":0},{"ph":"X","name":"void SkCanvas::drawPaint(const SkPaint &)","ts":473,"dur":2.67e+03,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":3.15e+03,"dur":2.25,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::render(SkCanvas *, const SkRect *, RenderFlags) const","ts":3.15e+03,"dur":216,"tid":1,"pid":0},{"ph":"X","name":"void SkCanvas::drawPath(const SkPath &, const SkPaint &)","ts":3.35e+03,"dur":15.1,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":3.37e+03,"dur":1.17,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::render(SkCanvas *, const SkRect *, RenderFlags) const","ts":3.37e+03,"dur":140,"tid":1,"pid":0}] 236""" 237 parse_trace_json = { 238 'frame_avg_us': 179.71, 239 'frame_min_us': 141.17, 240 'frame_max_us': 218.25 241 } 242 243 244 skottie_cpu_buildername = ('Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-' 245 'SkottieWASM') 246 yield ( 247 api.test('skottie_wasm_perf') + 248 api.properties(buildername=skottie_cpu_buildername, 249 repository='https://skia.googlesource.com/skia.git', 250 revision='abc123', 251 path_config='kitchen', 252 trace_test_data=trace_output, 253 swarm_out_dir='[SWARM_OUT_DIR]') + 254 api.step_data('parse lottie1.json trace', 255 api.json.output(parse_trace_json)) + 256 api.step_data('parse lottie2.json trace', 257 api.json.output(parse_trace_json)) + 258 api.step_data('parse lottie3.json trace', 259 api.json.output(parse_trace_json)) 260 ) 261 yield ( 262 api.test('skottie_wasm_perf_trybot') + 263 api.properties(buildername=skottie_cpu_buildername, 264 repository='https://skia.googlesource.com/skia.git', 265 revision='abc123', 266 path_config='kitchen', 267 trace_test_data=trace_output, 268 swarm_out_dir='[SWARM_OUT_DIR]', 269 patch_ref='89/456789/12', 270 patch_repo='https://skia.googlesource.com/skia.git', 271 patch_storage='gerrit', 272 patch_set=7, 273 patch_issue=1234, 274 gerrit_project='skia', 275 gerrit_url='https://skia-review.googlesource.com/') + 276 api.step_data('parse lottie1.json trace', 277 api.json.output(parse_trace_json)) + 278 api.step_data('parse lottie2.json trace', 279 api.json.output(parse_trace_json)) + 280 api.step_data('parse lottie3.json trace', 281 api.json.output(parse_trace_json)) 282 ) 283 284 skottie_gpu_buildername = ('Perf-Debian10-EMCC-NUC7i5BNK-GPU-IntelIris640-' 285 'wasm-Release-All-SkottieWASM') 286 yield ( 287 api.test('skottie_wasm_perf_gpu') + 288 api.properties(buildername=skottie_gpu_buildername, 289 repository='https://skia.googlesource.com/skia.git', 290 revision='abc123', 291 path_config='kitchen', 292 trace_test_data=trace_output, 293 swarm_out_dir='[SWARM_OUT_DIR]') + 294 api.step_data('parse lottie1.json trace', 295 api.json.output(parse_trace_json)) + 296 api.step_data('parse lottie2.json trace', 297 api.json.output(parse_trace_json)) + 298 api.step_data('parse lottie3.json trace', 299 api.json.output(parse_trace_json)) 300 ) 301 302 lottieweb_cpu_buildername = ('Perf-Debian10-none-GCE-CPU-AVX2-x86_64-Release-' 303 'All-LottieWeb') 304 yield ( 305 api.test('lottie_web_perf') + 306 api.properties(buildername=lottieweb_cpu_buildername, 307 repository='https://skia.googlesource.com/skia.git', 308 revision='abc123', 309 path_config='kitchen', 310 trace_test_data=trace_output, 311 swarm_out_dir='[SWARM_OUT_DIR]') + 312 api.step_data('parse lottie1.json trace', 313 api.json.output(parse_trace_json)) + 314 api.step_data('parse lottie2.json trace', 315 api.json.output(parse_trace_json)) + 316 api.step_data('parse lottie3.json trace', 317 api.json.output(parse_trace_json)) 318 ) 319 yield ( 320 api.test('lottie_web_perf_trybot') + 321 api.properties(buildername=lottieweb_cpu_buildername, 322 repository='https://skia.googlesource.com/skia.git', 323 revision='abc123', 324 path_config='kitchen', 325 trace_test_data=trace_output, 326 swarm_out_dir='[SWARM_OUT_DIR]', 327 patch_ref='89/456789/12', 328 patch_repo='https://skia.googlesource.com/skia.git', 329 patch_storage='gerrit', 330 patch_set=7, 331 patch_issue=1234, 332 gerrit_project='skia', 333 gerrit_url='https://skia-review.googlesource.com/') + 334 api.step_data('parse lottie1.json trace', 335 api.json.output(parse_trace_json)) + 336 api.step_data('parse lottie2.json trace', 337 api.json.output(parse_trace_json)) + 338 api.step_data('parse lottie3.json trace', 339 api.json.output(parse_trace_json)) 340 ) 341 342 lottieweb_canvas_cpu_buildername = ( 343 'Perf-Debian10-none-GCE-CPU-AVX2-x86_64-Release-All-LottieWeb_Canvas') 344 yield ( 345 api.test('lottie_web_canvas_perf') + 346 api.properties(buildername=lottieweb_canvas_cpu_buildername, 347 repository='https://skia.googlesource.com/skia.git', 348 revision='abc123', 349 path_config='kitchen', 350 trace_test_data=trace_output, 351 swarm_out_dir='[SWARM_OUT_DIR]') + 352 api.step_data('parse lottie1.json trace', 353 api.json.output(parse_trace_json)) + 354 api.step_data('parse lottie2.json trace', 355 api.json.output(parse_trace_json)) + 356 api.step_data('parse lottie3.json trace', 357 api.json.output(parse_trace_json)) 358 ) 359 yield ( 360 api.test('lottie_web_canvas_perf_trybot') + 361 api.properties(buildername=lottieweb_canvas_cpu_buildername, 362 repository='https://skia.googlesource.com/skia.git', 363 revision='abc123', 364 path_config='kitchen', 365 trace_test_data=trace_output, 366 swarm_out_dir='[SWARM_OUT_DIR]', 367 patch_ref='89/456789/12', 368 patch_repo='https://skia.googlesource.com/skia.git', 369 patch_storage='gerrit', 370 patch_set=7, 371 patch_issue=1234, 372 gerrit_project='skia', 373 gerrit_url='https://skia-review.googlesource.com/') + 374 api.step_data('parse lottie1.json trace', 375 api.json.output(parse_trace_json)) + 376 api.step_data('parse lottie2.json trace', 377 api.json.output(parse_trace_json)) + 378 api.step_data('parse lottie3.json trace', 379 api.json.output(parse_trace_json)) 380 ) 381 382 unrecognized_buildername = ('Perf-Debian10-none-GCE-CPU-AVX2-x86_64-Release-' 383 'All-Unrecognized') 384 yield ( 385 api.test('unrecognized_builder') + 386 api.properties(buildername=unrecognized_buildername, 387 repository='https://skia.googlesource.com/skia.git', 388 revision='abc123', 389 path_config='kitchen', 390 swarm_out_dir='[SWARM_OUT_DIR]') + 391 api.expect_exception('Exception') 392 ) 393