1# Copyright 2019 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5# Recipe which runs Skottie-WASM and Lottie-Web perf. 6 7import calendar 8import re 9 10 11# trim 12DEPS = [ 13 'flavor', 14 'checkout', 15 'env', 16 'infra', 17 'recipe_engine/context', 18 'recipe_engine/file', 19 'recipe_engine/json', 20 'recipe_engine/path', 21 'recipe_engine/properties', 22 'recipe_engine/python', 23 'recipe_engine/step', 24 'recipe_engine/tempfile', 25 'recipe_engine/time', 26 'run', 27 'vars', 28] 29 30LOTTIE_WEB_BLACKLIST = [ 31 # See https://bugs.chromium.org/p/skia/issues/detail?id=9187#c4 32 'lottiefiles.com - Progress Success.json', 33 # Fails with "val2 is not defined". 34 'lottiefiles.com - VR.json', 35 'vr_animation.json', 36 # Times out. 37 'obama_caricature.json', 38 'lottiefiles.com - Nudge.json', 39 'lottiefiles.com - Retweet.json', 40 # Trace file has majority main_frame_aborted terminations in it and < 25 41 # occurrences of submitted_frame + missed_frame. 42 # Static scenes (nothing animating) 43 'mask1.json', 44 'mask2.json', 45 'stacking.json', 46] 47 48SKOTTIE_WASM_BLACKLIST = [ 49 # Trace file has majority main_frame_aborted terminations in it and < 25 50 # occurrences of submitted_frame + missed_frame. 51 # Below descriptions are added from fmalita@'s comments in 52 # https://skia-review.googlesource.com/c/skia/+/229419 53 54 # Static scenes (nothing animating) 55 'mask1.json', 56 'mask2.json', 57 'stacking.json', 58 # Static in Skottie only due to unsupported feature (expressions). 59 'dna.json', 60 'elephant_trunk_swing.json', 61 # Looks all static in both skottie/lottie, not sure why lottie doesn't abort 62 # as many frames. 63 'hexadots.json', 64 # Very short transition, mostly static. 65 'screenhole.json', 66 # Broken in Skottie due to unidentified missing feature. 67 'interleague_golf_logo.json', 68 'loading.json', 69 'lottiefiles.com - Loading 2.json', 70 'streetby_loading.json', 71 'streetby_test_loading.json', 72] 73 74# These files work in SVG but not in Canvas. 75LOTTIE_WEB_CANVAS_BLACKLIST = LOTTIE_WEB_BLACKLIST + [ 76 'Hello World.json', 77 'interactive_menu.json', 78 'Name.json', 79] 80 81 82def RunSteps(api): 83 api.vars.setup() 84 api.flavor.setup() 85 checkout_root = api.checkout.default_checkout_root 86 api.checkout.bot_update(checkout_root=checkout_root) 87 buildername = api.properties['buildername'] 88 node_path = api.path['start_dir'].join('node', 'node', 'bin', 'node') 89 lottie_files = api.file.listdir( 90 'list lottie files', api.flavor.host_dirs.lotties_dir, 91 test_data=['lottie1.json', 'lottie2.json', 'lottie3.json', 'LICENSE']) 92 93 if 'SkottieWASM' in buildername: 94 source_type = 'skottie' 95 renderer = 'skottie-wasm' 96 97 perf_app_dir = checkout_root.join('skia', 'tools', 'skottie-wasm-perf') 98 canvaskit_js_path = api.vars.build_dir.join('canvaskit.js') 99 canvaskit_wasm_path = api.vars.build_dir.join('canvaskit.wasm') 100 skottie_wasm_js_path = perf_app_dir.join('skottie-wasm-perf.js') 101 perf_app_cmd = [ 102 node_path, skottie_wasm_js_path, 103 '--canvaskit_js', canvaskit_js_path, 104 '--canvaskit_wasm', canvaskit_wasm_path, 105 ] 106 lottie_files = [x for x in lottie_files 107 if api.path.basename(x) not in SKOTTIE_WASM_BLACKLIST] 108 elif 'LottieWeb' in buildername: 109 source_type = 'lottie-web' 110 renderer = 'lottie-web' 111 if 'Canvas' in buildername: 112 backend = 'canvas' 113 lottie_files = [ 114 x for x in lottie_files 115 if api.path.basename(x) not in LOTTIE_WEB_CANVAS_BLACKLIST] 116 else: 117 backend = 'svg' 118 lottie_files = [x for x in lottie_files 119 if api.path.basename(x) not in LOTTIE_WEB_BLACKLIST] 120 121 perf_app_dir = checkout_root.join('skia', 'tools', 'lottie-web-perf') 122 lottie_web_js_path = perf_app_dir.join('lottie-web-perf.js') 123 perf_app_cmd = [ 124 node_path, lottie_web_js_path, 125 '--backend', backend, 126 ] 127 else: 128 raise Exception('Could not recognize the buildername %s' % buildername) 129 130 if api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': 131 perf_app_cmd.append('--use_gpu') 132 133 # Install prerequisites. 134 env_prefixes = {'PATH': [api.path['start_dir'].join('node', 'node', 'bin')]} 135 with api.context(cwd=perf_app_dir, env_prefixes=env_prefixes): 136 api.step('npm install', cmd=['npm', 'install']) 137 138 perf_results = {} 139 with api.tempfile.temp_dir('g3_try') as output_dir: 140 # Run the perf_app_cmd on each lottie file and parse the trace files. 141 for _, lottie_file in enumerate(lottie_files): 142 lottie_filename = api.path.basename(lottie_file) 143 if not lottie_filename.endswith('.json'): 144 continue 145 output_file = output_dir.join(lottie_filename) 146 with api.context(cwd=perf_app_dir, env={'DISPLAY': ':0'}): 147 # This is occasionally flaky due to skbug.com/9207, adding retries. 148 attempts = 3 149 # Add output and input arguments to the cmd. 150 api.run.with_retry(api.step, 'Run perf cmd line app', attempts, 151 cmd=perf_app_cmd + [ 152 '--input', lottie_file, 153 '--output', output_file, 154 ], infra_step=True) 155 156 perf_results[lottie_filename] = { 157 'gl': parse_trace(output_file, lottie_filename, api, renderer), 158 } 159 160 # Construct contents of the output JSON. 161 perf_json = { 162 'gitHash': api.properties['revision'], 163 'swarming_bot_id': api.vars.swarming_bot_id, 164 'swarming_task_id': api.vars.swarming_task_id, 165 'key': { 166 'bench_type': 'tracing', 167 'source_type': source_type, 168 }, 169 'renderer': renderer, 170 'results': perf_results, 171 } 172 if api.vars.is_trybot: 173 perf_json['issue'] = api.vars.issue 174 perf_json['patchset'] = api.vars.patchset 175 perf_json['patch_storage'] = api.vars.patch_storage 176 # Add tokens from the builder name to the key. 177 reg = re.compile('Perf-(?P<os>[A-Za-z0-9_]+)-' 178 '(?P<compiler>[A-Za-z0-9_]+)-' 179 '(?P<model>[A-Za-z0-9_]+)-' 180 '(?P<cpu_or_gpu>[A-Z]+)-' 181 '(?P<cpu_or_gpu_value>[A-Za-z0-9_]+)-' 182 '(?P<arch>[A-Za-z0-9_]+)-' 183 '(?P<configuration>[A-Za-z0-9_]+)-' 184 'All(-(?P<extra_config>[A-Za-z0-9_]+)|)') 185 m = reg.match(api.properties['buildername']) 186 keys = ['os', 'compiler', 'model', 'cpu_or_gpu', 'cpu_or_gpu_value', 'arch', 187 'configuration', 'extra_config'] 188 for k in keys: 189 perf_json['key'][k] = m.group(k) 190 191 # Create the output JSON file in perf_data_dir for the Upload task to upload. 192 api.file.ensure_directory( 193 'makedirs perf_dir', 194 api.flavor.host_dirs.perf_data_dir) 195 now = api.time.utcnow() 196 ts = int(calendar.timegm(now.utctimetuple())) 197 json_path = api.flavor.host_dirs.perf_data_dir.join( 198 'perf_%s_%d.json' % (api.properties['revision'], ts)) 199 api.run( 200 api.python.inline, 201 'write output JSON', 202 program="""import json 203with open('%s', 'w') as outfile: 204 json.dump(obj=%s, fp=outfile, indent=4) 205 """ % (json_path, perf_json)) 206 207 208def parse_trace(trace_json, lottie_filename, api, renderer): 209 """parse_trace parses the specified trace JSON. 210 211 Parses the trace JSON and calculates the time of a single frame. 212 A dictionary is returned that has the following structure: 213 { 214 'frame_max_us': 100, 215 'frame_min_us': 90, 216 'frame_avg_us': 95, 217 } 218 """ 219 step_result = api.run( 220 api.python.inline, 221 'parse %s trace' % lottie_filename, 222 program=""" 223 import json 224 import sys 225 226 trace_output = sys.argv[1] 227 with open(trace_output, 'r') as f: 228 trace_json = json.load(f) 229 output_json_file = sys.argv[2] 230 renderer = sys.argv[3] # Unused for now but might be useful in the future. 231 232 # Output data about the GPU that was used. 233 print 'GPU data:' 234 print trace_json['metadata'].get('gpu-gl-renderer') 235 print trace_json['metadata'].get('gpu-driver') 236 print trace_json['metadata'].get('gpu-gl-vendor') 237 238 erroneous_termination_statuses = [ 239 'replaced_by_new_reporter_at_same_stage', 240 'did_not_produce_frame', 241 ] 242 accepted_termination_statuses = [ 243 'missed_frame', 244 'submitted_frame', 245 'main_frame_aborted' 246 ] 247 248 current_frame_duration = 0 249 total_frames = 0 250 frame_id_to_start_ts = {} 251 # Will contain tuples of frame_ids and their duration and status. 252 completed_frame_id_and_duration_status = [] 253 # Will contain tuples of drawn frame_ids and their duration. 254 drawn_frame_id_and_duration = [] 255 for trace in trace_json['traceEvents']: 256 if 'PipelineReporter' in trace['name']: 257 frame_id = trace['id'] 258 args = trace.get('args') 259 if args and args.get('step') == 'BeginImplFrameToSendBeginMainFrame': 260 frame_id_to_start_ts[frame_id] = trace['ts'] 261 elif args and (args.get('termination_status') in 262 accepted_termination_statuses): 263 if not frame_id_to_start_ts.get(frame_id): 264 print '[No start ts found for %s]' % frame_id 265 continue 266 current_frame_duration = trace['ts'] - frame_id_to_start_ts[frame_id] 267 total_frames += 1 268 completed_frame_id_and_duration_status.append( 269 (frame_id, current_frame_duration, args['termination_status'])) 270 if(args['termination_status'] == 'missed_frame' or 271 args['termination_status'] == 'submitted_frame'): 272 drawn_frame_id_and_duration.append((frame_id, current_frame_duration)) 273 274 # We are done with this frame_id so remove it from the dict. 275 frame_id_to_start_ts.pop(frame_id) 276 print '%d (%s with %s): %d' % ( 277 total_frames, frame_id, args['termination_status'], 278 current_frame_duration) 279 elif args and (args.get('termination_status') in 280 erroneous_termination_statuses): 281 # Invalidate previously collected results for this frame_id. 282 if frame_id_to_start_ts.get(frame_id): 283 print '[Invalidating %s due to %s]' % ( 284 frame_id, args['termination_status']) 285 frame_id_to_start_ts.pop(frame_id) 286 287 # Calculate metrics for total completed frames. 288 total_completed_frames = len(completed_frame_id_and_duration_status) 289 if total_completed_frames < 25: 290 raise Exception('Even with 3 loops found only %d frames' % 291 total_completed_frames) 292 # Get frame avg/min/max for the middle 25 frames. 293 start = (total_completed_frames - 25)/2 294 print 'Got %d total completed frames. Using indexes [%d, %d).' % ( 295 total_completed_frames, start, start+25) 296 frame_max = 0 297 frame_min = 0 298 frame_cumulative = 0 299 aborted_frames = 0 300 for frame_id, duration, status in ( 301 completed_frame_id_and_duration_status[start:start+25]): 302 frame_max = max(frame_max, duration) 303 frame_min = min(frame_min, duration) if frame_min else duration 304 frame_cumulative += duration 305 if status == 'main_frame_aborted': 306 aborted_frames += 1 307 308 perf_results = {} 309 perf_results['frame_max_us'] = frame_max 310 perf_results['frame_min_us'] = frame_min 311 perf_results['frame_avg_us'] = frame_cumulative/25 312 perf_results['aborted_frames'] = aborted_frames 313 314 # Now calculate metrics for only drawn frames. 315 drawn_frame_max = 0 316 drawn_frame_min = 0 317 drawn_frame_cumulative = 0 318 total_drawn_frames = len(drawn_frame_id_and_duration) 319 if total_drawn_frames < 25: 320 raise Exception('Even with 3 loops found only %d drawn frames' % 321 total_drawn_frames) 322 # Get drawn frame avg/min/max from the middle 25 frames. 323 start = (total_drawn_frames - 25)/2 324 print 'Got %d total drawn frames. Using indexes [%d-%d).' % ( 325 total_drawn_frames, start, start+25) 326 for frame_id, duration in drawn_frame_id_and_duration[start:start+25]: 327 drawn_frame_max = max(drawn_frame_max, duration) 328 drawn_frame_min = (min(drawn_frame_min, duration) 329 if drawn_frame_min else duration) 330 drawn_frame_cumulative += duration 331 # Add metrics to perf_results. 332 perf_results['drawn_frame_max_us'] = drawn_frame_max 333 perf_results['drawn_frame_min_us'] = drawn_frame_min 334 perf_results['drawn_frame_avg_us'] = drawn_frame_cumulative/25 335 336 print 'Final perf_results dict: %s' % perf_results 337 338 # Write perf_results to the output json. 339 with open(output_json_file, 'w') as f: 340 f.write(json.dumps(perf_results)) 341 """, args=[trace_json, api.json.output(), renderer]) 342 343 # Sanitize float outputs to 2 precision points. 344 output = dict(step_result.json.output) 345 output['frame_max_us'] = float("%.2f" % output['frame_max_us']) 346 output['frame_min_us'] = float("%.2f" % output['frame_min_us']) 347 output['frame_avg_us'] = float("%.2f" % output['frame_avg_us']) 348 return output 349 350 351def GenTests(api): 352 trace_output = """ 353[{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":452,"dur":2.57,"tid":1,"pid":0},{"ph":"X","name":"void SkCanvas::drawPaint(const SkPaint &)","ts":473,"dur":2.67e+03,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":3.15e+03,"dur":2.25,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::render(SkCanvas *, const SkRect *, RenderFlags) const","ts":3.15e+03,"dur":216,"tid":1,"pid":0},{"ph":"X","name":"void SkCanvas::drawPath(const SkPath &, const SkPaint &)","ts":3.35e+03,"dur":15.1,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":3.37e+03,"dur":1.17,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::render(SkCanvas *, const SkRect *, RenderFlags) const","ts":3.37e+03,"dur":140,"tid":1,"pid":0}] 354""" 355 parse_trace_json = { 356 'frame_avg_us': 179.71, 357 'frame_min_us': 141.17, 358 'frame_max_us': 218.25 359 } 360 361 362 skottie_cpu_buildername = ('Perf-Debian9-EMCC-GCE-CPU-AVX2-wasm-Release-All-' 363 'SkottieWASM') 364 yield ( 365 api.test('skottie_wasm_perf') + 366 api.properties(buildername=skottie_cpu_buildername, 367 repository='https://skia.googlesource.com/skia.git', 368 revision='abc123', 369 path_config='kitchen', 370 trace_test_data=trace_output, 371 swarm_out_dir='[SWARM_OUT_DIR]') + 372 api.step_data('parse lottie1.json trace', 373 api.json.output(parse_trace_json)) + 374 api.step_data('parse lottie2.json trace', 375 api.json.output(parse_trace_json)) + 376 api.step_data('parse lottie3.json trace', 377 api.json.output(parse_trace_json)) 378 ) 379 yield ( 380 api.test('skottie_wasm_perf_trybot') + 381 api.properties(buildername=skottie_cpu_buildername, 382 repository='https://skia.googlesource.com/skia.git', 383 revision='abc123', 384 path_config='kitchen', 385 trace_test_data=trace_output, 386 swarm_out_dir='[SWARM_OUT_DIR]', 387 patch_ref='89/456789/12', 388 patch_repo='https://skia.googlesource.com/skia.git', 389 patch_storage='gerrit', 390 patch_set=7, 391 patch_issue=1234, 392 gerrit_project='skia', 393 gerrit_url='https://skia-review.googlesource.com/') + 394 api.step_data('parse lottie1.json trace', 395 api.json.output(parse_trace_json)) + 396 api.step_data('parse lottie2.json trace', 397 api.json.output(parse_trace_json)) + 398 api.step_data('parse lottie3.json trace', 399 api.json.output(parse_trace_json)) 400 ) 401 402 skottie_gpu_buildername = ('Perf-Debian9-EMCC-NUC7i5BNK-GPU-IntelIris640-' 403 'wasm-Release-All-SkottieWASM') 404 yield ( 405 api.test('skottie_wasm_perf_gpu') + 406 api.properties(buildername=skottie_gpu_buildername, 407 repository='https://skia.googlesource.com/skia.git', 408 revision='abc123', 409 path_config='kitchen', 410 trace_test_data=trace_output, 411 swarm_out_dir='[SWARM_OUT_DIR]') + 412 api.step_data('parse lottie1.json trace', 413 api.json.output(parse_trace_json)) + 414 api.step_data('parse lottie2.json trace', 415 api.json.output(parse_trace_json)) + 416 api.step_data('parse lottie3.json trace', 417 api.json.output(parse_trace_json)) 418 ) 419 420 lottieweb_cpu_buildername = ('Perf-Debian9-none-GCE-CPU-AVX2-x86_64-Release-' 421 'All-LottieWeb') 422 yield ( 423 api.test('lottie_web_perf') + 424 api.properties(buildername=lottieweb_cpu_buildername, 425 repository='https://skia.googlesource.com/skia.git', 426 revision='abc123', 427 path_config='kitchen', 428 trace_test_data=trace_output, 429 swarm_out_dir='[SWARM_OUT_DIR]') + 430 api.step_data('parse lottie1.json trace', 431 api.json.output(parse_trace_json)) + 432 api.step_data('parse lottie2.json trace', 433 api.json.output(parse_trace_json)) + 434 api.step_data('parse lottie3.json trace', 435 api.json.output(parse_trace_json)) 436 ) 437 yield ( 438 api.test('lottie_web_perf_trybot') + 439 api.properties(buildername=lottieweb_cpu_buildername, 440 repository='https://skia.googlesource.com/skia.git', 441 revision='abc123', 442 path_config='kitchen', 443 trace_test_data=trace_output, 444 swarm_out_dir='[SWARM_OUT_DIR]', 445 patch_ref='89/456789/12', 446 patch_repo='https://skia.googlesource.com/skia.git', 447 patch_storage='gerrit', 448 patch_set=7, 449 patch_issue=1234, 450 gerrit_project='skia', 451 gerrit_url='https://skia-review.googlesource.com/') + 452 api.step_data('parse lottie1.json trace', 453 api.json.output(parse_trace_json)) + 454 api.step_data('parse lottie2.json trace', 455 api.json.output(parse_trace_json)) + 456 api.step_data('parse lottie3.json trace', 457 api.json.output(parse_trace_json)) 458 ) 459 460 lottieweb_canvas_cpu_buildername = ( 461 'Perf-Debian9-none-GCE-CPU-AVX2-x86_64-Release-All-LottieWeb_Canvas') 462 yield ( 463 api.test('lottie_web_canvas_perf') + 464 api.properties(buildername=lottieweb_canvas_cpu_buildername, 465 repository='https://skia.googlesource.com/skia.git', 466 revision='abc123', 467 path_config='kitchen', 468 trace_test_data=trace_output, 469 swarm_out_dir='[SWARM_OUT_DIR]') + 470 api.step_data('parse lottie1.json trace', 471 api.json.output(parse_trace_json)) + 472 api.step_data('parse lottie2.json trace', 473 api.json.output(parse_trace_json)) + 474 api.step_data('parse lottie3.json trace', 475 api.json.output(parse_trace_json)) 476 ) 477 yield ( 478 api.test('lottie_web_canvas_perf_trybot') + 479 api.properties(buildername=lottieweb_canvas_cpu_buildername, 480 repository='https://skia.googlesource.com/skia.git', 481 revision='abc123', 482 path_config='kitchen', 483 trace_test_data=trace_output, 484 swarm_out_dir='[SWARM_OUT_DIR]', 485 patch_ref='89/456789/12', 486 patch_repo='https://skia.googlesource.com/skia.git', 487 patch_storage='gerrit', 488 patch_set=7, 489 patch_issue=1234, 490 gerrit_project='skia', 491 gerrit_url='https://skia-review.googlesource.com/') + 492 api.step_data('parse lottie1.json trace', 493 api.json.output(parse_trace_json)) + 494 api.step_data('parse lottie2.json trace', 495 api.json.output(parse_trace_json)) + 496 api.step_data('parse lottie3.json trace', 497 api.json.output(parse_trace_json)) 498 ) 499 500 unrecognized_buildername = ('Perf-Debian9-none-GCE-CPU-AVX2-x86_64-Release-' 501 'All-Unrecognized') 502 yield ( 503 api.test('unrecognized_builder') + 504 api.properties(buildername=unrecognized_buildername, 505 repository='https://skia.googlesource.com/skia.git', 506 revision='abc123', 507 path_config='kitchen', 508 swarm_out_dir='[SWARM_OUT_DIR]') + 509 api.expect_exception('Exception') 510 ) 511