1# Copyright 2019 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5# Recipe which runs Skottie-WASM and Lottie-Web perf. 6 7import calendar 8import json 9import re 10 11PYTHON_VERSION_COMPATIBILITY = "PY3" 12 13# trim 14DEPS = [ 15 'flavor', 16 'checkout', 17 'env', 18 'infra', 19 'recipe_engine/context', 20 'recipe_engine/file', 21 'recipe_engine/json', 22 'recipe_engine/path', 23 'recipe_engine/properties', 24 'recipe_engine/python', 25 'recipe_engine/step', 26 'recipe_engine/time', 27 'run', 28 'vars', 29] 30 31LOTTIE_WEB_EXCLUDE = [ 32 # See https://bugs.chromium.org/p/skia/issues/detail?id=9187#c4 33 'lottiefiles.com - Progress Success.json', 34 # Fails with "val2 is not defined". 35 'lottiefiles.com - VR.json', 36 'vr_animation.json', 37 # Times out. 38 'lottiefiles.com - Nudge.json', 39 'lottiefiles.com - Retweet.json', 40 # Trace file has majority main_frame_aborted terminations in it and < 25 41 # occurrences of submitted_frame + missed_frame. 42 # Static scenes (nothing animating) 43 'mask1.json', 44 'mask2.json', 45 'stacking.json', 46] 47 48SKOTTIE_WASM_EXCLUDE = [ 49 # Trace file has majority main_frame_aborted terminations in it and < 25 50 # occurrences of submitted_frame + missed_frame. 51 # Below descriptions are added from fmalita@'s comments in 52 # https://skia-review.googlesource.com/c/skia/+/229419 53 54 # Static scenes (nothing animating) 55 'mask1.json', 56 'mask2.json', 57 'stacking.json', 58 # Static in Skottie only due to unsupported feature (expressions). 59 'dna.json', 60 'elephant_trunk_swing.json', 61 # Looks all static in both skottie/lottie, not sure why lottie doesn't abort 62 # as many frames. 63 'hexadots.json', 64 # Very short transition, mostly static. 65 'screenhole.json', 66 # Broken in Skottie due to unidentified missing feature. 67 'interleague_golf_logo.json', 68 'loading.json', 69 'lottiefiles.com - Loading 2.json', 70 'streetby_loading.json', 71 'streetby_test_loading.json', 72 # Times out 73 'beetle.json', 74] 75 76# These files work in SVG but not in Canvas. 77LOTTIE_WEB_CANVAS_EXCLUDE = LOTTIE_WEB_EXCLUDE + [ 78 'Hello World.json', 79 'interactive_menu.json', 80 'Name.json', 81] 82 83 84def RunSteps(api): 85 api.vars.setup() 86 api.flavor.setup(None) 87 checkout_root = api.path['start_dir'] 88 buildername = api.properties['buildername'] 89 node_path = api.path['start_dir'].join('node', 'node', 'bin', 'node') 90 lottie_files = api.file.listdir( 91 'list lottie files', api.flavor.host_dirs.lotties_dir, 92 test_data=['lottie1.json', 'lottie2.json', 'lottie3.json', 'LICENSE']) 93 94 if 'SkottieWASM' in buildername: 95 source_type = 'skottie' 96 renderer = 'skottie-wasm' 97 98 perf_app_dir = checkout_root.join('skia', 'tools', 'skottie-wasm-perf') 99 canvaskit_js_path = api.vars.build_dir.join('canvaskit.js') 100 canvaskit_wasm_path = api.vars.build_dir.join('canvaskit.wasm') 101 skottie_wasm_js_path = perf_app_dir.join('skottie-wasm-perf.js') 102 perf_app_cmd = [ 103 node_path, skottie_wasm_js_path, 104 '--canvaskit_js', canvaskit_js_path, 105 '--canvaskit_wasm', canvaskit_wasm_path, 106 ] 107 lottie_files = [x for x in lottie_files 108 if api.path.basename(x) not in SKOTTIE_WASM_EXCLUDE] 109 elif 'LottieWeb' in buildername: 110 source_type = 'lottie-web' 111 renderer = 'lottie-web' 112 if 'Canvas' in buildername: 113 backend = 'canvas' 114 lottie_files = [ 115 x for x in lottie_files 116 if api.path.basename(x) not in LOTTIE_WEB_CANVAS_EXCLUDE] 117 else: 118 backend = 'svg' 119 lottie_files = [x for x in lottie_files 120 if api.path.basename(x) not in LOTTIE_WEB_EXCLUDE] 121 122 perf_app_dir = checkout_root.join('skia', 'tools', 'lottie-web-perf') 123 lottie_web_js_path = perf_app_dir.join('lottie-web-perf.js') 124 perf_app_cmd = [ 125 node_path, lottie_web_js_path, 126 '--backend', backend, 127 ] 128 else: 129 raise Exception('Could not recognize the buildername %s' % buildername) 130 131 if api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': 132 perf_app_cmd.append('--use_gpu') 133 134 # Install prerequisites. 135 env_prefixes = {'PATH': [api.path['start_dir'].join('node', 'node', 'bin')]} 136 with api.context(cwd=perf_app_dir, env_prefixes=env_prefixes): 137 api.step('npm install', cmd=['npm', 'install']) 138 139 perf_results = {} 140 output_dir = api.path.mkdtemp('g3_try') 141 # Run the perf_app_cmd on each lottie file and parse the trace files. 142 for _, lottie_file in enumerate(lottie_files): 143 lottie_filename = api.path.basename(lottie_file) 144 if not lottie_filename.endswith('.json'): 145 continue 146 output_file = output_dir.join(lottie_filename) 147 with api.context(cwd=perf_app_dir, env={'DISPLAY': ':0'}): 148 # This is occasionally flaky due to skbug.com/9207, adding retries. 149 attempts = 3 150 # Add output and input arguments to the cmd. 151 api.run.with_retry(api.step, 'Run perf cmd line app', attempts, 152 cmd=perf_app_cmd + [ 153 '--input', lottie_file, 154 '--output', output_file, 155 ], infra_step=True) 156 157 perf_results[lottie_filename] = { 158 'gl': parse_trace(output_file, lottie_filename, api, renderer), 159 } 160 161 # Construct contents of the output JSON. 162 perf_json = { 163 'gitHash': api.properties['revision'], 164 'swarming_bot_id': api.vars.swarming_bot_id, 165 'swarming_task_id': api.vars.swarming_task_id, 166 'key': { 167 'bench_type': 'tracing', 168 'source_type': source_type, 169 }, 170 'renderer': renderer, 171 'results': perf_results, 172 } 173 if api.vars.is_trybot: 174 perf_json['issue'] = api.vars.issue 175 perf_json['patchset'] = api.vars.patchset 176 perf_json['patch_storage'] = api.vars.patch_storage 177 # Add tokens from the builder name to the key. 178 reg = re.compile('Perf-(?P<os>[A-Za-z0-9_]+)-' 179 '(?P<compiler>[A-Za-z0-9_]+)-' 180 '(?P<model>[A-Za-z0-9_]+)-' 181 '(?P<cpu_or_gpu>[A-Z]+)-' 182 '(?P<cpu_or_gpu_value>[A-Za-z0-9_]+)-' 183 '(?P<arch>[A-Za-z0-9_]+)-' 184 '(?P<configuration>[A-Za-z0-9_]+)-' 185 'All(-(?P<extra_config>[A-Za-z0-9_]+)|)') 186 m = reg.match(api.properties['buildername']) 187 keys = ['os', 'compiler', 'model', 'cpu_or_gpu', 'cpu_or_gpu_value', 'arch', 188 'configuration', 'extra_config'] 189 for k in keys: 190 perf_json['key'][k] = m.group(k) 191 192 # Create the output JSON file in perf_data_dir for the Upload task to upload. 193 api.file.ensure_directory( 194 'makedirs perf_dir', 195 api.flavor.host_dirs.perf_data_dir) 196 now = api.time.utcnow() 197 ts = int(calendar.timegm(now.utctimetuple())) 198 json_path = api.flavor.host_dirs.perf_data_dir.join( 199 'perf_%s_%d.json' % (api.properties['revision'], ts)) 200 json_contents = json.dumps( 201 perf_json, indent=4, sort_keys=True, separators=(',', ': ')) 202 api.file.write_text('write output JSON', json_path, json_contents) 203 204 205def parse_trace(trace_json, lottie_filename, api, renderer): 206 """parse_trace parses the specified trace JSON. 207 208 Parses the trace JSON and calculates the time of a single frame. 209 A dictionary is returned that has the following structure: 210 { 211 'frame_max_us': 100, 212 'frame_min_us': 90, 213 'frame_avg_us': 95, 214 } 215 """ 216 step_result = api.run( 217 api.python.inline, 218 'parse %s trace' % lottie_filename, 219 program=""" 220 import json 221 import sys 222 223 trace_output = sys.argv[1] 224 with open(trace_output, 'r') as f: 225 trace_json = json.load(f) 226 output_json_file = sys.argv[2] 227 renderer = sys.argv[3] # Unused for now but might be useful in the future. 228 229 # Output data about the GPU that was used. 230 print('GPU data:') 231 print(trace_json['metadata'].get('gpu-gl-renderer')) 232 print(trace_json['metadata'].get('gpu-driver')) 233 print(trace_json['metadata'].get('gpu-gl-vendor')) 234 235 erroneous_termination_statuses = [ 236 'replaced_by_new_reporter_at_same_stage', 237 'did_not_produce_frame', 238 ] 239 accepted_termination_statuses = [ 240 'missed_frame', 241 'submitted_frame', 242 'main_frame_aborted' 243 ] 244 245 current_frame_duration = 0 246 total_frames = 0 247 frame_id_to_start_ts = {} 248 # Will contain tuples of frame_ids and their duration and status. 249 completed_frame_id_and_duration_status = [] 250 # Will contain tuples of drawn frame_ids and their duration. 251 drawn_frame_id_and_duration = [] 252 for trace in trace_json['traceEvents']: 253 if 'PipelineReporter' in trace['name']: 254 frame_id = trace['id'] 255 args = trace.get('args') 256 if args and args.get('step') == 'BeginImplFrameToSendBeginMainFrame': 257 frame_id_to_start_ts[frame_id] = trace['ts'] 258 elif args and (args.get('termination_status') in 259 accepted_termination_statuses): 260 if not frame_id_to_start_ts.get(frame_id): 261 print('[No start ts found for %s]' % frame_id) 262 continue 263 current_frame_duration = trace['ts'] - frame_id_to_start_ts[frame_id] 264 total_frames += 1 265 completed_frame_id_and_duration_status.append( 266 (frame_id, current_frame_duration, args['termination_status'])) 267 if(args['termination_status'] == 'missed_frame' or 268 args['termination_status'] == 'submitted_frame'): 269 drawn_frame_id_and_duration.append((frame_id, current_frame_duration)) 270 271 # We are done with this frame_id so remove it from the dict. 272 frame_id_to_start_ts.pop(frame_id) 273 print('%d (%s with %s): %d' % ( 274 total_frames, frame_id, args['termination_status'], 275 current_frame_duration)) 276 elif args and (args.get('termination_status') in 277 erroneous_termination_statuses): 278 # Invalidate previously collected results for this frame_id. 279 if frame_id_to_start_ts.get(frame_id): 280 print('[Invalidating %s due to %s]' % ( 281 frame_id, args['termination_status'])) 282 frame_id_to_start_ts.pop(frame_id) 283 284 # Calculate metrics for total completed frames. 285 total_completed_frames = len(completed_frame_id_and_duration_status) 286 if total_completed_frames < 25: 287 raise Exception('Even with 3 loops found only %d frames' % 288 total_completed_frames) 289 # Get frame avg/min/max for the middle 25 frames. 290 start = (total_completed_frames - 25) // 2 291 print('Got %d total completed frames. Using indexes [%d, %d).' % ( 292 total_completed_frames, start, start+25)) 293 frame_max = 0 294 frame_min = 0 295 frame_cumulative = 0 296 aborted_frames = 0 297 for frame_id, duration, status in ( 298 completed_frame_id_and_duration_status[start:start+25]): 299 frame_max = max(frame_max, duration) 300 frame_min = min(frame_min, duration) if frame_min else duration 301 frame_cumulative += duration 302 if status == 'main_frame_aborted': 303 aborted_frames += 1 304 305 perf_results = {} 306 perf_results['frame_max_us'] = frame_max 307 perf_results['frame_min_us'] = frame_min 308 perf_results['frame_avg_us'] = frame_cumulative/25 309 perf_results['aborted_frames'] = aborted_frames 310 311 # Now calculate metrics for only drawn frames. 312 drawn_frame_max = 0 313 drawn_frame_min = 0 314 drawn_frame_cumulative = 0 315 total_drawn_frames = len(drawn_frame_id_and_duration) 316 if total_drawn_frames < 25: 317 raise Exception('Even with 3 loops found only %d drawn frames' % 318 total_drawn_frames) 319 # Get drawn frame avg/min/max from the middle 25 frames. 320 start = (total_drawn_frames - 25) // 2 321 print('Got %d total drawn frames. Using indexes [%d-%d).' % ( 322 total_drawn_frames, start, start+25)) 323 for frame_id, duration in drawn_frame_id_and_duration[start:start+25]: 324 drawn_frame_max = max(drawn_frame_max, duration) 325 drawn_frame_min = (min(drawn_frame_min, duration) 326 if drawn_frame_min else duration) 327 drawn_frame_cumulative += duration 328 # Add metrics to perf_results. 329 perf_results['drawn_frame_max_us'] = drawn_frame_max 330 perf_results['drawn_frame_min_us'] = drawn_frame_min 331 perf_results['drawn_frame_avg_us'] = drawn_frame_cumulative/25 332 333 print('Final perf_results dict: %s' % perf_results) 334 335 # Write perf_results to the output json. 336 with open(output_json_file, 'w') as f: 337 f.write(json.dumps(perf_results)) 338 """, args=[trace_json, api.json.output(), renderer]) 339 340 # Sanitize float outputs to 2 precision points. 341 output = dict(step_result.json.output) 342 output['frame_max_us'] = float("%.2f" % output['frame_max_us']) 343 output['frame_min_us'] = float("%.2f" % output['frame_min_us']) 344 output['frame_avg_us'] = float("%.2f" % output['frame_avg_us']) 345 return output 346 347 348def GenTests(api): 349 trace_output = """ 350[{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":452,"dur":2.57,"tid":1,"pid":0},{"ph":"X","name":"void SkCanvas::drawPaint(const SkPaint &)","ts":473,"dur":2.67e+03,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":3.15e+03,"dur":2.25,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::render(SkCanvas *, const SkRect *, RenderFlags) const","ts":3.15e+03,"dur":216,"tid":1,"pid":0},{"ph":"X","name":"void SkCanvas::drawPath(const SkPath &, const SkPaint &)","ts":3.35e+03,"dur":15.1,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":3.37e+03,"dur":1.17,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::render(SkCanvas *, const SkRect *, RenderFlags) const","ts":3.37e+03,"dur":140,"tid":1,"pid":0}] 351""" 352 parse_trace_json = { 353 'frame_avg_us': 179.71, 354 'frame_min_us': 141.17, 355 'frame_max_us': 218.25 356 } 357 358 359 skottie_cpu_buildername = ('Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-' 360 'SkottieWASM') 361 yield ( 362 api.test('skottie_wasm_perf') + 363 api.properties(buildername=skottie_cpu_buildername, 364 repository='https://skia.googlesource.com/skia.git', 365 revision='abc123', 366 path_config='kitchen', 367 trace_test_data=trace_output, 368 swarm_out_dir='[SWARM_OUT_DIR]') + 369 api.step_data('parse lottie1.json trace', 370 api.json.output(parse_trace_json)) + 371 api.step_data('parse lottie2.json trace', 372 api.json.output(parse_trace_json)) + 373 api.step_data('parse lottie3.json trace', 374 api.json.output(parse_trace_json)) 375 ) 376 yield ( 377 api.test('skottie_wasm_perf_trybot') + 378 api.properties(buildername=skottie_cpu_buildername, 379 repository='https://skia.googlesource.com/skia.git', 380 revision='abc123', 381 path_config='kitchen', 382 trace_test_data=trace_output, 383 swarm_out_dir='[SWARM_OUT_DIR]', 384 patch_ref='89/456789/12', 385 patch_repo='https://skia.googlesource.com/skia.git', 386 patch_storage='gerrit', 387 patch_set=7, 388 patch_issue=1234, 389 gerrit_project='skia', 390 gerrit_url='https://skia-review.googlesource.com/') + 391 api.step_data('parse lottie1.json trace', 392 api.json.output(parse_trace_json)) + 393 api.step_data('parse lottie2.json trace', 394 api.json.output(parse_trace_json)) + 395 api.step_data('parse lottie3.json trace', 396 api.json.output(parse_trace_json)) 397 ) 398 399 skottie_gpu_buildername = ('Perf-Debian10-EMCC-NUC7i5BNK-GPU-IntelIris640-' 400 'wasm-Release-All-SkottieWASM') 401 yield ( 402 api.test('skottie_wasm_perf_gpu') + 403 api.properties(buildername=skottie_gpu_buildername, 404 repository='https://skia.googlesource.com/skia.git', 405 revision='abc123', 406 path_config='kitchen', 407 trace_test_data=trace_output, 408 swarm_out_dir='[SWARM_OUT_DIR]') + 409 api.step_data('parse lottie1.json trace', 410 api.json.output(parse_trace_json)) + 411 api.step_data('parse lottie2.json trace', 412 api.json.output(parse_trace_json)) + 413 api.step_data('parse lottie3.json trace', 414 api.json.output(parse_trace_json)) 415 ) 416 417 lottieweb_cpu_buildername = ('Perf-Debian10-none-GCE-CPU-AVX2-x86_64-Release-' 418 'All-LottieWeb') 419 yield ( 420 api.test('lottie_web_perf') + 421 api.properties(buildername=lottieweb_cpu_buildername, 422 repository='https://skia.googlesource.com/skia.git', 423 revision='abc123', 424 path_config='kitchen', 425 trace_test_data=trace_output, 426 swarm_out_dir='[SWARM_OUT_DIR]') + 427 api.step_data('parse lottie1.json trace', 428 api.json.output(parse_trace_json)) + 429 api.step_data('parse lottie2.json trace', 430 api.json.output(parse_trace_json)) + 431 api.step_data('parse lottie3.json trace', 432 api.json.output(parse_trace_json)) 433 ) 434 yield ( 435 api.test('lottie_web_perf_trybot') + 436 api.properties(buildername=lottieweb_cpu_buildername, 437 repository='https://skia.googlesource.com/skia.git', 438 revision='abc123', 439 path_config='kitchen', 440 trace_test_data=trace_output, 441 swarm_out_dir='[SWARM_OUT_DIR]', 442 patch_ref='89/456789/12', 443 patch_repo='https://skia.googlesource.com/skia.git', 444 patch_storage='gerrit', 445 patch_set=7, 446 patch_issue=1234, 447 gerrit_project='skia', 448 gerrit_url='https://skia-review.googlesource.com/') + 449 api.step_data('parse lottie1.json trace', 450 api.json.output(parse_trace_json)) + 451 api.step_data('parse lottie2.json trace', 452 api.json.output(parse_trace_json)) + 453 api.step_data('parse lottie3.json trace', 454 api.json.output(parse_trace_json)) 455 ) 456 457 lottieweb_canvas_cpu_buildername = ( 458 'Perf-Debian10-none-GCE-CPU-AVX2-x86_64-Release-All-LottieWeb_Canvas') 459 yield ( 460 api.test('lottie_web_canvas_perf') + 461 api.properties(buildername=lottieweb_canvas_cpu_buildername, 462 repository='https://skia.googlesource.com/skia.git', 463 revision='abc123', 464 path_config='kitchen', 465 trace_test_data=trace_output, 466 swarm_out_dir='[SWARM_OUT_DIR]') + 467 api.step_data('parse lottie1.json trace', 468 api.json.output(parse_trace_json)) + 469 api.step_data('parse lottie2.json trace', 470 api.json.output(parse_trace_json)) + 471 api.step_data('parse lottie3.json trace', 472 api.json.output(parse_trace_json)) 473 ) 474 yield ( 475 api.test('lottie_web_canvas_perf_trybot') + 476 api.properties(buildername=lottieweb_canvas_cpu_buildername, 477 repository='https://skia.googlesource.com/skia.git', 478 revision='abc123', 479 path_config='kitchen', 480 trace_test_data=trace_output, 481 swarm_out_dir='[SWARM_OUT_DIR]', 482 patch_ref='89/456789/12', 483 patch_repo='https://skia.googlesource.com/skia.git', 484 patch_storage='gerrit', 485 patch_set=7, 486 patch_issue=1234, 487 gerrit_project='skia', 488 gerrit_url='https://skia-review.googlesource.com/') + 489 api.step_data('parse lottie1.json trace', 490 api.json.output(parse_trace_json)) + 491 api.step_data('parse lottie2.json trace', 492 api.json.output(parse_trace_json)) + 493 api.step_data('parse lottie3.json trace', 494 api.json.output(parse_trace_json)) 495 ) 496 497 unrecognized_buildername = ('Perf-Debian10-none-GCE-CPU-AVX2-x86_64-Release-' 498 'All-Unrecognized') 499 yield ( 500 api.test('unrecognized_builder') + 501 api.properties(buildername=unrecognized_buildername, 502 repository='https://skia.googlesource.com/skia.git', 503 revision='abc123', 504 path_config='kitchen', 505 swarm_out_dir='[SWARM_OUT_DIR]') + 506 api.expect_exception('Exception') 507 ) 508