1# Copyright 2019 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5# Recipe which runs DM with trace flag on lottie files and then parses the 6# trace output into output JSON files to ingest to perf.skia.org. 7# Design doc: go/skottie-tracing 8 9 10import calendar 11import re 12import string 13 14 15DEPS = [ 16 'flavor', 17 'recipe_engine/context', 18 'recipe_engine/file', 19 'recipe_engine/json', 20 'recipe_engine/path', 21 'recipe_engine/step', 22 'recipe_engine/time', 23 'recipe_engine/properties', 24 'recipe_engine/python', 25 'recipe_engine/raw_io', 26 'run', 27 'vars', 28] 29 30SEEK_TRACE_NAME = 'skottie::Animation::seek' 31RENDER_TRACE_NAME = 'skottie::Animation::render' 32EXPECTED_DM_FRAMES = 25 33 34 35def perf_steps(api): 36 """Run DM on lottie files with tracing turned on and then parse the output.""" 37 api.flavor.create_clean_device_dir( 38 api.flavor.device_dirs.dm_dir) 39 40 lottie_files = api.file.listdir( 41 'list lottie files', api.flavor.host_dirs.lotties_dir, 42 test_data=['lottie1.json', 'lottie(test)\'!2.json', 'lottie 3!.json', 43 'LICENSE']) 44 perf_results = {} 45 push_dm = True 46 # Run DM on each lottie file and parse the trace files. 47 for idx, lottie_file in enumerate(lottie_files): 48 lottie_filename = api.path.basename(lottie_file) 49 if not lottie_filename.endswith('.json'): 50 continue 51 52 trace_output_path = api.flavor.device_path_join( 53 api.flavor.device_dirs.dm_dir, '%s.json' % (idx + 1)) 54 # See go/skottie-tracing for how these flags were selected. 55 dm_args = [ 56 'dm', 57 '--resourcePath', api.flavor.device_dirs.resource_dir, 58 '--lotties', api.flavor.device_dirs.lotties_dir, 59 '--src', 'lottie', 60 '--nonativeFonts', 61 '--verbose', 62 '--traceMatch', 'skottie', # recipe can OOM without this. 63 '--trace', trace_output_path, 64 '--match', get_trace_match( 65 lottie_filename, 'Android' in api.properties['buildername']), 66 ] 67 if api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': 68 dm_args.extend(['--config', 'gles', '--nocpu']) 69 elif api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU': 70 dm_args.extend(['--config', '8888', '--nogpu']) 71 api.run(api.flavor.step, 'dm', cmd=dm_args, abort_on_failure=False, 72 skip_binary_push=not push_dm) 73 # We already pushed the binary once. No need to waste time by pushing 74 # the same binary for future runs. 75 push_dm = False 76 77 trace_test_data = api.properties.get('trace_test_data', '{}') 78 trace_file_content = api.flavor.read_file_on_device(trace_output_path) 79 if not trace_file_content and trace_test_data: 80 trace_file_content = trace_test_data 81 82 perf_results[lottie_filename] = { 83 'gles': parse_trace(trace_file_content, lottie_filename, api), 84 } 85 api.flavor.remove_file_on_device(trace_output_path) 86 87 # Construct contents of the output JSON. 88 perf_json = { 89 'gitHash': api.properties['revision'], 90 'swarming_bot_id': api.vars.swarming_bot_id, 91 'swarming_task_id': api.vars.swarming_task_id, 92 'renderer': 'skottie', 93 'key': { 94 'bench_type': 'tracing', 95 'source_type': 'skottie', 96 }, 97 'results': perf_results, 98 } 99 if api.vars.is_trybot: 100 perf_json['issue'] = api.vars.issue 101 perf_json['patchset'] = api.vars.patchset 102 perf_json['patch_storage'] = api.vars.patch_storage 103 # Add tokens from the builder name to the key. 104 reg = re.compile('Perf-(?P<os>[A-Za-z0-9_]+)-' 105 '(?P<compiler>[A-Za-z0-9_]+)-' 106 '(?P<model>[A-Za-z0-9_]+)-' 107 '(?P<cpu_or_gpu>[A-Z]+)-' 108 '(?P<cpu_or_gpu_value>[A-Za-z0-9_]+)-' 109 '(?P<arch>[A-Za-z0-9_]+)-' 110 '(?P<configuration>[A-Za-z0-9_]+)-' 111 'All(-(?P<extra_config>[A-Za-z0-9_]+)|)') 112 m = reg.match(api.properties['buildername']) 113 keys = ['os', 'compiler', 'model', 'cpu_or_gpu', 'cpu_or_gpu_value', 'arch', 114 'configuration', 'extra_config'] 115 for k in keys: 116 perf_json['key'][k] = m.group(k) 117 118 # Create the output JSON file in perf_data_dir for the Upload task to upload. 119 api.file.ensure_directory( 120 'makedirs perf_dir', 121 api.flavor.host_dirs.perf_data_dir) 122 now = api.time.utcnow() 123 ts = int(calendar.timegm(now.utctimetuple())) 124 json_path = api.flavor.host_dirs.perf_data_dir.join( 125 'perf_%s_%d.json' % (api.properties['revision'], ts)) 126 api.run( 127 api.python.inline, 128 'write output JSON', 129 program="""import json 130with open('%s', 'w') as outfile: 131 json.dump(obj=%s, fp=outfile, indent=4) 132 """ % (json_path, perf_json)) 133 134 135def get_trace_match(lottie_filename, is_android): 136 """Returns the DM regex to match the specified lottie file name.""" 137 trace_match = '^%s$' % lottie_filename 138 if is_android and ' ' not in trace_match: 139 # Punctuation characters confuse DM when shelled out over adb, so escape 140 # them. Do not need to do this when there is a space in the match because 141 # subprocess.list2cmdline automatically adds quotes in that case. 142 for sp_char in string.punctuation: 143 if sp_char == '\\': 144 # No need to escape the escape char. 145 continue 146 trace_match = trace_match.replace(sp_char, '\%s' % sp_char) 147 return trace_match 148 149 150def parse_trace(trace_json, lottie_filename, api): 151 """parse_trace parses the specified trace JSON. 152 153 Parses the trace JSON and calculates the time of a single frame. Frame time is 154 considered the same as seek time + render time. 155 Note: The first seek is ignored because it is a constructor call. 156 157 A dictionary is returned that has the following structure: 158 { 159 'frame_max_us': 100, 160 'frame_min_us': 90, 161 'frame_avg_us': 95, 162 } 163 """ 164 step_result = api.run( 165 api.python.inline, 166 'parse %s trace' % lottie_filename, 167 program=""" 168 import json 169 import sys 170 171 trace_output = sys.argv[1] 172 trace_json = json.loads(trace_output) 173 lottie_filename = sys.argv[2] 174 output_json_file = sys.argv[3] 175 176 perf_results = {} 177 frame_max = 0 178 frame_min = 0 179 frame_cumulative = 0 180 current_frame_duration = 0 181 total_frames = 0 182 frame_start = False 183 skipped_first_seek = False # Skip the first seek constructor call. 184 for trace in trace_json: 185 if '%s' in trace['name']: 186 if not skipped_first_seek: 187 skipped_first_seek = True 188 continue 189 if frame_start: 190 raise Exception('We got consecutive Animation::seek without a ' + 191 'render. Something is wrong.') 192 frame_start = True 193 current_frame_duration = trace['dur'] 194 elif '%s' in trace['name']: 195 if not frame_start: 196 raise Exception('We got an Animation::render without a seek first. ' + 197 'Something is wrong.') 198 199 current_frame_duration += trace['dur'] 200 frame_start = False 201 total_frames += 1 202 frame_max = max(frame_max, current_frame_duration) 203 frame_min = (min(frame_min, current_frame_duration) 204 if frame_min else current_frame_duration) 205 frame_cumulative += current_frame_duration 206 207 expected_dm_frames = %d 208 if total_frames != expected_dm_frames: 209 raise Exception( 210 'Got ' + str(total_frames) + ' frames instead of ' + 211 str(expected_dm_frames)) 212 perf_results['frame_max_us'] = frame_max 213 perf_results['frame_min_us'] = frame_min 214 perf_results['frame_avg_us'] = frame_cumulative/total_frames 215 216 # Write perf_results to the output json. 217 with open(output_json_file, 'w') as f: 218 f.write(json.dumps(perf_results)) 219 """ % (SEEK_TRACE_NAME, RENDER_TRACE_NAME, EXPECTED_DM_FRAMES), 220 args=[trace_json, lottie_filename, api.json.output()]) 221 222 # Sanitize float outputs to 2 precision points. 223 output = dict(step_result.json.output) 224 output['frame_max_us'] = float("%.2f" % output['frame_max_us']) 225 output['frame_min_us'] = float("%.2f" % output['frame_min_us']) 226 output['frame_avg_us'] = float("%.2f" % output['frame_avg_us']) 227 return output 228 229 230def RunSteps(api): 231 api.vars.setup() 232 api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) 233 api.flavor.setup() 234 235 with api.context(): 236 try: 237 api.flavor.install(resources=True, lotties=True) 238 perf_steps(api) 239 finally: 240 api.flavor.cleanup_steps() 241 api.run.check_failure() 242 243 244def GenTests(api): 245 trace_output = """ 246[{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":452,"dur":2.57,"tid":1,"pid":0},{"ph":"X","name":"void SkCanvas::drawPaint(const SkPaint &)","ts":473,"dur":2.67e+03,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":3.15e+03,"dur":2.25,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::render(SkCanvas *, const SkRect *, RenderFlags) const","ts":3.15e+03,"dur":216,"tid":1,"pid":0},{"ph":"X","name":"void SkCanvas::drawPath(const SkPath &, const SkPaint &)","ts":3.35e+03,"dur":15.1,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":3.37e+03,"dur":1.17,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::render(SkCanvas *, const SkRect *, RenderFlags) const","ts":3.37e+03,"dur":140,"tid":1,"pid":0}] 247""" 248 dm_json_test_data = """ 249{ 250 "gitHash": "bac53f089dbc473862bc5a2e328ba7600e0ed9c4", 251 "swarming_bot_id": "skia-rpi-094", 252 "swarming_task_id": "438f11c0e19eab11", 253 "key": { 254 "arch": "arm", 255 "compiler": "Clang", 256 "cpu_or_gpu": "GPU", 257 "cpu_or_gpu_value": "Mali400MP2", 258 "extra_config": "Android", 259 "model": "AndroidOne", 260 "os": "Android" 261 }, 262 "results": { 263 } 264} 265""" 266 parse_trace_json = { 267 'frame_avg_us': 179.71, 268 'frame_min_us': 141.17, 269 'frame_max_us': 218.25 270 } 271 android_buildername = ('Perf-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-' 272 'Release-All-Android_SkottieTracing') 273 gpu_buildername = ('Perf-Debian9-Clang-NUC7i5BNK-GPU-IntelIris640-x86_64-' 274 'Release-All-SkottieTracing') 275 cpu_buildername = ('Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-' 276 'SkottieTracing') 277 yield ( 278 api.test(android_buildername) + 279 api.properties(buildername=android_buildername, 280 repository='https://skia.googlesource.com/skia.git', 281 revision='abc123', 282 task_id='abc123', 283 trace_test_data=trace_output, 284 dm_json_test_data=dm_json_test_data, 285 path_config='kitchen', 286 swarm_out_dir='[SWARM_OUT_DIR]') + 287 api.step_data('parse lottie(test)\'!2.json trace', 288 api.json.output(parse_trace_json)) + 289 api.step_data('parse lottie1.json trace', 290 api.json.output(parse_trace_json)) + 291 api.step_data('parse lottie 3!.json trace', 292 api.json.output(parse_trace_json)) 293 ) 294 yield ( 295 api.test(gpu_buildername) + 296 api.properties(buildername=gpu_buildername, 297 repository='https://skia.googlesource.com/skia.git', 298 revision='abc123', 299 task_id='abc123', 300 trace_test_data=trace_output, 301 dm_json_test_data=dm_json_test_data, 302 path_config='kitchen', 303 swarm_out_dir='[SWARM_OUT_DIR]') + 304 api.step_data('parse lottie(test)\'!2.json trace', 305 api.json.output(parse_trace_json)) + 306 api.step_data('parse lottie1.json trace', 307 api.json.output(parse_trace_json)) + 308 api.step_data('parse lottie 3!.json trace', 309 api.json.output(parse_trace_json)) 310 ) 311 yield ( 312 api.test(cpu_buildername) + 313 api.properties(buildername=cpu_buildername, 314 repository='https://skia.googlesource.com/skia.git', 315 revision='abc123', 316 task_id='abc123', 317 trace_test_data=trace_output, 318 dm_json_test_data=dm_json_test_data, 319 path_config='kitchen', 320 swarm_out_dir='[SWARM_OUT_DIR]') + 321 api.step_data('parse lottie(test)\'!2.json trace', 322 api.json.output(parse_trace_json)) + 323 api.step_data('parse lottie1.json trace', 324 api.json.output(parse_trace_json)) + 325 api.step_data('parse lottie 3!.json trace', 326 api.json.output(parse_trace_json)) 327 ) 328 yield ( 329 api.test('skottietracing_parse_trace_error') + 330 api.properties(buildername=android_buildername, 331 repository='https://skia.googlesource.com/skia.git', 332 revision='abc123', 333 task_id='abc123', 334 trace_test_data=trace_output, 335 dm_json_test_data=dm_json_test_data, 336 path_config='kitchen', 337 swarm_out_dir='[SWARM_OUT_DIR]') + 338 api.step_data('parse lottie 3!.json trace', 339 api.json.output(parse_trace_json), retcode=1) 340 ) 341 yield ( 342 api.test('skottietracing_trybot') + 343 api.properties(buildername=android_buildername, 344 repository='https://skia.googlesource.com/skia.git', 345 revision='abc123', 346 task_id='abc123', 347 trace_test_data=trace_output, 348 dm_json_test_data=dm_json_test_data, 349 path_config='kitchen', 350 swarm_out_dir='[SWARM_OUT_DIR]', 351 patch_ref='89/456789/12', 352 patch_repo='https://skia.googlesource.com/skia.git', 353 patch_storage='gerrit', 354 patch_set=7, 355 patch_issue=1234, 356 gerrit_project='skia', 357 gerrit_url='https://skia-review.googlesource.com/') + 358 api.step_data('parse lottie(test)\'!2.json trace', 359 api.json.output(parse_trace_json)) + 360 api.step_data('parse lottie1.json trace', 361 api.json.output(parse_trace_json)) + 362 api.step_data('parse lottie 3!.json trace', 363 api.json.output(parse_trace_json)) 364 ) 365