1# Copyright 2019 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5# Recipe which runs DM with trace flag on lottie files and then parses the 6# trace output into output JSON files to ingest to perf.skia.org. 7# Design doc: go/skottie-tracing 8 9 10import calendar 11import json 12import re 13import string 14 15PYTHON_VERSION_COMPATIBILITY = "PY2+3" 16 17DEPS = [ 18 'flavor', 19 'recipe_engine/context', 20 'recipe_engine/file', 21 'recipe_engine/json', 22 'recipe_engine/path', 23 'recipe_engine/step', 24 'recipe_engine/time', 25 'recipe_engine/properties', 26 'recipe_engine/python', 27 'recipe_engine/raw_io', 28 'run', 29 'vars', 30] 31 32SEEK_TRACE_NAME = 'skottie::Animation::seek' 33RENDER_TRACE_NAME = 'skottie::Animation::render' 34EXPECTED_DM_FRAMES = 25 35 36 37def perf_steps(api): 38 """Run DM on lottie files with tracing turned on and then parse the output.""" 39 api.flavor.create_clean_device_dir( 40 api.flavor.device_dirs.dm_dir) 41 42 lottie_files = api.file.listdir( 43 'list lottie files', api.flavor.host_dirs.lotties_dir, 44 test_data=['lottie1.json', 'lottie(test)\'!2.json', 'lottie 3!.json', 45 'LICENSE']) 46 perf_results = {} 47 # Run DM on each lottie file and parse the trace files. 48 for idx, lottie_file in enumerate(lottie_files): 49 lottie_filename = api.path.basename(lottie_file) 50 if not lottie_filename.endswith('.json'): 51 continue 52 53 trace_output_path = api.flavor.device_path_join( 54 api.flavor.device_dirs.dm_dir, '%s.json' % (idx + 1)) 55 # See go/skottie-tracing for how these flags were selected. 56 dm_args = [ 57 'dm', 58 '--resourcePath', api.flavor.device_dirs.resource_dir, 59 '--lotties', api.flavor.device_dirs.lotties_dir, 60 '--src', 'lottie', 61 '--nonativeFonts', 62 '--verbose', 63 '--traceMatch', 'skottie', # recipe can OOM without this. 64 '--trace', trace_output_path, 65 '--match', get_trace_match( 66 lottie_filename, 'Android' in api.properties['buildername']), 67 ] 68 if api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': 69 dm_args.extend(['--config', 'gles', '--nocpu']) 70 elif api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU': 71 dm_args.extend(['--config', '8888', '--nogpu']) 72 api.run(api.flavor.step, 'dm', cmd=dm_args, abort_on_failure=False) 73 74 trace_test_data = api.properties.get('trace_test_data', '{}') 75 trace_file_content = api.flavor.read_file_on_device(trace_output_path) 76 if not trace_file_content and trace_test_data: 77 trace_file_content = trace_test_data 78 79 perf_results[lottie_filename] = { 80 'gles': parse_trace(trace_file_content, lottie_filename, api), 81 } 82 api.flavor.remove_file_on_device(trace_output_path) 83 84 # Construct contents of the output JSON. 85 perf_json = { 86 'gitHash': api.properties['revision'], 87 'swarming_bot_id': api.vars.swarming_bot_id, 88 'swarming_task_id': api.vars.swarming_task_id, 89 'renderer': 'skottie', 90 'key': { 91 'bench_type': 'tracing', 92 'source_type': 'skottie', 93 }, 94 'results': perf_results, 95 } 96 if api.vars.is_trybot: 97 perf_json['issue'] = api.vars.issue 98 perf_json['patchset'] = api.vars.patchset 99 perf_json['patch_storage'] = api.vars.patch_storage 100 # Add tokens from the builder name to the key. 101 reg = re.compile('Perf-(?P<os>[A-Za-z0-9_]+)-' 102 '(?P<compiler>[A-Za-z0-9_]+)-' 103 '(?P<model>[A-Za-z0-9_]+)-' 104 '(?P<cpu_or_gpu>[A-Z]+)-' 105 '(?P<cpu_or_gpu_value>[A-Za-z0-9_]+)-' 106 '(?P<arch>[A-Za-z0-9_]+)-' 107 '(?P<configuration>[A-Za-z0-9_]+)-' 108 'All(-(?P<extra_config>[A-Za-z0-9_]+)|)') 109 m = reg.match(api.properties['buildername']) 110 keys = ['os', 'compiler', 'model', 'cpu_or_gpu', 'cpu_or_gpu_value', 'arch', 111 'configuration', 'extra_config'] 112 for k in keys: 113 perf_json['key'][k] = m.group(k) 114 115 # Create the output JSON file in perf_data_dir for the Upload task to upload. 116 api.file.ensure_directory( 117 'makedirs perf_dir', 118 api.flavor.host_dirs.perf_data_dir) 119 now = api.time.utcnow() 120 ts = int(calendar.timegm(now.utctimetuple())) 121 json_path = api.flavor.host_dirs.perf_data_dir.join( 122 'perf_%s_%d.json' % (api.properties['revision'], ts)) 123 json_contents = json.dumps( 124 perf_json, indent=4, sort_keys=True, separators=(',', ': ')) 125 api.file.write_text('write output JSON', json_path, json_contents) 126 127 128def get_trace_match(lottie_filename, is_android): 129 """Returns the DM regex to match the specified lottie file name.""" 130 trace_match = '^%s$' % lottie_filename 131 if is_android and ' ' not in trace_match: 132 # Punctuation characters confuse DM when shelled out over adb, so escape 133 # them. Do not need to do this when there is a space in the match because 134 # subprocess.list2cmdline automatically adds quotes in that case. 135 for sp_char in string.punctuation: 136 if sp_char == '\\': 137 # No need to escape the escape char. 138 continue 139 trace_match = trace_match.replace(sp_char, '\%s' % sp_char) 140 return trace_match 141 142 143def parse_trace(trace_json, lottie_filename, api): 144 """parse_trace parses the specified trace JSON. 145 146 Parses the trace JSON and calculates the time of a single frame. Frame time is 147 considered the same as seek time + render time. 148 Note: The first seek is ignored because it is a constructor call. 149 150 A dictionary is returned that has the following structure: 151 { 152 'frame_max_us': 100, 153 'frame_min_us': 90, 154 'frame_avg_us': 95, 155 } 156 """ 157 step_result = api.run( 158 api.python.inline, 159 'parse %s trace' % lottie_filename, 160 program=""" 161 import json 162 import sys 163 164 trace_output = sys.argv[1] 165 trace_json = json.loads(trace_output) 166 lottie_filename = sys.argv[2] 167 output_json_file = sys.argv[3] 168 169 perf_results = {} 170 frame_max = 0 171 frame_min = 0 172 frame_cumulative = 0 173 current_frame_duration = 0 174 total_frames = 0 175 frame_start = False 176 for trace in trace_json: 177 if '%s' in trace['name']: 178 if frame_start: 179 raise Exception('We got consecutive Animation::seek without a ' + 180 'render. Something is wrong.') 181 frame_start = True 182 current_frame_duration = trace['dur'] 183 elif '%s' in trace['name']: 184 if not frame_start: 185 raise Exception('We got an Animation::render without a seek first. ' + 186 'Something is wrong.') 187 188 current_frame_duration += trace['dur'] 189 frame_start = False 190 total_frames += 1 191 frame_max = max(frame_max, current_frame_duration) 192 frame_min = (min(frame_min, current_frame_duration) 193 if frame_min else current_frame_duration) 194 frame_cumulative += current_frame_duration 195 196 expected_dm_frames = %d 197 if total_frames != expected_dm_frames: 198 raise Exception( 199 'Got ' + str(total_frames) + ' frames instead of ' + 200 str(expected_dm_frames)) 201 perf_results['frame_max_us'] = frame_max 202 perf_results['frame_min_us'] = frame_min 203 perf_results['frame_avg_us'] = frame_cumulative/total_frames 204 205 # Write perf_results to the output json. 206 with open(output_json_file, 'w') as f: 207 f.write(json.dumps(perf_results)) 208 """ % (SEEK_TRACE_NAME, RENDER_TRACE_NAME, EXPECTED_DM_FRAMES), 209 args=[trace_json, lottie_filename, api.json.output()]) 210 211 # Sanitize float outputs to 2 precision points. 212 output = dict(step_result.json.output) 213 output['frame_max_us'] = float("%.2f" % output['frame_max_us']) 214 output['frame_min_us'] = float("%.2f" % output['frame_min_us']) 215 output['frame_avg_us'] = float("%.2f" % output['frame_avg_us']) 216 return output 217 218 219def RunSteps(api): 220 api.vars.setup() 221 api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) 222 api.flavor.setup('dm') 223 224 with api.context(): 225 try: 226 api.flavor.install(resources=True, lotties=True) 227 perf_steps(api) 228 finally: 229 api.flavor.cleanup_steps() 230 api.run.check_failure() 231 232 233def GenTests(api): 234 trace_output = """ 235[{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":452,"dur":2.57,"tid":1,"pid":0},{"ph":"X","name":"void SkCanvas::drawPaint(const SkPaint &)","ts":473,"dur":2.67e+03,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":3.15e+03,"dur":2.25,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::render(SkCanvas *, const SkRect *, RenderFlags) const","ts":3.15e+03,"dur":216,"tid":1,"pid":0},{"ph":"X","name":"void SkCanvas::drawPath(const SkPath &, const SkPaint &)","ts":3.35e+03,"dur":15.1,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":3.37e+03,"dur":1.17,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::render(SkCanvas *, const SkRect *, RenderFlags) const","ts":3.37e+03,"dur":140,"tid":1,"pid":0}] 236""" 237 dm_json_test_data = """ 238{ 239 "gitHash": "bac53f089dbc473862bc5a2e328ba7600e0ed9c4", 240 "swarming_bot_id": "skia-rpi-094", 241 "swarming_task_id": "438f11c0e19eab11", 242 "key": { 243 "arch": "arm", 244 "compiler": "Clang", 245 "cpu_or_gpu": "GPU", 246 "cpu_or_gpu_value": "Mali400MP2", 247 "extra_config": "Android", 248 "model": "AndroidOne", 249 "os": "Android" 250 }, 251 "results": { 252 } 253} 254""" 255 parse_trace_json = { 256 'frame_avg_us': 179.71, 257 'frame_min_us': 141.17, 258 'frame_max_us': 218.25 259 } 260 android_buildername = ('Perf-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-' 261 'Release-All-Android_SkottieTracing') 262 gpu_buildername = ('Perf-Debian10-Clang-NUC7i5BNK-GPU-IntelIris640-x86_64-' 263 'Release-All-SkottieTracing') 264 cpu_buildername = ('Perf-Debian10-Clang-GCE-CPU-AVX2-x86_64-Release-All-' 265 'SkottieTracing') 266 yield ( 267 api.test(android_buildername) + 268 api.properties(buildername=android_buildername, 269 repository='https://skia.googlesource.com/skia.git', 270 revision='abc123', 271 task_id='abc123', 272 trace_test_data=trace_output, 273 dm_json_test_data=dm_json_test_data, 274 path_config='kitchen', 275 swarm_out_dir='[SWARM_OUT_DIR]') + 276 api.step_data('parse lottie(test)\'!2.json trace', 277 api.json.output(parse_trace_json)) + 278 api.step_data('parse lottie1.json trace', 279 api.json.output(parse_trace_json)) + 280 api.step_data('parse lottie 3!.json trace', 281 api.json.output(parse_trace_json)) 282 ) 283 yield ( 284 api.test(gpu_buildername) + 285 api.properties(buildername=gpu_buildername, 286 repository='https://skia.googlesource.com/skia.git', 287 revision='abc123', 288 task_id='abc123', 289 trace_test_data=trace_output, 290 dm_json_test_data=dm_json_test_data, 291 path_config='kitchen', 292 swarm_out_dir='[SWARM_OUT_DIR]') + 293 api.step_data('parse lottie(test)\'!2.json trace', 294 api.json.output(parse_trace_json)) + 295 api.step_data('parse lottie1.json trace', 296 api.json.output(parse_trace_json)) + 297 api.step_data('parse lottie 3!.json trace', 298 api.json.output(parse_trace_json)) 299 ) 300 yield ( 301 api.test(cpu_buildername) + 302 api.properties(buildername=cpu_buildername, 303 repository='https://skia.googlesource.com/skia.git', 304 revision='abc123', 305 task_id='abc123', 306 trace_test_data=trace_output, 307 dm_json_test_data=dm_json_test_data, 308 path_config='kitchen', 309 swarm_out_dir='[SWARM_OUT_DIR]') + 310 api.step_data('parse lottie(test)\'!2.json trace', 311 api.json.output(parse_trace_json)) + 312 api.step_data('parse lottie1.json trace', 313 api.json.output(parse_trace_json)) + 314 api.step_data('parse lottie 3!.json trace', 315 api.json.output(parse_trace_json)) 316 ) 317 yield ( 318 api.test('skottietracing_parse_trace_error') + 319 api.properties(buildername=android_buildername, 320 repository='https://skia.googlesource.com/skia.git', 321 revision='abc123', 322 task_id='abc123', 323 trace_test_data=trace_output, 324 dm_json_test_data=dm_json_test_data, 325 path_config='kitchen', 326 swarm_out_dir='[SWARM_OUT_DIR]') + 327 api.step_data('parse lottie 3!.json trace', 328 api.json.output(parse_trace_json), retcode=1) 329 ) 330 yield ( 331 api.test('skottietracing_trybot') + 332 api.properties(buildername=android_buildername, 333 repository='https://skia.googlesource.com/skia.git', 334 revision='abc123', 335 task_id='abc123', 336 trace_test_data=trace_output, 337 dm_json_test_data=dm_json_test_data, 338 path_config='kitchen', 339 swarm_out_dir='[SWARM_OUT_DIR]', 340 patch_ref='89/456789/12', 341 patch_repo='https://skia.googlesource.com/skia.git', 342 patch_storage='gerrit', 343 patch_set=7, 344 patch_issue=1234, 345 gerrit_project='skia', 346 gerrit_url='https://skia-review.googlesource.com/') + 347 api.step_data('parse lottie(test)\'!2.json trace', 348 api.json.output(parse_trace_json)) + 349 api.step_data('parse lottie1.json trace', 350 api.json.output(parse_trace_json)) + 351 api.step_data('parse lottie 3!.json trace', 352 api.json.output(parse_trace_json)) 353 ) 354