1# Copyright 2019 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5# Recipe which runs DM with trace flag on lottie files and then parses the 6# trace output into output JSON files to ingest to perf.skia.org. 7# Design doc: go/skottie-tracing 8 9 10import calendar 11import re 12import string 13 14 15DEPS = [ 16 'flavor', 17 'recipe_engine/context', 18 'recipe_engine/file', 19 'recipe_engine/json', 20 'recipe_engine/path', 21 'recipe_engine/step', 22 'recipe_engine/time', 23 'recipe_engine/properties', 24 'recipe_engine/python', 25 'recipe_engine/raw_io', 26 'run', 27 'vars', 28] 29 30SEEK_TRACE_NAME = 'skottie::Animation::seek' 31RENDER_TRACE_NAME = 'skottie::Animation::render' 32EXPECTED_DM_FRAMES = 25 33 34 35def perf_steps(api): 36 """Run DM on lottie files with tracing turned on and then parse the output.""" 37 api.flavor.create_clean_device_dir( 38 api.flavor.device_dirs.dm_dir) 39 40 lottie_files = api.file.listdir( 41 'list lottie files', api.flavor.host_dirs.lotties_dir, 42 test_data=['lottie1.json', 'lottie(test)\'!2.json', 'lottie 3!.json', 43 'LICENSE']) 44 perf_results = {} 45 # Run DM on each lottie file and parse the trace files. 46 for idx, lottie_file in enumerate(lottie_files): 47 lottie_filename = api.path.basename(lottie_file) 48 if not lottie_filename.endswith('.json'): 49 continue 50 51 trace_output_path = api.flavor.device_path_join( 52 api.flavor.device_dirs.dm_dir, '%s.json' % (idx + 1)) 53 # See go/skottie-tracing for how these flags were selected. 54 dm_args = [ 55 'dm', 56 '--resourcePath', api.flavor.device_dirs.resource_dir, 57 '--lotties', api.flavor.device_dirs.lotties_dir, 58 '--src', 'lottie', 59 '--nonativeFonts', 60 '--verbose', 61 '--traceMatch', 'skottie', # recipe can OOM without this. 62 '--trace', trace_output_path, 63 '--match', get_trace_match( 64 lottie_filename, 'Android' in api.properties['buildername']), 65 ] 66 if api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': 67 dm_args.extend(['--config', 'gles', '--nocpu']) 68 elif api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU': 69 dm_args.extend(['--config', '8888', '--nogpu']) 70 api.run(api.flavor.step, 'dm', cmd=dm_args, abort_on_failure=False) 71 72 trace_test_data = api.properties.get('trace_test_data', '{}') 73 trace_file_content = api.flavor.read_file_on_device(trace_output_path) 74 if not trace_file_content and trace_test_data: 75 trace_file_content = trace_test_data 76 77 perf_results[lottie_filename] = { 78 'gles': parse_trace(trace_file_content, lottie_filename, api), 79 } 80 api.flavor.remove_file_on_device(trace_output_path) 81 82 # Construct contents of the output JSON. 83 perf_json = { 84 'gitHash': api.properties['revision'], 85 'swarming_bot_id': api.vars.swarming_bot_id, 86 'swarming_task_id': api.vars.swarming_task_id, 87 'renderer': 'skottie', 88 'key': { 89 'bench_type': 'tracing', 90 'source_type': 'skottie', 91 }, 92 'results': perf_results, 93 } 94 if api.vars.is_trybot: 95 perf_json['issue'] = api.vars.issue 96 perf_json['patchset'] = api.vars.patchset 97 perf_json['patch_storage'] = api.vars.patch_storage 98 # Add tokens from the builder name to the key. 99 reg = re.compile('Perf-(?P<os>[A-Za-z0-9_]+)-' 100 '(?P<compiler>[A-Za-z0-9_]+)-' 101 '(?P<model>[A-Za-z0-9_]+)-' 102 '(?P<cpu_or_gpu>[A-Z]+)-' 103 '(?P<cpu_or_gpu_value>[A-Za-z0-9_]+)-' 104 '(?P<arch>[A-Za-z0-9_]+)-' 105 '(?P<configuration>[A-Za-z0-9_]+)-' 106 'All(-(?P<extra_config>[A-Za-z0-9_]+)|)') 107 m = reg.match(api.properties['buildername']) 108 keys = ['os', 'compiler', 'model', 'cpu_or_gpu', 'cpu_or_gpu_value', 'arch', 109 'configuration', 'extra_config'] 110 for k in keys: 111 perf_json['key'][k] = m.group(k) 112 113 # Create the output JSON file in perf_data_dir for the Upload task to upload. 114 api.file.ensure_directory( 115 'makedirs perf_dir', 116 api.flavor.host_dirs.perf_data_dir) 117 now = api.time.utcnow() 118 ts = int(calendar.timegm(now.utctimetuple())) 119 json_path = api.flavor.host_dirs.perf_data_dir.join( 120 'perf_%s_%d.json' % (api.properties['revision'], ts)) 121 api.run( 122 api.python.inline, 123 'write output JSON', 124 program="""import json 125with open('%s', 'w') as outfile: 126 json.dump(obj=%s, fp=outfile, indent=4) 127 """ % (json_path, perf_json)) 128 129 130def get_trace_match(lottie_filename, is_android): 131 """Returns the DM regex to match the specified lottie file name.""" 132 trace_match = '^%s$' % lottie_filename 133 if is_android and ' ' not in trace_match: 134 # Punctuation characters confuse DM when shelled out over adb, so escape 135 # them. Do not need to do this when there is a space in the match because 136 # subprocess.list2cmdline automatically adds quotes in that case. 137 for sp_char in string.punctuation: 138 if sp_char == '\\': 139 # No need to escape the escape char. 140 continue 141 trace_match = trace_match.replace(sp_char, '\%s' % sp_char) 142 return trace_match 143 144 145def parse_trace(trace_json, lottie_filename, api): 146 """parse_trace parses the specified trace JSON. 147 148 Parses the trace JSON and calculates the time of a single frame. Frame time is 149 considered the same as seek time + render time. 150 Note: The first seek is ignored because it is a constructor call. 151 152 A dictionary is returned that has the following structure: 153 { 154 'frame_max_us': 100, 155 'frame_min_us': 90, 156 'frame_avg_us': 95, 157 } 158 """ 159 step_result = api.run( 160 api.python.inline, 161 'parse %s trace' % lottie_filename, 162 program=""" 163 import json 164 import sys 165 166 trace_output = sys.argv[1] 167 trace_json = json.loads(trace_output) 168 lottie_filename = sys.argv[2] 169 output_json_file = sys.argv[3] 170 171 perf_results = {} 172 frame_max = 0 173 frame_min = 0 174 frame_cumulative = 0 175 current_frame_duration = 0 176 total_frames = 0 177 frame_start = False 178 for trace in trace_json: 179 if '%s' in trace['name']: 180 if frame_start: 181 raise Exception('We got consecutive Animation::seek without a ' + 182 'render. Something is wrong.') 183 frame_start = True 184 current_frame_duration = trace['dur'] 185 elif '%s' in trace['name']: 186 if not frame_start: 187 raise Exception('We got an Animation::render without a seek first. ' + 188 'Something is wrong.') 189 190 current_frame_duration += trace['dur'] 191 frame_start = False 192 total_frames += 1 193 frame_max = max(frame_max, current_frame_duration) 194 frame_min = (min(frame_min, current_frame_duration) 195 if frame_min else current_frame_duration) 196 frame_cumulative += current_frame_duration 197 198 expected_dm_frames = %d 199 if total_frames != expected_dm_frames: 200 raise Exception( 201 'Got ' + str(total_frames) + ' frames instead of ' + 202 str(expected_dm_frames)) 203 perf_results['frame_max_us'] = frame_max 204 perf_results['frame_min_us'] = frame_min 205 perf_results['frame_avg_us'] = frame_cumulative/total_frames 206 207 # Write perf_results to the output json. 208 with open(output_json_file, 'w') as f: 209 f.write(json.dumps(perf_results)) 210 """ % (SEEK_TRACE_NAME, RENDER_TRACE_NAME, EXPECTED_DM_FRAMES), 211 args=[trace_json, lottie_filename, api.json.output()]) 212 213 # Sanitize float outputs to 2 precision points. 214 output = dict(step_result.json.output) 215 output['frame_max_us'] = float("%.2f" % output['frame_max_us']) 216 output['frame_min_us'] = float("%.2f" % output['frame_min_us']) 217 output['frame_avg_us'] = float("%.2f" % output['frame_avg_us']) 218 return output 219 220 221def RunSteps(api): 222 api.vars.setup() 223 api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) 224 api.flavor.setup('dm') 225 226 with api.context(): 227 try: 228 api.flavor.install(resources=True, lotties=True) 229 perf_steps(api) 230 finally: 231 api.flavor.cleanup_steps() 232 api.run.check_failure() 233 234 235def GenTests(api): 236 trace_output = """ 237[{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":452,"dur":2.57,"tid":1,"pid":0},{"ph":"X","name":"void SkCanvas::drawPaint(const SkPaint &)","ts":473,"dur":2.67e+03,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":3.15e+03,"dur":2.25,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::render(SkCanvas *, const SkRect *, RenderFlags) const","ts":3.15e+03,"dur":216,"tid":1,"pid":0},{"ph":"X","name":"void SkCanvas::drawPath(const SkPath &, const SkPaint &)","ts":3.35e+03,"dur":15.1,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::seek(SkScalar)","ts":3.37e+03,"dur":1.17,"tid":1,"pid":0},{"ph":"X","name":"void skottie::Animation::render(SkCanvas *, const SkRect *, RenderFlags) const","ts":3.37e+03,"dur":140,"tid":1,"pid":0}] 238""" 239 dm_json_test_data = """ 240{ 241 "gitHash": "bac53f089dbc473862bc5a2e328ba7600e0ed9c4", 242 "swarming_bot_id": "skia-rpi-094", 243 "swarming_task_id": "438f11c0e19eab11", 244 "key": { 245 "arch": "arm", 246 "compiler": "Clang", 247 "cpu_or_gpu": "GPU", 248 "cpu_or_gpu_value": "Mali400MP2", 249 "extra_config": "Android", 250 "model": "AndroidOne", 251 "os": "Android" 252 }, 253 "results": { 254 } 255} 256""" 257 parse_trace_json = { 258 'frame_avg_us': 179.71, 259 'frame_min_us': 141.17, 260 'frame_max_us': 218.25 261 } 262 android_buildername = ('Perf-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-' 263 'Release-All-Android_SkottieTracing') 264 gpu_buildername = ('Perf-Debian10-Clang-NUC7i5BNK-GPU-IntelIris640-x86_64-' 265 'Release-All-SkottieTracing') 266 cpu_buildername = ('Perf-Debian10-Clang-GCE-CPU-AVX2-x86_64-Release-All-' 267 'SkottieTracing') 268 yield ( 269 api.test(android_buildername) + 270 api.properties(buildername=android_buildername, 271 repository='https://skia.googlesource.com/skia.git', 272 revision='abc123', 273 task_id='abc123', 274 trace_test_data=trace_output, 275 dm_json_test_data=dm_json_test_data, 276 path_config='kitchen', 277 swarm_out_dir='[SWARM_OUT_DIR]') + 278 api.step_data('parse lottie(test)\'!2.json trace', 279 api.json.output(parse_trace_json)) + 280 api.step_data('parse lottie1.json trace', 281 api.json.output(parse_trace_json)) + 282 api.step_data('parse lottie 3!.json trace', 283 api.json.output(parse_trace_json)) 284 ) 285 yield ( 286 api.test(gpu_buildername) + 287 api.properties(buildername=gpu_buildername, 288 repository='https://skia.googlesource.com/skia.git', 289 revision='abc123', 290 task_id='abc123', 291 trace_test_data=trace_output, 292 dm_json_test_data=dm_json_test_data, 293 path_config='kitchen', 294 swarm_out_dir='[SWARM_OUT_DIR]') + 295 api.step_data('parse lottie(test)\'!2.json trace', 296 api.json.output(parse_trace_json)) + 297 api.step_data('parse lottie1.json trace', 298 api.json.output(parse_trace_json)) + 299 api.step_data('parse lottie 3!.json trace', 300 api.json.output(parse_trace_json)) 301 ) 302 yield ( 303 api.test(cpu_buildername) + 304 api.properties(buildername=cpu_buildername, 305 repository='https://skia.googlesource.com/skia.git', 306 revision='abc123', 307 task_id='abc123', 308 trace_test_data=trace_output, 309 dm_json_test_data=dm_json_test_data, 310 path_config='kitchen', 311 swarm_out_dir='[SWARM_OUT_DIR]') + 312 api.step_data('parse lottie(test)\'!2.json trace', 313 api.json.output(parse_trace_json)) + 314 api.step_data('parse lottie1.json trace', 315 api.json.output(parse_trace_json)) + 316 api.step_data('parse lottie 3!.json trace', 317 api.json.output(parse_trace_json)) 318 ) 319 yield ( 320 api.test('skottietracing_parse_trace_error') + 321 api.properties(buildername=android_buildername, 322 repository='https://skia.googlesource.com/skia.git', 323 revision='abc123', 324 task_id='abc123', 325 trace_test_data=trace_output, 326 dm_json_test_data=dm_json_test_data, 327 path_config='kitchen', 328 swarm_out_dir='[SWARM_OUT_DIR]') + 329 api.step_data('parse lottie 3!.json trace', 330 api.json.output(parse_trace_json), retcode=1) 331 ) 332 yield ( 333 api.test('skottietracing_trybot') + 334 api.properties(buildername=android_buildername, 335 repository='https://skia.googlesource.com/skia.git', 336 revision='abc123', 337 task_id='abc123', 338 trace_test_data=trace_output, 339 dm_json_test_data=dm_json_test_data, 340 path_config='kitchen', 341 swarm_out_dir='[SWARM_OUT_DIR]', 342 patch_ref='89/456789/12', 343 patch_repo='https://skia.googlesource.com/skia.git', 344 patch_storage='gerrit', 345 patch_set=7, 346 patch_issue=1234, 347 gerrit_project='skia', 348 gerrit_url='https://skia-review.googlesource.com/') + 349 api.step_data('parse lottie(test)\'!2.json trace', 350 api.json.output(parse_trace_json)) + 351 api.step_data('parse lottie1.json trace', 352 api.json.output(parse_trace_json)) + 353 api.step_data('parse lottie 3!.json trace', 354 api.json.output(parse_trace_json)) 355 ) 356