1// Copyright 2020 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5// This executable is meant to be a general way to gather perf data using puppeteer. The logic 6// (e.g. what bench to run, how to process that particular output) is selected using the ExtraConfig 7// part of the task name. 8package main 9 10import ( 11 "context" 12 "encoding/json" 13 "flag" 14 "fmt" 15 "io/ioutil" 16 "math" 17 "os" 18 "path/filepath" 19 "sort" 20 "strings" 21 22 "go.skia.org/infra/go/exec" 23 "go.skia.org/infra/go/skerr" 24 "go.skia.org/infra/go/sklog" 25 "go.skia.org/infra/go/util" 26 "go.skia.org/infra/task_driver/go/lib/os_steps" 27 "go.skia.org/infra/task_driver/go/td" 28) 29 30const perfKeyWebGLVersion = "webgl_version" 31 32func main() { 33 var ( 34 // Required properties for this task. 35 projectID = flag.String("project_id", "", "ID of the Google Cloud project.") 36 taskName = flag.String("task_name", "", "Name of the task.") 37 benchmarkPath = flag.String("benchmark_path", "", "Path to location of the benchmark files (e.g. //tools/perf-puppeteer).") 38 outputPath = flag.String("output_path", "", "Perf Output will be produced here") 39 gitHash = flag.String("git_hash", "", "Git hash this data corresponds to") 40 taskID = flag.String("task_id", "", "task id this data was generated on") 41 nodeBinPath = flag.String("node_bin_path", "", "Path to the node bin directory (should have npm also). This directory *must* be on the PATH when this executable is called, otherwise, the wrong node or npm version may be found (e.g. the one on the system), even if we are explicitly calling npm with the absolute path.") 42 43 // These flags feed into the perf trace keys associated with the output data. 44 osTrace = flag.String("os_trace", "", "OS this is running on.") 45 modelTrace = flag.String("model_trace", "", "Description of host machine.") 46 cpuOrGPUTrace = flag.String("cpu_or_gpu_trace", "", "If this is a CPU or GPU configuration.") 47 cpuOrGPUValueTrace = flag.String("cpu_or_gpu_value_trace", "", "The hardware of this CPU/GPU") 48 webGLVersion = flag.String("webgl_version", "", "Major WebGl version to use when creating gl drawing context. 1 or 2") 49 50 // Flags that may be required for certain configs 51 canvaskitBinPath = flag.String("canvaskit_bin_path", "", "The location of a canvaskit.js and canvaskit.wasm") 52 lottiesPath = flag.String("lotties_path", "", "Path to location of lottie files.") 53 54 // Debugging flags. 55 local = flag.Bool("local", false, "True if running locally (as opposed to on the bots)") 56 outputSteps = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.") 57 ) 58 59 // Setup. 60 ctx := td.StartRun(projectID, taskID, taskName, outputSteps, local) 61 defer td.EndRun(ctx) 62 63 keys := map[string]string{ 64 "os": *osTrace, 65 "model": *modelTrace, 66 perfKeyCpuOrGPU: *cpuOrGPUTrace, 67 "cpu_or_gpu_value": *cpuOrGPUValueTrace, 68 perfKeyWebGLVersion: *webGLVersion, 69 } 70 71 outputWithoutResults, err := makePerfObj(*gitHash, *taskID, os.Getenv("SWARMING_BOT_ID"), keys) 72 if err != nil { 73 td.Fatal(ctx, skerr.Wrap(err)) 74 } 75 // Absolute paths work more consistently than relative paths. 76 nodeBinAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *nodeBinPath, "node_bin_path") 77 benchmarkAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *benchmarkPath, "benchmark_path") 78 canvaskitBinAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *canvaskitBinPath, "canvaskit_bin_path") 79 lottiesAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *lottiesPath, "lotties_path") 80 outputAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *outputPath, "output_path") 81 82 if err := setup(ctx, benchmarkAbsPath, nodeBinAbsPath); err != nil { 83 td.Fatal(ctx, skerr.Wrap(err)) 84 } 85 86 if err := benchSkottieFrames(ctx, outputWithoutResults, benchmarkAbsPath, canvaskitBinAbsPath, lottiesAbsPath, nodeBinAbsPath); err != nil { 87 td.Fatal(ctx, skerr.Wrap(err)) 88 } 89 90 // outputFile name should be unique between tasks, so as to avoid having duplicate name files 91 // uploaded to GCS. 92 outputFile := filepath.Join(outputAbsPath, fmt.Sprintf("perf-%s.json", *taskID)) 93 if err := processSkottieFramesData(ctx, outputWithoutResults, benchmarkAbsPath, outputFile); err != nil { 94 td.Fatal(ctx, skerr.Wrap(err)) 95 } 96} 97 98const perfKeyCpuOrGPU = "cpu_or_gpu" 99 100func makePerfObj(gitHash, taskID, machineID string, keys map[string]string) (perfJSONFormat, error) { 101 rv := perfJSONFormat{} 102 if gitHash == "" { 103 return rv, skerr.Fmt("Must provide --git_hash") 104 } 105 if taskID == "" { 106 return rv, skerr.Fmt("Must provide --task_id") 107 } 108 rv.GitHash = gitHash 109 rv.SwarmingTaskID = taskID 110 rv.SwarmingMachineID = machineID 111 rv.Key = keys 112 rv.Key["arch"] = "wasm" 113 rv.Key["browser"] = "Chromium" 114 rv.Key["configuration"] = "Release" 115 rv.Key["extra_config"] = "SkottieFrames" 116 rv.Key["binary"] = "CanvasKit" 117 rv.Results = map[string]map[string]perfResult{} 118 return rv, nil 119} 120 121func setup(ctx context.Context, benchmarkPath, nodeBinPath string) error { 122 ctx = td.StartStep(ctx, td.Props("setup").Infra()) 123 defer td.EndStep(ctx) 124 125 if _, err := exec.RunCwd(ctx, benchmarkPath, filepath.Join(nodeBinPath, "npm"), "ci"); err != nil { 126 return td.FailStep(ctx, skerr.Wrap(err)) 127 } 128 129 if err := os.MkdirAll(filepath.Join(benchmarkPath, "out"), 0777); err != nil { 130 return td.FailStep(ctx, skerr.Wrap(err)) 131 } 132 return nil 133} 134 135var cpuSkiplist = []string{ 136 "Curly_Hair", // Times out after drawing ~200 frames. 137 "Day_Night", // Times out after drawing ~400 frames. 138 "an_endless_hike_on_a_tiny_world_", // Times out after drawing ~200 frames. 139 "beetle", // Times out after drawing ~500 frames. 140 "day_night_cycle", // Times out after drawing ~400 frames. 141 "fidget_spinner", // Times out after drawing ~400 frames. 142 "intelia_logo_animation", // Times out after drawing ~300 frames. 143 "siren", // Times out after drawing ~500 frames. 144 "truecosmos", // Times out after drawing ~200 frames. 145 "skottie_asset_000", 146 "skottie_asset_001", 147 "skottie_asset_003", 148 "skottie_asset_009", 149 "skottie_asset_012", 150 "skottie_asset_016", 151 "skottie_asset_018", 152 "skottie_asset_019", 153 "skottie_asset_021", 154 "navis_loader", 155 "ciclista_salita", 156 "loading", 157 "Loading_2", 158 "animacion1_-_payme", 159} 160var gpuSkiplist = []string{} 161 162// benchSkottieFrames serves lotties and assets from a folder and runs the skottie-frames-load 163// benchmark on each of them individually. The output for each will be a JSON file in 164// $benchmarkPath/out/ corresponding to the animation name. 165func benchSkottieFrames(ctx context.Context, perf perfJSONFormat, benchmarkPath, canvaskitBinPath, lottiesPath, nodeBinPath string) error { 166 ctx = td.StartStep(ctx, td.Props("perf lotties in "+lottiesPath)) 167 defer td.EndStep(ctx) 168 169 // We expect the lottiesPath to be a series of folders, each with a data.json and a subfolder of 170 // images. For example: 171 // lottiesPath 172 // /first-animation/ 173 // data.json 174 // /images/ 175 // img001.png 176 // img002.png 177 // my-font.ttf 178 var lottieFolders []string 179 err := td.Do(ctx, td.Props("locate lottie folders"), func(ctx context.Context) error { 180 return filepath.Walk(lottiesPath, func(path string, info os.FileInfo, _ error) error { 181 if path == lottiesPath { 182 return nil 183 } 184 if info.IsDir() { 185 lottieFolders = append(lottieFolders, path) 186 return filepath.SkipDir 187 } 188 return nil 189 }) 190 }) 191 if err != nil { 192 return td.FailStep(ctx, skerr.Wrap(err)) 193 } 194 skiplist := cpuSkiplist 195 if perf.Key[perfKeyCpuOrGPU] != "CPU" { 196 skiplist = gpuSkiplist 197 } 198 199 sklog.Infof("Identified %d lottie folders to benchmark", len(lottieFolders)) 200 201 var lastErr error 202 for _, lottie := range lottieFolders { 203 name := filepath.Base(lottie) 204 if util.In(name, skiplist) { 205 sklog.Infof("Skipping lottie %s", name) 206 continue 207 } 208 err = td.Do(ctx, td.Props("Benchmark "+name), func(ctx context.Context) error { 209 // See comment in setup about why we specify the absolute path for node. 210 args := []string{filepath.Join(nodeBinPath, "node"), 211 "perf-canvaskit-with-puppeteer", 212 "--bench_html", "skottie-frames.html", 213 "--canvaskit_js", filepath.Join(canvaskitBinPath, "canvaskit.js"), 214 "--canvaskit_wasm", filepath.Join(canvaskitBinPath, "canvaskit.wasm"), 215 "--input_lottie", filepath.Join(lottie, "data.json"), 216 "--assets", filepath.Join(lottie, "images"), 217 "--output", filepath.Join(benchmarkPath, "out", name+".json"), 218 } 219 if perf.Key[perfKeyCpuOrGPU] != "CPU" { 220 args = append(args, "--use_gpu") 221 if perf.Key[perfKeyWebGLVersion] == "1" { 222 args = append(args, "--query_params webgl1") 223 } 224 } else { 225 args = append(args, "--timeout=90") 226 } 227 228 _, err := exec.RunCwd(ctx, benchmarkPath, args...) 229 if err != nil { 230 return skerr.Wrap(err) 231 } 232 return nil 233 }) 234 if err != nil { 235 lastErr = td.FailStep(ctx, skerr.Wrap(err)) 236 // Don't return - we want to try to test all the inputs. 237 } 238 } 239 return lastErr // will be nil if no lottie failed. 240} 241 242type perfJSONFormat struct { 243 GitHash string `json:"gitHash"` 244 SwarmingTaskID string `json:"swarming_task_id"` 245 SwarmingMachineID string `json:"swarming_machine_id"` 246 Key map[string]string `json:"key"` 247 // Maps bench name -> "config" -> result key -> value 248 Results map[string]map[string]perfResult `json:"results"` 249} 250 251type perfResult map[string]float32 252 253// processSkottieFramesData looks at the result of benchSkottieFrames, computes summary data on 254// those files and adds them as Results into the provided perf object. The perf object is then 255// written in JSON format to outputPath. 256func processSkottieFramesData(ctx context.Context, perf perfJSONFormat, benchmarkPath, outputFilePath string) error { 257 perfJSONPath := filepath.Join(benchmarkPath, "out") 258 ctx = td.StartStep(ctx, td.Props("process perf output "+perfJSONPath)) 259 defer td.EndStep(ctx) 260 261 var jsonInputs []string 262 err := td.Do(ctx, td.Props("locate input JSON files"), func(ctx context.Context) error { 263 return filepath.Walk(perfJSONPath, func(path string, info os.FileInfo, _ error) error { 264 if strings.HasSuffix(path, ".json") { 265 jsonInputs = append(jsonInputs, path) 266 return nil 267 } 268 return nil 269 }) 270 }) 271 if err != nil { 272 return td.FailStep(ctx, skerr.Wrap(err)) 273 } 274 275 sklog.Infof("Identified %d JSON inputs to process", len(jsonInputs)) 276 277 for _, lottie := range jsonInputs { 278 err = td.Do(ctx, td.Props("Process "+lottie), func(ctx context.Context) error { 279 name := strings.TrimSuffix(filepath.Base(lottie), ".json") 280 config := "software" 281 if perf.Key[perfKeyCpuOrGPU] != "CPU" { 282 config = "webgl2" 283 if perf.Key[perfKeyWebGLVersion] == "1" { 284 config = "webgl1" 285 } 286 } 287 b, err := os_steps.ReadFile(ctx, lottie) 288 if err != nil { 289 return skerr.Wrap(err) 290 } 291 metrics, err := parseSkottieFramesMetrics(b) 292 if err != nil { 293 return skerr.Wrap(err) 294 } 295 perf.Results[name] = map[string]perfResult{ 296 config: metrics, 297 } 298 return nil 299 }) 300 if err != nil { 301 return td.FailStep(ctx, skerr.Wrap(err)) 302 } 303 } 304 305 err = td.Do(ctx, td.Props("Writing perf JSON file to "+outputFilePath), func(ctx context.Context) error { 306 if err := os.MkdirAll(filepath.Dir(outputFilePath), 0777); err != nil { 307 return skerr.Wrap(err) 308 } 309 b, err := json.MarshalIndent(perf, "", " ") 310 if err != nil { 311 return skerr.Wrap(err) 312 } 313 if err = ioutil.WriteFile(outputFilePath, b, 0666); err != nil { 314 return skerr.Wrap(err) 315 } 316 return nil 317 }) 318 if err != nil { 319 return td.FailStep(ctx, skerr.Wrap(err)) 320 } 321 322 return nil 323} 324 325type skottieFramesJSONFormat struct { 326 WithoutFlushMS []float32 `json:"without_flush_ms"` 327 WithFlushMS []float32 `json:"with_flush_ms"` 328 TotalFrameMS []float32 `json:"total_frame_ms"` 329 JSONLoadMS float32 `json:"json_load_ms"` 330} 331 332func parseSkottieFramesMetrics(b []byte) (map[string]float32, error) { 333 var metrics skottieFramesJSONFormat 334 if err := json.Unmarshal(b, &metrics); err != nil { 335 return nil, skerr.Wrap(err) 336 } 337 338 getNthFrame := func(n int) float32 { 339 if n >= len(metrics.TotalFrameMS) { 340 return 0 341 } 342 return metrics.TotalFrameMS[n] 343 } 344 345 avgFirstFive := float32(0) 346 if len(metrics.TotalFrameMS) >= 5 { 347 avgFirstFive = computeAverage(metrics.TotalFrameMS[:5]) 348 } 349 350 avgWithoutFlushMS, medianWithoutFlushMS, stddevWithoutFlushMS, _, _, _ := summarize(metrics.WithoutFlushMS) 351 avgWithFlushMS, medianWithFlushMS, stddevWithFlushMS, _, _, _ := summarize(metrics.WithFlushMS) 352 avgFrame, medFrame, stdFrame, p90Frame, p95Frame, p99Frame := summarize(metrics.TotalFrameMS) 353 354 rv := map[string]float32{ 355 "json_load_ms": metrics.JSONLoadMS, 356 357 "avg_render_without_flush_ms": avgWithoutFlushMS, 358 "median_render_without_flush_ms": medianWithoutFlushMS, 359 "stddev_render_without_flush_ms": stddevWithoutFlushMS, 360 361 "avg_render_with_flush_ms": avgWithFlushMS, 362 "median_render_with_flush_ms": medianWithFlushMS, 363 "stddev_render_with_flush_ms": stddevWithFlushMS, 364 365 "avg_render_frame_ms": avgFrame, 366 "median_render_frame_ms": medFrame, 367 "stddev_render_frame_ms": stdFrame, 368 369 // more detailed statistics on total frame times 370 "1st_frame_ms": getNthFrame(0), 371 "2nd_frame_ms": getNthFrame(1), 372 "3rd_frame_ms": getNthFrame(2), 373 "4th_frame_ms": getNthFrame(3), 374 "5th_frame_ms": getNthFrame(4), 375 "avg_first_five_frames_ms": avgFirstFive, 376 "90th_percentile_frame_ms": p90Frame, 377 "95th_percentile_frame_ms": p95Frame, 378 "99th_percentile_frame_ms": p99Frame, 379 } 380 return rv, nil 381} 382 383func summarize(input []float32) (float32, float32, float32, float32, float32, float32) { 384 // Make a copy of the data so we don't mutate the order of the original 385 sorted := make([]float32, len(input)) 386 copy(sorted, input) 387 sort.Slice(sorted, func(i, j int) bool { 388 return sorted[i] < sorted[j] 389 }) 390 391 avg := computeAverage(sorted) 392 variance := float32(0) 393 for i := 0; i < len(sorted); i++ { 394 variance += (sorted[i] - avg) * (sorted[i] - avg) 395 } 396 stddev := float32(math.Sqrt(float64(variance / float32(len(sorted))))) 397 398 medIdx := (len(sorted) * 50) / 100 399 p90Idx := (len(sorted) * 90) / 100 400 p95Idx := (len(sorted) * 95) / 100 401 p99Idx := (len(sorted) * 99) / 100 402 403 return avg, sorted[medIdx], stddev, sorted[p90Idx], sorted[p95Idx], sorted[p99Idx] 404} 405 406func computeAverage(d []float32) float32 { 407 avg := float32(0) 408 for i := 0; i < len(d); i++ { 409 avg += d[i] 410 } 411 avg /= float32(len(d)) 412 return avg 413} 414