1// Copyright 2020 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5package main 6 7import ( 8 "context" 9 "io/ioutil" 10 "os" 11 "path/filepath" 12 "testing" 13 14 "github.com/stretchr/testify/assert" 15 "github.com/stretchr/testify/require" 16 "go.skia.org/infra/go/exec" 17 "go.skia.org/infra/go/testutils" 18 "go.skia.org/infra/task_driver/go/td" 19) 20 21func TestSetup_NPMInitializedBenchmarkOutCreated(t *testing.T) { 22 benchmarkPath, err := ioutil.TempDir("", "benchmark") 23 require.NoError(t, err) 24 defer testutils.RemoveAll(t, benchmarkPath) 25 26 const fakeNodeBinPath = "/fake/path/to/node/bin" 27 28 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 29 mock := exec.CommandCollector{} 30 ctx = td.WithExecRunFn(ctx, mock.Run) 31 err := setup(ctx, benchmarkPath, fakeNodeBinPath) 32 if err != nil { 33 assert.NoError(t, err) 34 return err 35 } 36 require.Len(t, mock.Commands(), 1) 37 cmd := mock.Commands()[0] 38 assert.Equal(t, "/fake/path/to/node/bin/npm", cmd.Name) 39 assert.Equal(t, []string{"ci"}, cmd.Args) 40 return nil 41 }) 42 require.Empty(t, res.Errors) 43 require.Empty(t, res.Exceptions) 44 45 fi, err := os.Stat(filepath.Join(benchmarkPath, "out")) 46 require.NoError(t, err) 47 assert.True(t, fi.IsDir()) 48} 49 50func TestBenchSkottieFrames_CPUHasNoUseGPUFlag(t *testing.T) { 51 lotties, err := ioutil.TempDir("", "lotties") 52 require.NoError(t, err) 53 defer testutils.RemoveAll(t, lotties) 54 55 require.NoError(t, os.MkdirAll(filepath.Join(lotties, "animation_1"), 0777)) 56 57 const fakeNodeBinPath = "/fake/path/to/node/bin" 58 const fakeCanvasKitPath = "/fake/path/to/canvaskit" 59 const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer" 60 61 perfObj := perfJSONFormat{ 62 Key: map[string]string{ 63 perfKeyCpuOrGPU: "CPU", 64 }, 65 } 66 67 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 68 mock := exec.CommandCollector{} 69 ctx = td.WithExecRunFn(ctx, mock.Run) 70 err := benchSkottieFrames(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, lotties, fakeNodeBinPath) 71 if err != nil { 72 assert.NoError(t, err) 73 return err 74 } 75 require.Len(t, mock.Commands(), 1) 76 cmd := mock.Commands()[0] 77 assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name) 78 assert.Equal(t, []string{"perf-canvaskit-with-puppeteer", 79 "--bench_html", "skottie-frames.html", 80 "--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js", 81 "--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm", 82 "--input_lottie", filepath.Join(lotties, "animation_1", "data.json"), 83 "--assets", filepath.Join(lotties, "animation_1", "images"), 84 "--output", "/fake/path/to/perf-puppeteer/out/animation_1.json"}, cmd.Args) 85 return nil 86 }) 87 require.Empty(t, res.Errors) 88 require.Empty(t, res.Exceptions) 89} 90 91func TestBenchSkottieFrames_GPUHasFlag(t *testing.T) { 92 lotties, err := ioutil.TempDir("", "lotties") 93 require.NoError(t, err) 94 defer testutils.RemoveAll(t, lotties) 95 96 require.NoError(t, os.MkdirAll(filepath.Join(lotties, "animation_1"), 0777)) 97 98 const fakeNodeBinPath = "/fake/path/to/node/bin" 99 const fakeCanvasKitPath = "/fake/path/to/canvaskit" 100 const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer" 101 102 perfObj := perfJSONFormat{ 103 Key: map[string]string{ 104 perfKeyCpuOrGPU: "GPU", 105 }, 106 } 107 108 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 109 mock := exec.CommandCollector{} 110 ctx = td.WithExecRunFn(ctx, mock.Run) 111 err := benchSkottieFrames(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, lotties, fakeNodeBinPath) 112 if err != nil { 113 assert.NoError(t, err) 114 return err 115 } 116 require.Len(t, mock.Commands(), 1) 117 cmd := mock.Commands()[0] 118 assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name) 119 assert.Equal(t, []string{"perf-canvaskit-with-puppeteer", 120 "--bench_html", "skottie-frames.html", 121 "--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js", 122 "--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm", 123 "--input_lottie", filepath.Join(lotties, "animation_1", "data.json"), 124 "--assets", filepath.Join(lotties, "animation_1", "images"), 125 "--output", "/fake/path/to/perf-puppeteer/out/animation_1.json", 126 "--use_gpu"}, cmd.Args) 127 return nil 128 }) 129 require.Empty(t, res.Errors) 130 require.Empty(t, res.Exceptions) 131} 132 133// TestProcessSkottieFramesData_CPUTwoInputsGetSummarizedAndCombined tests the scenario where we 134// have multiple inputs to process. The input directory should get scanned for all json files; 135// these JSON files should be read in and converted to perf results, using the name of the file 136// as the name (w/o the .json suffix). 137func TestProcessSkottieFramesData_CPUTwoInputsGetSummarizedAndCombined(t *testing.T) { 138 input, err := ioutil.TempDir("", "inputs") 139 require.NoError(t, err) 140 defer testutils.RemoveAll(t, input) 141 err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{ 142 "first_animation.json": skottieFramesSampleOne, 143 "second_animation.json": skottieFramesSampleTwo, 144 }) 145 require.NoError(t, err) 146 output, err := ioutil.TempDir("", "perf") 147 require.NoError(t, err) 148 defer testutils.RemoveAll(t, output) 149 150 keys := map[string]string{ 151 "os": "Debian10", 152 "model": "GCE", 153 perfKeyCpuOrGPU: "CPU", 154 "cpu_or_gpu_value": "AVX2", 155 } 156 157 perfObj, err := makePerfObj(someGitHash, someTaskID, someMachineID, keys) 158 require.NoError(t, err) 159 160 outputFile := filepath.Join(output, "perf-taskid1352.json") 161 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 162 return processSkottieFramesData(ctx, perfObj, input, outputFile) 163 }) 164 require.Empty(t, res.Errors) 165 require.Empty(t, res.Exceptions) 166 167 b, err := ioutil.ReadFile(outputFile) 168 require.NoError(t, err) 169 170 assert.Equal(t, `{ 171 "gitHash": "032631e490db494128e0610a19adce4cab9706d1", 172 "swarming_task_id": "4bdd43ed7c906c11", 173 "swarming_machine_id": "skia-e-gce-203", 174 "key": { 175 "arch": "wasm", 176 "binary": "CanvasKit", 177 "browser": "Chromium", 178 "configuration": "Release", 179 "cpu_or_gpu": "CPU", 180 "cpu_or_gpu_value": "AVX2", 181 "extra_config": "SkottieFrames", 182 "model": "GCE", 183 "os": "Debian10" 184 }, 185 "results": { 186 "first_animation": { 187 "software": { 188 "1st_frame_ms": 31.555, 189 "2nd_frame_ms": 87.795, 190 "3rd_frame_ms": 0.43, 191 "4th_frame_ms": 1.845, 192 "5th_frame_ms": 3.61, 193 "90th_percentile_frame_ms": 4.455, 194 "95th_percentile_frame_ms": 31.555, 195 "99th_percentile_frame_ms": 87.795, 196 "avg_first_five_frames_ms": 25.047, 197 "avg_render_frame_ms": 5.662692, 198 "avg_render_with_flush_ms": 1.75, 199 "avg_render_without_flush_ms": 1.875, 200 "json_load_ms": 16.05, 201 "median_render_frame_ms": 0.795, 202 "median_render_with_flush_ms": 1.8, 203 "median_render_without_flush_ms": 1.88, 204 "stddev_render_frame_ms": 17.463467, 205 "stddev_render_with_flush_ms": 0.74999994, 206 "stddev_render_without_flush_ms": 0.07500001 207 } 208 }, 209 "second_animation": { 210 "software": { 211 "1st_frame_ms": 210.555, 212 "2nd_frame_ms": 770.795, 213 "3rd_frame_ms": 10.43, 214 "4th_frame_ms": 31.845, 215 "5th_frame_ms": 3.61, 216 "90th_percentile_frame_ms": 210.555, 217 "95th_percentile_frame_ms": 400.455, 218 "99th_percentile_frame_ms": 770.795, 219 "avg_first_five_frames_ms": 205.44699, 220 "avg_render_frame_ms": 55.58577, 221 "avg_render_with_flush_ms": 3.75, 222 "avg_render_without_flush_ms": 5.125, 223 "json_load_ms": 28.15, 224 "median_render_frame_ms": 0.8, 225 "median_render_with_flush_ms": 3.8, 226 "median_render_without_flush_ms": 5.13, 227 "stddev_render_frame_ms": 166.36926, 228 "stddev_render_with_flush_ms": 0.75, 229 "stddev_render_without_flush_ms": 0.074999936 230 } 231 } 232 } 233}`, string(b)) 234} 235 236func TestProcessSkottieFramesData_GPUTwoInputsGetSummarizedAndCombined(t *testing.T) { 237 input, err := ioutil.TempDir("", "inputs") 238 require.NoError(t, err) 239 defer testutils.RemoveAll(t, input) 240 err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{ 241 "first_animation.json": skottieFramesSampleOne, 242 "second_animation.json": skottieFramesSampleTwo, 243 }) 244 require.NoError(t, err) 245 output, err := ioutil.TempDir("", "perf") 246 require.NoError(t, err) 247 defer testutils.RemoveAll(t, output) 248 249 // These are based off of realistic values. 250 keys := map[string]string{ 251 "os": "Ubuntu18", 252 "model": "Golo", 253 perfKeyCpuOrGPU: "GPU", 254 "cpu_or_gpu_value": "QuadroP400", 255 } 256 257 perfObj, err := makePerfObj(someGitHash, someTaskID, someMachineID, keys) 258 require.NoError(t, err) 259 260 outputFile := filepath.Join(output, "perf-taskid1352.json") 261 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 262 return processSkottieFramesData(ctx, perfObj, input, outputFile) 263 }) 264 require.Empty(t, res.Errors) 265 266 b, err := ioutil.ReadFile(outputFile) 267 require.NoError(t, err) 268 269 assert.Equal(t, `{ 270 "gitHash": "032631e490db494128e0610a19adce4cab9706d1", 271 "swarming_task_id": "4bdd43ed7c906c11", 272 "swarming_machine_id": "skia-e-gce-203", 273 "key": { 274 "arch": "wasm", 275 "binary": "CanvasKit", 276 "browser": "Chromium", 277 "configuration": "Release", 278 "cpu_or_gpu": "GPU", 279 "cpu_or_gpu_value": "QuadroP400", 280 "extra_config": "SkottieFrames", 281 "model": "Golo", 282 "os": "Ubuntu18" 283 }, 284 "results": { 285 "first_animation": { 286 "webgl2": { 287 "1st_frame_ms": 31.555, 288 "2nd_frame_ms": 87.795, 289 "3rd_frame_ms": 0.43, 290 "4th_frame_ms": 1.845, 291 "5th_frame_ms": 3.61, 292 "90th_percentile_frame_ms": 4.455, 293 "95th_percentile_frame_ms": 31.555, 294 "99th_percentile_frame_ms": 87.795, 295 "avg_first_five_frames_ms": 25.047, 296 "avg_render_frame_ms": 5.662692, 297 "avg_render_with_flush_ms": 1.75, 298 "avg_render_without_flush_ms": 1.875, 299 "json_load_ms": 16.05, 300 "median_render_frame_ms": 0.795, 301 "median_render_with_flush_ms": 1.8, 302 "median_render_without_flush_ms": 1.88, 303 "stddev_render_frame_ms": 17.463467, 304 "stddev_render_with_flush_ms": 0.74999994, 305 "stddev_render_without_flush_ms": 0.07500001 306 } 307 }, 308 "second_animation": { 309 "webgl2": { 310 "1st_frame_ms": 210.555, 311 "2nd_frame_ms": 770.795, 312 "3rd_frame_ms": 10.43, 313 "4th_frame_ms": 31.845, 314 "5th_frame_ms": 3.61, 315 "90th_percentile_frame_ms": 210.555, 316 "95th_percentile_frame_ms": 400.455, 317 "99th_percentile_frame_ms": 770.795, 318 "avg_first_five_frames_ms": 205.44699, 319 "avg_render_frame_ms": 55.58577, 320 "avg_render_with_flush_ms": 3.75, 321 "avg_render_without_flush_ms": 5.125, 322 "json_load_ms": 28.15, 323 "median_render_frame_ms": 0.8, 324 "median_render_with_flush_ms": 3.8, 325 "median_render_without_flush_ms": 5.13, 326 "stddev_render_frame_ms": 166.36926, 327 "stddev_render_with_flush_ms": 0.75, 328 "stddev_render_without_flush_ms": 0.074999936 329 } 330 } 331 } 332}`, string(b)) 333} 334 335func writeFilesToDisk(path string, fileNamesToContent map[string]string) error { 336 if err := os.MkdirAll(path, 0777); err != nil { 337 return err 338 } 339 for name, content := range fileNamesToContent { 340 if err := ioutil.WriteFile(filepath.Join(path, name), []byte(content), 0666); err != nil { 341 return err 342 } 343 } 344 return nil 345} 346 347const ( 348 someGitHash = "032631e490db494128e0610a19adce4cab9706d1" 349 someTaskID = "4bdd43ed7c906c11" 350 someMachineID = "skia-e-gce-203" 351) 352 353const skottieFramesSampleOne = ` 354{ 355 "total_frame_ms": [ 356 31.555, 357 87.795, 358 0.430, 359 1.845, 360 3.610, 361 1.105, 362 0.545, 363 2.315, 364 1.685, 365 0.615, 366 0.425, 367 0.815, 368 0.355, 369 0.655, 370 0.390, 371 4.455, 372 0.800, 373 0.685, 374 2.630, 375 0.325, 376 0.355, 377 0.740, 378 0.785, 379 0.795, 380 0.72, 381 0.80 382 ], 383 "without_flush_ms": [ 384 2.0, 385 1.99, 386 1.98, 387 1.97, 388 1.96, 389 1.95, 390 1.94, 391 1.93, 392 1.92, 393 1.91, 394 1.9, 395 1.89, 396 1.88, 397 1.87, 398 1.86, 399 1.85, 400 1.84, 401 1.83, 402 1.82, 403 1.81, 404 1.8, 405 1.79, 406 1.78, 407 1.77, 408 1.76, 409 1.75 410 ], 411 "with_flush_ms": [ 412 3.0, 413 2.9, 414 2.8, 415 2.7, 416 2.6, 417 2.5, 418 2.4, 419 2.3, 420 2.2, 421 2.1, 422 2.0, 423 1.9, 424 1.8, 425 1.7, 426 1.6, 427 1.5, 428 1.4, 429 1.3, 430 1.2, 431 1.1, 432 1.0, 433 0.9, 434 0.8, 435 0.7, 436 0.6, 437 0.5 438 ], 439 "json_load_ms": 16.05 440}` 441 442const skottieFramesSampleTwo = ` 443{ 444 "total_frame_ms": [ 445 210.555, 446 770.795, 447 10.430, 448 31.845, 449 3.610, 450 1.105, 451 0.545, 452 2.315, 453 1.685, 454 0.615, 455 0.425, 456 0.815, 457 0.355, 458 0.655, 459 0.390, 460 400.455, 461 0.800, 462 0.685, 463 2.630, 464 0.325, 465 0.355, 466 0.740, 467 0.785, 468 0.795, 469 0.72, 470 0.80 471 ], 472 "without_flush_ms": [ 473 5.0, 474 5.01, 475 5.02, 476 5.03, 477 5.04, 478 5.05, 479 5.06, 480 5.07, 481 5.08, 482 5.09, 483 5.1, 484 5.11, 485 5.12, 486 5.13, 487 5.14, 488 5.15, 489 5.16, 490 5.17, 491 5.18, 492 5.19, 493 5.2, 494 5.21, 495 5.22, 496 5.23, 497 5.24, 498 5.25 499 ], 500 "with_flush_ms": [ 501 5.0, 502 4.9, 503 4.8, 504 4.7, 505 4.6, 506 4.5, 507 4.4, 508 4.3, 509 4.2, 510 4.1, 511 4.0, 512 3.9, 513 3.8, 514 3.7, 515 3.6, 516 3.5, 517 3.4, 518 3.3, 519 3.2, 520 3.1, 521 3.0, 522 2.9, 523 2.8, 524 2.7, 525 2.6, 526 2.5 527 ], 528 "json_load_ms": 28.15 529}` 530