1// Copyright 2020 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5package main 6 7import ( 8 "context" 9 "io/ioutil" 10 "os" 11 "path/filepath" 12 "testing" 13 14 "github.com/stretchr/testify/assert" 15 "github.com/stretchr/testify/require" 16 "go.skia.org/infra/go/exec" 17 "go.skia.org/infra/go/testutils" 18 "go.skia.org/infra/task_driver/go/td" 19) 20 21func TestSetup_NPMInitializedBenchmarkOutCreated(t *testing.T) { 22 benchmarkPath, err := ioutil.TempDir("", "benchmark") 23 require.NoError(t, err) 24 defer testutils.RemoveAll(t, benchmarkPath) 25 26 const fakeNodeBinPath = "/fake/path/to/node/bin" 27 28 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 29 mock := exec.CommandCollector{} 30 ctx = td.WithExecRunFn(ctx, mock.Run) 31 err := setup(ctx, benchmarkPath, fakeNodeBinPath) 32 if err != nil { 33 assert.NoError(t, err) 34 return err 35 } 36 require.Len(t, mock.Commands(), 1) 37 cmd := mock.Commands()[0] 38 assert.Equal(t, "/fake/path/to/node/bin/npm", cmd.Name) 39 assert.Equal(t, []string{"ci"}, cmd.Args) 40 return nil 41 }) 42 require.Empty(t, res.Errors) 43 require.Empty(t, res.Exceptions) 44 45 fi, err := os.Stat(filepath.Join(benchmarkPath, "out")) 46 require.NoError(t, err) 47 assert.True(t, fi.IsDir()) 48} 49 50func TestBenchSkottieFrames_CPUHasNoUseGPUFlag(t *testing.T) { 51 lotties, err := ioutil.TempDir("", "lotties") 52 require.NoError(t, err) 53 defer testutils.RemoveAll(t, lotties) 54 55 require.NoError(t, os.MkdirAll(filepath.Join(lotties, "animation_1"), 0777)) 56 57 const fakeNodeBinPath = "/fake/path/to/node/bin" 58 const fakeCanvasKitPath = "/fake/path/to/canvaskit" 59 const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer" 60 61 perfObj := perfJSONFormat{ 62 Key: map[string]string{ 63 perfKeyCpuOrGPU: "CPU", 64 }, 65 } 66 67 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 68 mock := exec.CommandCollector{} 69 ctx = td.WithExecRunFn(ctx, mock.Run) 70 err := benchSkottieFrames(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, lotties, fakeNodeBinPath) 71 if err != nil { 72 assert.NoError(t, err) 73 return err 74 } 75 require.Len(t, mock.Commands(), 1) 76 cmd := mock.Commands()[0] 77 assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name) 78 assert.Equal(t, []string{"perf-canvaskit-with-puppeteer", 79 "--bench_html", "skottie-frames.html", 80 "--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js", 81 "--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm", 82 "--input_lottie", filepath.Join(lotties, "animation_1", "data.json"), 83 "--assets", filepath.Join(lotties, "animation_1", "images"), 84 "--output", "/fake/path/to/perf-puppeteer/out/animation_1.json", 85 "--timeout=90"}, cmd.Args) 86 return nil 87 }) 88 require.Empty(t, res.Errors) 89 require.Empty(t, res.Exceptions) 90} 91 92func TestBenchSkottieFrames_GPUHasFlag(t *testing.T) { 93 lotties, err := ioutil.TempDir("", "lotties") 94 require.NoError(t, err) 95 defer testutils.RemoveAll(t, lotties) 96 97 require.NoError(t, os.MkdirAll(filepath.Join(lotties, "animation_1"), 0777)) 98 99 const fakeNodeBinPath = "/fake/path/to/node/bin" 100 const fakeCanvasKitPath = "/fake/path/to/canvaskit" 101 const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer" 102 103 perfObj := perfJSONFormat{ 104 Key: map[string]string{ 105 perfKeyCpuOrGPU: "GPU", 106 }, 107 } 108 109 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 110 mock := exec.CommandCollector{} 111 ctx = td.WithExecRunFn(ctx, mock.Run) 112 err := benchSkottieFrames(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, lotties, fakeNodeBinPath) 113 if err != nil { 114 assert.NoError(t, err) 115 return err 116 } 117 require.Len(t, mock.Commands(), 1) 118 cmd := mock.Commands()[0] 119 assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name) 120 assert.Equal(t, []string{"perf-canvaskit-with-puppeteer", 121 "--bench_html", "skottie-frames.html", 122 "--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js", 123 "--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm", 124 "--input_lottie", filepath.Join(lotties, "animation_1", "data.json"), 125 "--assets", filepath.Join(lotties, "animation_1", "images"), 126 "--output", "/fake/path/to/perf-puppeteer/out/animation_1.json", 127 "--use_gpu"}, cmd.Args) 128 return nil 129 }) 130 require.Empty(t, res.Errors) 131 require.Empty(t, res.Exceptions) 132} 133 134// TestProcessSkottieFramesData_CPUTwoInputsGetSummarizedAndCombined tests the scenario where we 135// have multiple inputs to process. The input directory should get scanned for all json files; 136// these JSON files should be read in and converted to perf results, using the name of the file 137// as the name (w/o the .json suffix). 138func TestProcessSkottieFramesData_CPUTwoInputsGetSummarizedAndCombined(t *testing.T) { 139 input, err := ioutil.TempDir("", "inputs") 140 require.NoError(t, err) 141 defer testutils.RemoveAll(t, input) 142 err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{ 143 "first_animation.json": skottieFramesSampleOne, 144 "second_animation.json": skottieFramesSampleTwo, 145 }) 146 require.NoError(t, err) 147 output, err := ioutil.TempDir("", "perf") 148 require.NoError(t, err) 149 defer testutils.RemoveAll(t, output) 150 151 keys := map[string]string{ 152 "os": "Debian10", 153 "model": "GCE", 154 perfKeyCpuOrGPU: "CPU", 155 "cpu_or_gpu_value": "AVX2", 156 } 157 158 perfObj, err := makePerfObj(someGitHash, someTaskID, someMachineID, keys) 159 require.NoError(t, err) 160 161 outputFile := filepath.Join(output, "perf-taskid1352.json") 162 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 163 return processSkottieFramesData(ctx, perfObj, input, outputFile) 164 }) 165 require.Empty(t, res.Errors) 166 require.Empty(t, res.Exceptions) 167 168 b, err := ioutil.ReadFile(outputFile) 169 require.NoError(t, err) 170 171 assert.Equal(t, `{ 172 "gitHash": "032631e490db494128e0610a19adce4cab9706d1", 173 "swarming_task_id": "4bdd43ed7c906c11", 174 "swarming_machine_id": "skia-e-gce-203", 175 "key": { 176 "arch": "wasm", 177 "binary": "CanvasKit", 178 "browser": "Chromium", 179 "configuration": "Release", 180 "cpu_or_gpu": "CPU", 181 "cpu_or_gpu_value": "AVX2", 182 "extra_config": "SkottieFrames", 183 "model": "GCE", 184 "os": "Debian10" 185 }, 186 "results": { 187 "first_animation": { 188 "software": { 189 "1st_frame_ms": 31.555, 190 "2nd_frame_ms": 87.795, 191 "3rd_frame_ms": 0.43, 192 "4th_frame_ms": 1.845, 193 "5th_frame_ms": 3.61, 194 "90th_percentile_frame_ms": 4.455, 195 "95th_percentile_frame_ms": 31.555, 196 "99th_percentile_frame_ms": 87.795, 197 "avg_first_five_frames_ms": 25.047, 198 "avg_render_frame_ms": 5.662692, 199 "avg_render_with_flush_ms": 1.75, 200 "avg_render_without_flush_ms": 1.875, 201 "json_load_ms": 16.05, 202 "median_render_frame_ms": 0.795, 203 "median_render_with_flush_ms": 1.8, 204 "median_render_without_flush_ms": 1.88, 205 "stddev_render_frame_ms": 17.463467, 206 "stddev_render_with_flush_ms": 0.74999994, 207 "stddev_render_without_flush_ms": 0.07500001 208 } 209 }, 210 "second_animation": { 211 "software": { 212 "1st_frame_ms": 210.555, 213 "2nd_frame_ms": 770.795, 214 "3rd_frame_ms": 10.43, 215 "4th_frame_ms": 31.845, 216 "5th_frame_ms": 3.61, 217 "90th_percentile_frame_ms": 210.555, 218 "95th_percentile_frame_ms": 400.455, 219 "99th_percentile_frame_ms": 770.795, 220 "avg_first_five_frames_ms": 205.44699, 221 "avg_render_frame_ms": 55.58577, 222 "avg_render_with_flush_ms": 3.75, 223 "avg_render_without_flush_ms": 5.125, 224 "json_load_ms": 28.15, 225 "median_render_frame_ms": 0.8, 226 "median_render_with_flush_ms": 3.8, 227 "median_render_without_flush_ms": 5.13, 228 "stddev_render_frame_ms": 166.36926, 229 "stddev_render_with_flush_ms": 0.75, 230 "stddev_render_without_flush_ms": 0.074999936 231 } 232 } 233 } 234}`, string(b)) 235} 236 237func TestProcessSkottieFramesData_GPUTwoInputsGetSummarizedAndCombined(t *testing.T) { 238 input, err := ioutil.TempDir("", "inputs") 239 require.NoError(t, err) 240 defer testutils.RemoveAll(t, input) 241 err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{ 242 "first_animation.json": skottieFramesSampleOne, 243 "second_animation.json": skottieFramesSampleTwo, 244 }) 245 require.NoError(t, err) 246 output, err := ioutil.TempDir("", "perf") 247 require.NoError(t, err) 248 defer testutils.RemoveAll(t, output) 249 250 // These are based off of realistic values. 251 keys := map[string]string{ 252 "os": "Ubuntu18", 253 "model": "Golo", 254 perfKeyCpuOrGPU: "GPU", 255 "cpu_or_gpu_value": "QuadroP400", 256 } 257 258 perfObj, err := makePerfObj(someGitHash, someTaskID, someMachineID, keys) 259 require.NoError(t, err) 260 261 outputFile := filepath.Join(output, "perf-taskid1352.json") 262 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 263 return processSkottieFramesData(ctx, perfObj, input, outputFile) 264 }) 265 require.Empty(t, res.Errors) 266 267 b, err := ioutil.ReadFile(outputFile) 268 require.NoError(t, err) 269 270 assert.Equal(t, `{ 271 "gitHash": "032631e490db494128e0610a19adce4cab9706d1", 272 "swarming_task_id": "4bdd43ed7c906c11", 273 "swarming_machine_id": "skia-e-gce-203", 274 "key": { 275 "arch": "wasm", 276 "binary": "CanvasKit", 277 "browser": "Chromium", 278 "configuration": "Release", 279 "cpu_or_gpu": "GPU", 280 "cpu_or_gpu_value": "QuadroP400", 281 "extra_config": "SkottieFrames", 282 "model": "Golo", 283 "os": "Ubuntu18" 284 }, 285 "results": { 286 "first_animation": { 287 "webgl2": { 288 "1st_frame_ms": 31.555, 289 "2nd_frame_ms": 87.795, 290 "3rd_frame_ms": 0.43, 291 "4th_frame_ms": 1.845, 292 "5th_frame_ms": 3.61, 293 "90th_percentile_frame_ms": 4.455, 294 "95th_percentile_frame_ms": 31.555, 295 "99th_percentile_frame_ms": 87.795, 296 "avg_first_five_frames_ms": 25.047, 297 "avg_render_frame_ms": 5.662692, 298 "avg_render_with_flush_ms": 1.75, 299 "avg_render_without_flush_ms": 1.875, 300 "json_load_ms": 16.05, 301 "median_render_frame_ms": 0.795, 302 "median_render_with_flush_ms": 1.8, 303 "median_render_without_flush_ms": 1.88, 304 "stddev_render_frame_ms": 17.463467, 305 "stddev_render_with_flush_ms": 0.74999994, 306 "stddev_render_without_flush_ms": 0.07500001 307 } 308 }, 309 "second_animation": { 310 "webgl2": { 311 "1st_frame_ms": 210.555, 312 "2nd_frame_ms": 770.795, 313 "3rd_frame_ms": 10.43, 314 "4th_frame_ms": 31.845, 315 "5th_frame_ms": 3.61, 316 "90th_percentile_frame_ms": 210.555, 317 "95th_percentile_frame_ms": 400.455, 318 "99th_percentile_frame_ms": 770.795, 319 "avg_first_five_frames_ms": 205.44699, 320 "avg_render_frame_ms": 55.58577, 321 "avg_render_with_flush_ms": 3.75, 322 "avg_render_without_flush_ms": 5.125, 323 "json_load_ms": 28.15, 324 "median_render_frame_ms": 0.8, 325 "median_render_with_flush_ms": 3.8, 326 "median_render_without_flush_ms": 5.13, 327 "stddev_render_frame_ms": 166.36926, 328 "stddev_render_with_flush_ms": 0.75, 329 "stddev_render_without_flush_ms": 0.074999936 330 } 331 } 332 } 333}`, string(b)) 334} 335 336func writeFilesToDisk(path string, fileNamesToContent map[string]string) error { 337 if err := os.MkdirAll(path, 0777); err != nil { 338 return err 339 } 340 for name, content := range fileNamesToContent { 341 if err := ioutil.WriteFile(filepath.Join(path, name), []byte(content), 0666); err != nil { 342 return err 343 } 344 } 345 return nil 346} 347 348const ( 349 someGitHash = "032631e490db494128e0610a19adce4cab9706d1" 350 someTaskID = "4bdd43ed7c906c11" 351 someMachineID = "skia-e-gce-203" 352) 353 354const skottieFramesSampleOne = ` 355{ 356 "total_frame_ms": [ 357 31.555, 358 87.795, 359 0.430, 360 1.845, 361 3.610, 362 1.105, 363 0.545, 364 2.315, 365 1.685, 366 0.615, 367 0.425, 368 0.815, 369 0.355, 370 0.655, 371 0.390, 372 4.455, 373 0.800, 374 0.685, 375 2.630, 376 0.325, 377 0.355, 378 0.740, 379 0.785, 380 0.795, 381 0.72, 382 0.80 383 ], 384 "without_flush_ms": [ 385 2.0, 386 1.99, 387 1.98, 388 1.97, 389 1.96, 390 1.95, 391 1.94, 392 1.93, 393 1.92, 394 1.91, 395 1.9, 396 1.89, 397 1.88, 398 1.87, 399 1.86, 400 1.85, 401 1.84, 402 1.83, 403 1.82, 404 1.81, 405 1.8, 406 1.79, 407 1.78, 408 1.77, 409 1.76, 410 1.75 411 ], 412 "with_flush_ms": [ 413 3.0, 414 2.9, 415 2.8, 416 2.7, 417 2.6, 418 2.5, 419 2.4, 420 2.3, 421 2.2, 422 2.1, 423 2.0, 424 1.9, 425 1.8, 426 1.7, 427 1.6, 428 1.5, 429 1.4, 430 1.3, 431 1.2, 432 1.1, 433 1.0, 434 0.9, 435 0.8, 436 0.7, 437 0.6, 438 0.5 439 ], 440 "json_load_ms": 16.05 441}` 442 443const skottieFramesSampleTwo = ` 444{ 445 "total_frame_ms": [ 446 210.555, 447 770.795, 448 10.430, 449 31.845, 450 3.610, 451 1.105, 452 0.545, 453 2.315, 454 1.685, 455 0.615, 456 0.425, 457 0.815, 458 0.355, 459 0.655, 460 0.390, 461 400.455, 462 0.800, 463 0.685, 464 2.630, 465 0.325, 466 0.355, 467 0.740, 468 0.785, 469 0.795, 470 0.72, 471 0.80 472 ], 473 "without_flush_ms": [ 474 5.0, 475 5.01, 476 5.02, 477 5.03, 478 5.04, 479 5.05, 480 5.06, 481 5.07, 482 5.08, 483 5.09, 484 5.1, 485 5.11, 486 5.12, 487 5.13, 488 5.14, 489 5.15, 490 5.16, 491 5.17, 492 5.18, 493 5.19, 494 5.2, 495 5.21, 496 5.22, 497 5.23, 498 5.24, 499 5.25 500 ], 501 "with_flush_ms": [ 502 5.0, 503 4.9, 504 4.8, 505 4.7, 506 4.6, 507 4.5, 508 4.4, 509 4.3, 510 4.2, 511 4.1, 512 4.0, 513 3.9, 514 3.8, 515 3.7, 516 3.6, 517 3.5, 518 3.4, 519 3.3, 520 3.2, 521 3.1, 522 3.0, 523 2.9, 524 2.8, 525 2.7, 526 2.6, 527 2.5 528 ], 529 "json_load_ms": 28.15 530}` 531