1// Copyright 2020 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5package main 6 7import ( 8 "context" 9 "io/ioutil" 10 "os" 11 "path/filepath" 12 "testing" 13 14 "github.com/stretchr/testify/assert" 15 "github.com/stretchr/testify/require" 16 "go.skia.org/infra/go/exec" 17 "go.skia.org/infra/go/testutils" 18 "go.skia.org/infra/task_driver/go/td" 19) 20 21func TestSetup_NPMInitializedBenchmarkOutCreated(t *testing.T) { 22 benchmarkPath, err := ioutil.TempDir("", "benchmark") 23 require.NoError(t, err) 24 defer testutils.RemoveAll(t, benchmarkPath) 25 26 const fakeNodeBinPath = "/fake/path/to/node/bin" 27 28 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 29 mock := exec.CommandCollector{} 30 ctx = td.WithExecRunFn(ctx, mock.Run) 31 err := setup(ctx, benchmarkPath, fakeNodeBinPath) 32 if err != nil { 33 assert.NoError(t, err) 34 return err 35 } 36 require.Len(t, mock.Commands(), 1) 37 cmd := mock.Commands()[0] 38 assert.Equal(t, "/fake/path/to/node/bin/npm", cmd.Name) 39 assert.Equal(t, []string{"ci"}, cmd.Args) 40 return nil 41 }) 42 require.Empty(t, res.Errors) 43 require.Empty(t, res.Exceptions) 44 45 fi, err := os.Stat(filepath.Join(benchmarkPath, "out")) 46 require.NoError(t, err) 47 assert.True(t, fi.IsDir()) 48} 49 50func TestBenchCanvas_CPUHasNoUseGPUFlag(t *testing.T) { 51 const fakeNodeBinPath = "/fake/path/to/node/bin" 52 const fakeCanvasKitPath = "/fake/path/to/canvaskit" 53 const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer" 54 55 perfObj := perfJSONFormat{ 56 Key: map[string]string{ 57 perfKeyCpuOrGPU: "CPU", 58 }, 59 } 60 61 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 62 mock := exec.CommandCollector{} 63 ctx = td.WithExecRunFn(ctx, mock.Run) 64 err := benchCanvas(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, fakeNodeBinPath) 65 if err != nil { 66 assert.NoError(t, err) 67 return err 68 } 69 require.Len(t, mock.Commands(), 1) 70 cmd := mock.Commands()[0] 71 assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name) 72 assert.Equal(t, []string{"perf-canvaskit-with-puppeteer", 73 "--bench_html", "canvas_perf.html", 74 "--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js", 75 "--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm", 76 "--assets", "canvas_perf_assets", 77 "--output", "/fake/path/to/perf-puppeteer/out/perf.json", 78 "--timeout", "240", 79 }, cmd.Args) 80 return nil 81 }) 82 require.Empty(t, res.Errors) 83 require.Empty(t, res.Exceptions) 84} 85 86func TestBenchCanvas_GPUHasFlag(t *testing.T) { 87 const fakeNodeBinPath = "/fake/path/to/node/bin" 88 const fakeCanvasKitPath = "/fake/path/to/canvaskit" 89 const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer" 90 91 perfObj := perfJSONFormat{ 92 Key: map[string]string{ 93 perfKeyCpuOrGPU: "GPU", 94 }, 95 } 96 97 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 98 mock := exec.CommandCollector{} 99 ctx = td.WithExecRunFn(ctx, mock.Run) 100 err := benchCanvas(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, fakeNodeBinPath) 101 if err != nil { 102 assert.NoError(t, err) 103 return err 104 } 105 require.Len(t, mock.Commands(), 1) 106 cmd := mock.Commands()[0] 107 assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name) 108 assert.Equal(t, []string{"perf-canvaskit-with-puppeteer", 109 "--bench_html", "canvas_perf.html", 110 "--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js", 111 "--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm", 112 "--assets", "canvas_perf_assets", 113 "--output", "/fake/path/to/perf-puppeteer/out/perf.json", 114 "--timeout", "240", 115 "--use_gpu"}, cmd.Args) 116 return nil 117 }) 118 require.Empty(t, res.Errors) 119 require.Empty(t, res.Exceptions) 120} 121 122// TestProcessFramesData_GPUTwoInputsGetSummarizedAndCombined tests the reading of the 123// output JSON file, and reorganizing and computing perf results and checks the result format. 124func TestProcessFramesData_GPUTwoInputsGetSummarizedAndCombined(t *testing.T) { 125 input, err := ioutil.TempDir("", "inputs") 126 require.NoError(t, err) 127 defer testutils.RemoveAll(t, input) 128 err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{ 129 "perf.json": sampleData, 130 }) 131 require.NoError(t, err) 132 output, err := ioutil.TempDir("", "perf") 133 require.NoError(t, err) 134 defer testutils.RemoveAll(t, output) 135 136 // These are based off of realistic values. 137 keys := map[string]string{ 138 "os": "Ubuntu18", 139 "model": "Golo", 140 perfKeyCpuOrGPU: "GPU", 141 "cpu_or_gpu_value": "QuadroP400", 142 } 143 144 perfObj, err := makePerfObj(someGitHash, someTaskID, someMachineID, keys) 145 require.NoError(t, err) 146 147 outputFile := filepath.Join(output, "perf-taskid1352.json") 148 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 149 return processFramesData(ctx, perfObj, input, outputFile) 150 }) 151 require.Empty(t, res.Errors) 152 153 b, err := ioutil.ReadFile(outputFile) 154 require.NoError(t, err) 155 156 assert.Equal(t, `{ 157 "gitHash": "032631e490db494128e0610a19adce4cab9706d1", 158 "swarming_task_id": "4bdd43ed7c906c11", 159 "swarming_machine_id": "skia-e-gce-203", 160 "key": { 161 "arch": "wasm", 162 "binary": "CanvasKit", 163 "browser": "Chromium", 164 "configuration": "Release", 165 "cpu_or_gpu": "GPU", 166 "cpu_or_gpu_value": "QuadroP400", 167 "extra_config": "CanvasPerf", 168 "model": "Golo", 169 "os": "Ubuntu18" 170 }, 171 "results": { 172 "canvas_drawColor": { 173 "webgl2": { 174 "90th_percentile_frame_ms": 4.455, 175 "95th_percentile_frame_ms": 31.555, 176 "99th_percentile_frame_ms": 87.795, 177 "avg_render_frame_ms": 5.662692, 178 "avg_render_with_flush_ms": 1.75, 179 "avg_render_without_flush_ms": 1.875, 180 "median_render_frame_ms": 0.795, 181 "median_render_with_flush_ms": 1.8, 182 "median_render_without_flush_ms": 1.88, 183 "stddev_render_frame_ms": 17.463467, 184 "stddev_render_with_flush_ms": 0.74999994, 185 "stddev_render_without_flush_ms": 0.07500001 186 } 187 }, 188 "canvas_drawHugeGradient": { 189 "webgl2": { 190 "90th_percentile_frame_ms": 210.555, 191 "95th_percentile_frame_ms": 400.455, 192 "99th_percentile_frame_ms": 770.795, 193 "avg_render_frame_ms": 55.58577, 194 "avg_render_with_flush_ms": 3.75, 195 "avg_render_without_flush_ms": 5.125, 196 "median_render_frame_ms": 0.8, 197 "median_render_with_flush_ms": 3.8, 198 "median_render_without_flush_ms": 5.13, 199 "stddev_render_frame_ms": 166.36926, 200 "stddev_render_with_flush_ms": 0.75, 201 "stddev_render_without_flush_ms": 0.074999936 202 } 203 } 204 } 205}`, string(b)) 206} 207 208func writeFilesToDisk(path string, fileNamesToContent map[string]string) error { 209 if err := os.MkdirAll(path, 0777); err != nil { 210 return err 211 } 212 for name, content := range fileNamesToContent { 213 if err := ioutil.WriteFile(filepath.Join(path, name), []byte(content), 0666); err != nil { 214 return err 215 } 216 } 217 return nil 218} 219 220const ( 221 someGitHash = "032631e490db494128e0610a19adce4cab9706d1" 222 someTaskID = "4bdd43ed7c906c11" 223 someMachineID = "skia-e-gce-203" 224) 225 226const sampleData = `{ 227"canvas_drawColor": { 228 "total_frame_ms": [ 229 31.555, 230 87.795, 231 0.430, 232 1.845, 233 3.610, 234 1.105, 235 0.545, 236 2.315, 237 1.685, 238 0.615, 239 0.425, 240 0.815, 241 0.355, 242 0.655, 243 0.390, 244 4.455, 245 0.800, 246 0.685, 247 2.630, 248 0.325, 249 0.355, 250 0.740, 251 0.785, 252 0.795, 253 0.72, 254 0.80 255 ], 256 "without_flush_ms": [ 257 2.0, 258 1.99, 259 1.98, 260 1.97, 261 1.96, 262 1.95, 263 1.94, 264 1.93, 265 1.92, 266 1.91, 267 1.9, 268 1.89, 269 1.88, 270 1.87, 271 1.86, 272 1.85, 273 1.84, 274 1.83, 275 1.82, 276 1.81, 277 1.8, 278 1.79, 279 1.78, 280 1.77, 281 1.76, 282 1.75 283 ], 284 "with_flush_ms": [ 285 3.0, 286 2.9, 287 2.8, 288 2.7, 289 2.6, 290 2.5, 291 2.4, 292 2.3, 293 2.2, 294 2.1, 295 2.0, 296 1.9, 297 1.8, 298 1.7, 299 1.6, 300 1.5, 301 1.4, 302 1.3, 303 1.2, 304 1.1, 305 1.0, 306 0.9, 307 0.8, 308 0.7, 309 0.6, 310 0.5 311 ] 312}, 313"canvas_drawHugeGradient": { 314 "total_frame_ms": [ 315 210.555, 316 770.795, 317 10.430, 318 31.845, 319 3.610, 320 1.105, 321 0.545, 322 2.315, 323 1.685, 324 0.615, 325 0.425, 326 0.815, 327 0.355, 328 0.655, 329 0.390, 330 400.455, 331 0.800, 332 0.685, 333 2.630, 334 0.325, 335 0.355, 336 0.740, 337 0.785, 338 0.795, 339 0.72, 340 0.80 341 ], 342 "without_flush_ms": [ 343 5.0, 344 5.01, 345 5.02, 346 5.03, 347 5.04, 348 5.05, 349 5.06, 350 5.07, 351 5.08, 352 5.09, 353 5.1, 354 5.11, 355 5.12, 356 5.13, 357 5.14, 358 5.15, 359 5.16, 360 5.17, 361 5.18, 362 5.19, 363 5.2, 364 5.21, 365 5.22, 366 5.23, 367 5.24, 368 5.25 369 ], 370 "with_flush_ms": [ 371 5.0, 372 4.9, 373 4.8, 374 4.7, 375 4.6, 376 4.5, 377 4.4, 378 4.3, 379 4.2, 380 4.1, 381 4.0, 382 3.9, 383 3.8, 384 3.7, 385 3.6, 386 3.5, 387 3.4, 388 3.3, 389 3.2, 390 3.1, 391 3.0, 392 2.9, 393 2.8, 394 2.7, 395 2.6, 396 2.5 397 ] 398} 399}` 400