1// Copyright 2020 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5package main 6 7import ( 8 "context" 9 "io/ioutil" 10 "os" 11 "path/filepath" 12 "testing" 13 14 "github.com/stretchr/testify/assert" 15 "github.com/stretchr/testify/require" 16 "go.skia.org/infra/go/exec" 17 "go.skia.org/infra/go/testutils" 18 "go.skia.org/infra/task_driver/go/td" 19) 20 21func TestSetup_NPMInitializedChromeStoppedBenchmarkOutCreated(t *testing.T) { 22 benchmarkPath, err := ioutil.TempDir("", "benchmark") 23 require.NoError(t, err) 24 defer testutils.RemoveAll(t, benchmarkPath) 25 26 const fakeNodeBinPath = "/fake/path/to/node/bin" 27 28 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 29 mock := exec.CommandCollector{} 30 ctx = td.WithExecRunFn(ctx, mock.Run) 31 err := setup(ctx, benchmarkPath, fakeNodeBinPath) 32 if err != nil { 33 assert.NoError(t, err) 34 return err 35 } 36 cmds := mock.Commands() 37 require.Len(t, cmds, 2) 38 cmd := cmds[0] 39 assert.Equal(t, "/fake/path/to/node/bin/npm", cmd.Name) 40 assert.Equal(t, []string{"ci"}, cmd.Args) 41 42 cmd = cmds[1] 43 assert.Equal(t, "killall", cmd.Name) 44 assert.Equal(t, []string{"chrome"}, cmd.Args) 45 return nil 46 }) 47 require.Empty(t, res.Errors) 48 require.Empty(t, res.Exceptions) 49 50 fi, err := os.Stat(filepath.Join(benchmarkPath, "out")) 51 require.NoError(t, err) 52 assert.True(t, fi.IsDir()) 53} 54 55func TestBenchSKPs_CPUHasNoUseGPUFlag(t *testing.T) { 56 skps, err := ioutil.TempDir("", "skps") 57 require.NoError(t, err) 58 defer testutils.RemoveAll(t, skps) 59 60 require.NoError(t, ioutil.WriteFile(filepath.Join(skps, "first_skp"), []byte("doesnt matter"), 0777)) 61 62 const fakeNodeBinPath = "/fake/path/to/node/bin" 63 const fakeCanvasKitPath = "/fake/path/to/canvaskit" 64 const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer" 65 66 perfObj := perfJSONFormat{ 67 Key: map[string]string{ 68 perfKeyCpuOrGPU: "CPU", 69 }, 70 } 71 72 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 73 mock := exec.CommandCollector{} 74 ctx = td.WithExecRunFn(ctx, mock.Run) 75 err := benchSKPs(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, skps, fakeNodeBinPath) 76 if err != nil { 77 assert.NoError(t, err) 78 return err 79 } 80 require.Len(t, mock.Commands(), 1) 81 cmd := mock.Commands()[0] 82 assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name) 83 assert.Equal(t, []string{"perf-canvaskit-with-puppeteer", 84 "--bench_html", "render-skp.html", 85 "--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js", 86 "--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm", 87 "--timeout", "90", 88 "--input_skp", filepath.Join(skps, "first_skp"), 89 "--output", "/fake/path/to/perf-puppeteer/out/first_skp.json"}, cmd.Args) 90 return nil 91 }) 92 require.Empty(t, res.Errors) 93 require.Empty(t, res.Exceptions) 94} 95 96func TestBenchSKPs_SkiplistIsUsed(t *testing.T) { 97 skps, err := ioutil.TempDir("", "skps") 98 require.NoError(t, err) 99 defer testutils.RemoveAll(t, skps) 100 101 require.NoError(t, ioutil.WriteFile(filepath.Join(skps, "desk_carsvg.skp"), []byte("doesnt matter"), 0777)) 102 103 const fakeNodeBinPath = "/fake/path/to/node/bin" 104 const fakeCanvasKitPath = "/fake/path/to/canvaskit" 105 const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer" 106 107 perfObj := perfJSONFormat{ 108 Key: map[string]string{ 109 perfKeyCpuOrGPU: "CPU", 110 }, 111 } 112 113 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 114 mock := exec.CommandCollector{} 115 ctx = td.WithExecRunFn(ctx, mock.Run) 116 err := benchSKPs(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, skps, fakeNodeBinPath) 117 if err != nil { 118 assert.NoError(t, err) 119 return err 120 } 121 // Should be skipped 122 require.Len(t, mock.Commands(), 0) 123 return nil 124 }) 125 require.Empty(t, res.Errors) 126 require.Empty(t, res.Exceptions) 127} 128 129func TestBenchSKPs_GPUHasFlag(t *testing.T) { 130 skps, err := ioutil.TempDir("", "skps") 131 require.NoError(t, err) 132 defer testutils.RemoveAll(t, skps) 133 134 require.NoError(t, ioutil.WriteFile(filepath.Join(skps, "first_skp"), []byte("doesnt matter"), 0777)) 135 136 const fakeNodeBinPath = "/fake/path/to/node/bin" 137 const fakeCanvasKitPath = "/fake/path/to/canvaskit" 138 const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer" 139 140 perfObj := perfJSONFormat{ 141 Key: map[string]string{ 142 perfKeyCpuOrGPU: "GPU", 143 }, 144 } 145 146 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 147 mock := exec.CommandCollector{} 148 ctx = td.WithExecRunFn(ctx, mock.Run) 149 err := benchSKPs(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, skps, fakeNodeBinPath) 150 if err != nil { 151 assert.NoError(t, err) 152 return err 153 } 154 require.Len(t, mock.Commands(), 1) 155 cmd := mock.Commands()[0] 156 assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name) 157 assert.Equal(t, []string{"perf-canvaskit-with-puppeteer", 158 "--bench_html", "render-skp.html", 159 "--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js", 160 "--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm", 161 "--timeout", "90", 162 "--input_skp", filepath.Join(skps, "first_skp"), 163 "--output", "/fake/path/to/perf-puppeteer/out/first_skp.json", 164 "--use_gpu"}, cmd.Args) 165 return nil 166 }) 167 require.Empty(t, res.Errors) 168 require.Empty(t, res.Exceptions) 169} 170 171func TestBenchSKPs_WebGL1(t *testing.T) { 172 skps, err := ioutil.TempDir("", "skps") 173 require.NoError(t, err) 174 defer testutils.RemoveAll(t, skps) 175 176 require.NoError(t, ioutil.WriteFile(filepath.Join(skps, "first_skp"), []byte("doesnt matter"), 0777)) 177 178 const fakeNodeBinPath = "/fake/path/to/node/bin" 179 const fakeCanvasKitPath = "/fake/path/to/canvaskit" 180 const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer" 181 182 perfObj := perfJSONFormat{ 183 Key: map[string]string{ 184 perfKeyCpuOrGPU: "GPU", 185 perfKeyWebGLVersion: "1", 186 }, 187 } 188 189 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 190 mock := exec.CommandCollector{} 191 ctx = td.WithExecRunFn(ctx, mock.Run) 192 err := benchSKPs(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, skps, fakeNodeBinPath) 193 if err != nil { 194 assert.NoError(t, err) 195 return err 196 } 197 require.Len(t, mock.Commands(), 1) 198 cmd := mock.Commands()[0] 199 assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name) 200 assert.Equal(t, []string{"perf-canvaskit-with-puppeteer", 201 "--bench_html", "render-skp.html", 202 "--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js", 203 "--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm", 204 "--timeout", "90", 205 "--input_skp", filepath.Join(skps, "first_skp"), 206 "--output", "/fake/path/to/perf-puppeteer/out/first_skp.json", 207 "--use_gpu", 208 "--query_params webgl1"}, cmd.Args) 209 return nil 210 }) 211 require.Empty(t, res.Errors) 212 require.Empty(t, res.Exceptions) 213} 214 215func TestProcessSkottieFramesData_GPUTwoInputsGetSummarizedAndCombined(t *testing.T) { 216 input, err := ioutil.TempDir("", "inputs") 217 require.NoError(t, err) 218 defer testutils.RemoveAll(t, input) 219 err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{ 220 "first_skp.json": firstSKP, 221 "second_skp.json": secondSKP, 222 }) 223 require.NoError(t, err) 224 output, err := ioutil.TempDir("", "perf") 225 require.NoError(t, err) 226 defer testutils.RemoveAll(t, output) 227 228 // These are based off of realistic values. 229 keys := map[string]string{ 230 "os": "Ubuntu18", 231 "model": "Golo", 232 perfKeyCpuOrGPU: "GPU", 233 "cpu_or_gpu_value": "QuadroP400", 234 } 235 236 perfObj, err := makePerfObj(someGitHash, someTaskID, someMachineID, keys) 237 require.NoError(t, err) 238 239 outputFile := filepath.Join(output, "perf-taskid1352.json") 240 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 241 return processSKPData(ctx, perfObj, input, outputFile) 242 }) 243 require.Empty(t, res.Errors) 244 require.Empty(t, res.Exceptions) 245 246 b, err := ioutil.ReadFile(outputFile) 247 require.NoError(t, err) 248 249 assert.Equal(t, `{ 250 "gitHash": "032631e490db494128e0610a19adce4cab9706d1", 251 "swarming_task_id": "4bdd43ed7c906c11", 252 "swarming_machine_id": "skia-e-gce-203", 253 "key": { 254 "arch": "wasm", 255 "binary": "CanvasKit", 256 "browser": "Chromium", 257 "configuration": "Release", 258 "cpu_or_gpu": "GPU", 259 "cpu_or_gpu_value": "QuadroP400", 260 "extra_config": "RenderSKP", 261 "model": "Golo", 262 "os": "Ubuntu18" 263 }, 264 "results": { 265 "first_skp": { 266 "webgl2": { 267 "avg_render_frame_ms": 150.065, 268 "avg_render_with_flush_ms": 133.91167, 269 "avg_render_without_flush_ms": 45.398335, 270 "median_render_frame_ms": 143.71, 271 "median_render_with_flush_ms": 125.185, 272 "median_render_without_flush_ms": 37.445, 273 "skp_load_ms": 1.715, 274 "stddev_render_frame_ms": 15.210527, 275 "stddev_render_with_flush_ms": 15.47429, 276 "stddev_render_without_flush_ms": 21.69691 277 } 278 }, 279 "second_skp": { 280 "webgl2": { 281 "avg_render_frame_ms": 316.7317, 282 "avg_render_with_flush_ms": 233.91167, 283 "avg_render_without_flush_ms": 85.39834, 284 "median_render_frame_ms": 243.71, 285 "median_render_with_flush_ms": 225.185, 286 "median_render_without_flush_ms": 67.445, 287 "skp_load_ms": 3.715, 288 "stddev_render_frame_ms": 109.164635, 289 "stddev_render_with_flush_ms": 15.474287, 290 "stddev_render_without_flush_ms": 43.27188 291 } 292 } 293 } 294}`, string(b)) 295} 296 297func writeFilesToDisk(path string, fileNamesToContent map[string]string) error { 298 if err := os.MkdirAll(path, 0777); err != nil { 299 return err 300 } 301 for name, content := range fileNamesToContent { 302 if err := ioutil.WriteFile(filepath.Join(path, name), []byte(content), 0666); err != nil { 303 return err 304 } 305 } 306 return nil 307} 308 309const ( 310 someGitHash = "032631e490db494128e0610a19adce4cab9706d1" 311 someTaskID = "4bdd43ed7c906c11" 312 someMachineID = "skia-e-gce-203" 313) 314 315const firstSKP = `{ 316 "without_flush_ms": [23.71, 37.445, 75.04], 317 "with_flush_ms": [125.185, 120.895, 155.655], 318 "total_frame_ms": [143.71, 135.445, 171.04], 319 "skp_load_ms":1.715 320}` 321 322const secondSKP = `{ 323 "without_flush_ms": [43.71, 67.445, 145.04], 324 "with_flush_ms": [225.185, 220.895, 255.655], 325 "total_frame_ms": [243.71, 235.445, 471.04], 326 "skp_load_ms":3.715 327}` 328