1// Copyright 2023 Google LLC 2// 3// Use of this source code is governed by a BSD-style license that can be 4// found in the LICENSE file. 5 6package common 7 8import ( 9 "context" 10 "path/filepath" 11 "testing" 12 "time" 13 14 "github.com/stretchr/testify/assert" 15 "github.com/stretchr/testify/require" 16 "go.skia.org/infra/go/gcs" 17 "go.skia.org/infra/go/gcs/mocks" 18 "go.skia.org/infra/go/now" 19 20 infra_testutils "go.skia.org/infra/go/testutils" 21 "go.skia.org/infra/task_driver/go/lib/os_steps" 22 "go.skia.org/infra/task_driver/go/td" 23 "go.skia.org/skia/infra/bots/task_drivers/testutils" 24) 25 26func TestComputeBenchmarkTestRunnerCLIFlags_Success(t *testing.T) { 27 test := func(name string, benchmarkInfo BenchmarkInfo, expectedFlags []string) { 28 t.Run(name, func(t *testing.T) { 29 actualFlags := ComputeBenchmarkTestRunnerCLIFlags(benchmarkInfo) 30 assert.Equal(t, expectedFlags, actualFlags) 31 }) 32 } 33 34 test("post-submit task", BenchmarkInfo{ 35 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 36 TaskName: "BazelTest-Foo-Bar", 37 TaskID: "1234567890", 38 }, []string{ 39 "--gitHash", "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 40 "--links", 41 "task", "https://task-scheduler.skia.org/task/1234567890", 42 }) 43 44 test("CL task", BenchmarkInfo{ 45 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 46 TaskName: "BazelTest-Foo-Bar", 47 TaskID: "1234567890", 48 ChangelistID: "12345", 49 PatchsetOrder: "3", 50 }, []string{ 51 "--gitHash", "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 52 "--issue", "12345", 53 "--patchset", "3", 54 "--links", 55 "task", "https://task-scheduler.skia.org/task/1234567890", 56 "changelist", "https://skia-review.googlesource.com/c/skia/+/12345/3", 57 }) 58} 59 60func TestUploadToPerf_NoOutputsZIPOrDir_Error(t *testing.T) { 61 test := func(name string, benchmarkInfo BenchmarkInfo) { 62 t.Run(name, func(t *testing.T) { 63 gcsClient := mocks.NewGCSClient(t) 64 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 65 err := UploadToPerf(ctx, gcsClient, benchmarkInfo, "/no/such/outputs.zip") 66 assert.Error(t, err) 67 assert.Contains(t, err.Error(), "stat /no/such/outputs.zip: no such file or directory") 68 return err 69 }) 70 71 require.Empty(t, res.Errors) 72 require.Empty(t, res.Exceptions) 73 testutils.AssertStepNames(t, res) // No steps. 74 75 gcsClient.AssertNotCalled(t, "Bucket") 76 gcsClient.AssertNotCalled(t, "SetFileContents") 77 gcsClient.AssertExpectations(t) 78 }) 79 } 80 81 test("post-submit task", BenchmarkInfo{ 82 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 83 TaskName: "BazelTest-Foo-Bar", 84 TaskID: "1234567890", 85 }) 86 87 test("CL task", BenchmarkInfo{ 88 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 89 TaskName: "BazelTest-Foo-Bar", 90 TaskID: "1234567890", 91 ChangelistID: "12345", 92 PatchsetOrder: "3", 93 }) 94} 95 96func TestUploadToPerf_OutputsZip_NoResultsJSONFile_Error(t *testing.T) { 97 test := func(name string, benchmarkInfo BenchmarkInfo) { 98 t.Run(name, func(t *testing.T) { 99 undeclaredTestOutputs := map[string]string{ 100 // Does not contain a results.json file. File contents do not matter for this test. 101 "some-image.png": "fake PNG", 102 "some-plaintext.txt": "fake TXT", 103 } 104 105 // Write undeclared test outputs to disk. 106 outputsZIP := filepath.Join(t.TempDir(), "outputs.zip") 107 testutils.MakeZIP(t, outputsZIP, undeclaredTestOutputs) 108 109 // Will be returned by the mocked os_steps.TempDir() when the task driver tries to create a 110 // directory in which to extract the undeclared outputs ZIP archive. 111 outputsZIPExtractionDir := t.TempDir() 112 113 gcsClient := mocks.NewGCSClient(t) 114 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 115 // We don't need to assert the exact number of times that os_steps.TempDir() is called 116 // because said function produces a "Creating TempDir" task driver step, and we check the 117 // exact set of steps produced. 118 ctx = context.WithValue(ctx, os_steps.TempDirContextKey, testutils.MakeTempDirMockFn(t, outputsZIPExtractionDir)) 119 120 err := UploadToPerf(ctx, gcsClient, benchmarkInfo, outputsZIP) 121 assert.Error(t, err) 122 assert.Contains(t, err.Error(), "stat "+outputsZIPExtractionDir+"/results.json: no such file or directory") 123 return err 124 }) 125 126 require.Empty(t, res.Errors) 127 require.Empty(t, res.Exceptions) 128 129 testutils.AssertStepNames(t, res, 130 "Creating TempDir", 131 "Extract undeclared outputs archive "+outputsZIP+" into "+outputsZIPExtractionDir, 132 "Extracting file: some-image.png", 133 "Not extracting non-PNG / non-JSON file: some-plaintext.txt", 134 "Stat "+outputsZIPExtractionDir+"/results.json", 135 ) 136 137 gcsClient.AssertNotCalled(t, "Bucket") 138 gcsClient.AssertNotCalled(t, "SetFileContents") 139 gcsClient.AssertExpectations(t) 140 }) 141 } 142 143 test("post-submit task", BenchmarkInfo{ 144 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 145 TaskName: "BazelTest-Foo-Bar", 146 TaskID: "1234567890", 147 }) 148 149 test("CL task", BenchmarkInfo{ 150 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 151 TaskName: "BazelTest-Foo-Bar", 152 TaskID: "1234567890", 153 ChangelistID: "12345", 154 PatchsetOrder: "3", 155 }) 156} 157 158func TestUploadToPerf_OutputsDirectory_NoResultsJSONFile_Error(t *testing.T) { 159 test := func(name string, benchmarkInfo BenchmarkInfo) { 160 t.Run(name, func(t *testing.T) { 161 undeclaredTestOutputs := map[string]string{ 162 // Does not contain a results.json file. File contents do not matter for this test. 163 "some-image.png": "fake PNG", 164 "some-plaintext.txt": "fake TXT", 165 } 166 167 // Write undeclared test outputs to disk. 168 outputsDir := t.TempDir() 169 testutils.PopulateDir(t, outputsDir, undeclaredTestOutputs) 170 171 gcsClient := mocks.NewGCSClient(t) 172 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 173 err := UploadToPerf(ctx, gcsClient, benchmarkInfo, outputsDir) 174 assert.Error(t, err) 175 assert.Contains(t, err.Error(), "stat "+outputsDir+"/results.json: no such file or directory") 176 return err 177 }) 178 179 require.Empty(t, res.Errors) 180 require.Empty(t, res.Exceptions) 181 182 testutils.AssertStepNames(t, res, "Stat "+outputsDir+"/results.json") 183 184 gcsClient.AssertNotCalled(t, "Bucket") 185 gcsClient.AssertNotCalled(t, "SetFileContents") 186 gcsClient.AssertExpectations(t) 187 }) 188 } 189 190 test("post-submit task", BenchmarkInfo{ 191 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 192 TaskName: "BazelTest-Foo-Bar", 193 TaskID: "1234567890", 194 }) 195 196 test("CL task", BenchmarkInfo{ 197 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 198 TaskName: "BazelTest-Foo-Bar", 199 TaskID: "1234567890", 200 ChangelistID: "12345", 201 PatchsetOrder: "3", 202 }) 203} 204 205func TestUploadToPerf_OutputsZip_Success(t *testing.T) { 206 test := func(name string, benchmarkInfo BenchmarkInfo) { 207 t.Run(name, func(t *testing.T) { 208 resultsJSONFileContents := `{"foo": "this test requires that this file exists; its contents do not matter"}` 209 undeclaredTestOutputs := map[string]string{ 210 "results.json": resultsJSONFileContents, 211 "some-image.png": "fake PNG", 212 "some-plaintext.txt": "fake TXT", 213 } 214 215 // Write undeclared test outputs to disk. 216 outputsZIP := filepath.Join(t.TempDir(), "outputs.zip") 217 testutils.MakeZIP(t, outputsZIP, undeclaredTestOutputs) 218 219 // Will be returned by the mocked os_steps.TempDir() when the task driver tries to create a 220 // directory in which to extract the undeclared outputs ZIP archive. 221 outputsZIPExtractionDir := t.TempDir() 222 223 gcsClient := mocks.NewGCSClient(t) 224 gcsClient.On("Bucket").Return("skia-perf") 225 gcsClient.On( 226 "SetFileContents", 227 infra_testutils.AnyContext, 228 "nano-json-v1/2022/01/31/01/ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99/BazelTest-Foo-Bar/results_1234567890.json", 229 gcs.FILE_WRITE_OPTS_TEXT, 230 []byte(resultsJSONFileContents)). 231 Return(nil).Once() 232 233 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 234 // Make sure we use UTC instead of the system timezone. The GCS path reflects the fact that 235 // we convert from UTC+1 to UTC. 236 fakeNow := time.Date(2022, time.January, 31, 2, 2, 3, 0, time.FixedZone("UTC+1", 60*60)) 237 ctx = now.TimeTravelingContext(fakeNow).WithContext(ctx) 238 239 // We don't need to assert the exact number of times that os_steps.TempDir() is called 240 // because said function produces a "Creating TempDir" task driver step, and we check the 241 // exact set of steps produced. 242 ctx = context.WithValue(ctx, os_steps.TempDirContextKey, testutils.MakeTempDirMockFn(t, outputsZIPExtractionDir)) 243 244 err := UploadToPerf(ctx, gcsClient, benchmarkInfo, outputsZIP) 245 assert.NoError(t, err) 246 return err 247 }) 248 249 require.Empty(t, res.Errors) 250 require.Empty(t, res.Exceptions) 251 252 testutils.AssertStepNames(t, res, 253 "Creating TempDir", 254 "Extract undeclared outputs archive "+outputsZIP+" into "+outputsZIPExtractionDir, 255 "Extracting file: results.json", 256 "Extracting file: some-image.png", 257 "Not extracting non-PNG / non-JSON file: some-plaintext.txt", 258 "Stat "+outputsZIPExtractionDir+"/results.json", 259 "Read "+outputsZIPExtractionDir+"/results.json", 260 "Upload gs://skia-perf/nano-json-v1/2022/01/31/01/ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99/BazelTest-Foo-Bar/results_1234567890.json", 261 ) 262 263 gcsClient.AssertExpectations(t) 264 }) 265 } 266 267 test("post-submit task", BenchmarkInfo{ 268 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 269 TaskName: "BazelTest-Foo-Bar", 270 TaskID: "1234567890", 271 }) 272 273 test("CL task", BenchmarkInfo{ 274 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 275 TaskName: "BazelTest-Foo-Bar", 276 TaskID: "1234567890", 277 ChangelistID: "12345", 278 PatchsetOrder: "3", 279 }) 280} 281 282func TestUploadToPerf_OutputsDirectory_Success(t *testing.T) { 283 test := func(name string, benchmarkInfo BenchmarkInfo) { 284 t.Run(name, func(t *testing.T) { 285 resultsJSONFileContents := `{"foo": "this test requires that this file exists; its contents do not matter"}` 286 undeclaredTestOutputs := map[string]string{ 287 "results.json": resultsJSONFileContents, 288 "some-image.png": "fake PNG", 289 "some-plaintext.txt": "fake TXT", 290 } 291 292 // Write undeclared test outputs to disk. 293 outputsDir := t.TempDir() 294 testutils.PopulateDir(t, outputsDir, undeclaredTestOutputs) 295 296 gcsClient := mocks.NewGCSClient(t) 297 gcsClient.On("Bucket").Return("skia-perf") 298 gcsClient.On( 299 "SetFileContents", 300 infra_testutils.AnyContext, 301 "nano-json-v1/2022/01/31/01/ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99/BazelTest-Foo-Bar/results_1234567890.json", 302 gcs.FILE_WRITE_OPTS_TEXT, 303 []byte(resultsJSONFileContents)). 304 Return(nil).Once() 305 306 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 307 // Make sure we use UTC instead of the system timezone. The GCS path reflects the fact that 308 // we convert from UTC+1 to UTC. 309 fakeNow := time.Date(2022, time.January, 31, 2, 2, 3, 0, time.FixedZone("UTC+1", 60*60)) 310 ctx = now.TimeTravelingContext(fakeNow).WithContext(ctx) 311 312 err := UploadToPerf(ctx, gcsClient, benchmarkInfo, outputsDir) 313 assert.NoError(t, err) 314 return err 315 }) 316 317 require.Empty(t, res.Errors) 318 require.Empty(t, res.Exceptions) 319 320 testutils.AssertStepNames(t, res, 321 "Stat "+outputsDir+"/results.json", 322 "Read "+outputsDir+"/results.json", 323 "Upload gs://skia-perf/nano-json-v1/2022/01/31/01/ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99/BazelTest-Foo-Bar/results_1234567890.json", 324 ) 325 326 gcsClient.AssertExpectations(t) 327 }) 328 } 329 330 test("post-submit task, directory", BenchmarkInfo{ 331 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 332 TaskName: "BazelTest-Foo-Bar", 333 TaskID: "1234567890", 334 }) 335 336 test("CL task, directory", BenchmarkInfo{ 337 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 338 TaskName: "BazelTest-Foo-Bar", 339 TaskID: "1234567890", 340 ChangelistID: "12345", 341 PatchsetOrder: "3", 342 }) 343} 344