1// Copyright 2023 Google LLC 2// 3// Use of this source code is governed by a BSD-style license that can be 4// found in the LICENSE file. 5 6package main 7 8import ( 9 "context" 10 "strings" 11 "testing" 12 "time" 13 14 "github.com/stretchr/testify/assert" 15 "github.com/stretchr/testify/require" 16 "go.skia.org/infra/go/exec" 17 exec_testutils "go.skia.org/infra/go/exec/testutils" 18 "go.skia.org/infra/go/gcs" 19 "go.skia.org/infra/go/gcs/mocks" 20 "go.skia.org/infra/go/now" 21 infra_testutils "go.skia.org/infra/go/testutils" 22 "go.skia.org/infra/task_driver/go/lib/os_steps" 23 "go.skia.org/infra/task_driver/go/td" 24 "go.skia.org/skia/infra/bots/task_drivers/common" 25 "go.skia.org/skia/infra/bots/task_drivers/testutils" 26) 27 28func TestRun_UnitTest_Success(t *testing.T) { 29 commandCollector := exec.CommandCollector{} 30 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 31 ctx = td.WithExecRunFn(ctx, commandCollector.Run) 32 33 err := run(ctx, taskDriverArgs{ 34 commandPath: "/path/to/command", 35 commandWorkDir: "/path/to/workdir", 36 testKind: unitTest, 37 }) 38 39 assert.NoError(t, err) 40 return err 41 }) 42 43 require.Empty(t, res.Errors) 44 require.Empty(t, res.Exceptions) 45 46 testutils.AssertStepNames(t, res, "/path/to/command") 47 48 assert.Equal(t, "/path/to/workdir", commandCollector.Commands()[0].Dir) 49 exec_testutils.AssertCommandsMatch(t, [][]string{{"/path/to/command"}}, commandCollector.Commands()) 50} 51 52func TestRun_GMTest_Success(t *testing.T) { 53 // Given that we have tests for common.UploadToGold(), it suffices to test a couple of 54 // "happy" cases here. 55 56 test := func(name string, tdArgs taskDriverArgs, goldctlWorkDir string, goldctlImgtestInitStepName string, goldctlImgtestInitArgs []string) { 57 t.Run(name, func(t *testing.T) { 58 // Create directory with fake undeclared test outputs. 59 testutils.PopulateDir(t, tdArgs.undeclaredOutputsDir, map[string]string{ 60 // The contents of PNG files does not matter for this test. 61 "alfa.png": "fake PNG", 62 "alfa.json": `{ 63 "md5": "a01a01a01a01a01a01a01a01a01a01a0", 64 "keys": { 65 "build_system": "bazel", 66 "name": "alfa", 67 "source_type": "gm" 68 } 69 }`, 70 "beta.png": "fake PNG", 71 "beta.json": `{ 72 "md5": "b02b02b02b02b02b02b02b02b02b02b0", 73 "keys": { 74 "build_system": "bazel", 75 "name": "beta", 76 "source_type": "gm" 77 } 78 }`, 79 }) 80 81 commandCollector := exec.CommandCollector{} 82 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 83 ctx = td.WithExecRunFn(ctx, commandCollector.Run) 84 85 // We don't need to assert the exact number of times that os_steps.TempDir() is called 86 // because said function produces a "Creating TempDir" task driver step, and we check the 87 // exact set of steps produced. 88 ctx = context.WithValue(ctx, os_steps.TempDirContextKey, testutils.MakeTempDirMockFn(t, goldctlWorkDir)) 89 90 err := run(ctx, tdArgs) 91 92 assert.NoError(t, err) 93 return err 94 }) 95 96 require.Empty(t, res.Errors) 97 require.Empty(t, res.Exceptions) 98 99 testutils.AssertStepNames(t, res, 100 "/path/to/command --device-specific-bazel-config Pixel5 --key arch arm64 model Pixel5 os Android --gpuName Adreno620", 101 "Gather JSON and PNG files produced by GMs", 102 "Gather \"alfa.png\"", 103 "Gather \"beta.png\"", 104 "Upload GM outputs to Gold", 105 "Creating TempDir", 106 "/path/to/goldctl auth --work-dir "+goldctlWorkDir+" --luci", 107 goldctlImgtestInitStepName, 108 "/path/to/goldctl imgtest add --work-dir "+goldctlWorkDir+" --test-name alfa --png-file "+tdArgs.undeclaredOutputsDir+"/alfa.png --png-digest a01a01a01a01a01a01a01a01a01a01a0 --add-test-key build_system:bazel --add-test-key name:alfa --add-test-key source_type:gm", 109 "/path/to/goldctl imgtest add --work-dir "+goldctlWorkDir+" --test-name beta --png-file "+tdArgs.undeclaredOutputsDir+"/beta.png --png-digest b02b02b02b02b02b02b02b02b02b02b0 --add-test-key build_system:bazel --add-test-key name:beta --add-test-key source_type:gm", 110 "/path/to/goldctl imgtest finalize --work-dir "+goldctlWorkDir, 111 ) 112 113 assert.Equal(t, "/path/to/workdir", commandCollector.Commands()[0].Dir) 114 exec_testutils.AssertCommandsMatch(t, [][]string{ 115 { 116 "/path/to/command", 117 "--device-specific-bazel-config", "Pixel5", 118 "--key", 119 "arch", "arm64", 120 "model", "Pixel5", 121 "os", "Android", 122 "--gpuName", "Adreno620", 123 }, 124 { 125 "/path/to/goldctl", 126 "auth", 127 "--work-dir", goldctlWorkDir, 128 "--luci", 129 }, 130 append([]string{"/path/to/goldctl"}, goldctlImgtestInitArgs...), 131 { 132 "/path/to/goldctl", 133 "imgtest", 134 "add", 135 "--work-dir", goldctlWorkDir, 136 "--test-name", "alfa", 137 "--png-file", tdArgs.undeclaredOutputsDir + "/alfa.png", 138 "--png-digest", "a01a01a01a01a01a01a01a01a01a01a0", 139 "--add-test-key", "build_system:bazel", 140 "--add-test-key", "name:alfa", 141 "--add-test-key", "source_type:gm", 142 }, 143 { 144 "/path/to/goldctl", 145 "imgtest", 146 "add", 147 "--work-dir", goldctlWorkDir, 148 "--test-name", "beta", 149 "--png-file", tdArgs.undeclaredOutputsDir + "/beta.png", 150 "--png-digest", "b02b02b02b02b02b02b02b02b02b02b0", 151 "--add-test-key", "build_system:bazel", 152 "--add-test-key", "name:beta", 153 "--add-test-key", "source_type:gm", 154 }, 155 { 156 "/path/to/goldctl", 157 "imgtest", 158 "finalize", 159 "--work-dir", goldctlWorkDir, 160 }, 161 }, commandCollector.Commands()) 162 }) 163 } 164 165 goldctlWorkDir := t.TempDir() 166 test( 167 "post-submit task", 168 taskDriverArgs{ 169 UploadToGoldArgs: common.UploadToGoldArgs{ 170 TestOnlyAllowAnyBazelLabel: true, 171 BazelLabel: "//some/test:target", 172 DeviceSpecificBazelConfig: "Pixel5", 173 GoldctlPath: "/path/to/goldctl", 174 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 175 }, 176 commandPath: "/path/to/command", 177 commandWorkDir: "/path/to/workdir", 178 testKind: gmTest, 179 deviceSpecificBazelConfigName: "Pixel5", 180 undeclaredOutputsDir: t.TempDir(), 181 }, 182 goldctlWorkDir, 183 "/path/to/goldctl imgtest init --work-dir "+goldctlWorkDir+" --instance skia --url https://gold.skia.org --bucket skia-infra-gm --git_hash ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99 --key arch:arm64 --key model:Pixel5 --key os:Android", 184 []string{ 185 "imgtest", 186 "init", 187 "--work-dir", goldctlWorkDir, 188 "--instance", "skia", 189 "--url", "https://gold.skia.org", 190 "--bucket", "skia-infra-gm", 191 "--git_hash", "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 192 "--key", "arch:arm64", 193 "--key", "model:Pixel5", 194 "--key", "os:Android", 195 }, 196 ) 197 198 goldctlWorkDir = t.TempDir() 199 test( 200 "CL task", 201 taskDriverArgs{ 202 UploadToGoldArgs: common.UploadToGoldArgs{ 203 TestOnlyAllowAnyBazelLabel: true, 204 BazelLabel: "//some/test:target", 205 DeviceSpecificBazelConfig: "Pixel5", 206 GoldctlPath: "/path/to/goldctl", 207 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 208 ChangelistID: "changelist-id", 209 PatchsetOrder: "1", 210 TryjobID: "tryjob-id", 211 }, 212 commandPath: "/path/to/command", 213 commandWorkDir: "/path/to/workdir", 214 testKind: gmTest, 215 deviceSpecificBazelConfigName: "Pixel5", 216 undeclaredOutputsDir: t.TempDir(), 217 }, 218 goldctlWorkDir, 219 "/path/to/goldctl imgtest init --work-dir "+goldctlWorkDir+" --instance skia --url https://gold.skia.org --bucket skia-infra-gm --git_hash ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99 --crs gerrit --cis buildbucket --changelist changelist-id --patchset 1 --jobid tryjob-id --key arch:arm64 --key model:Pixel5 --key os:Android", 220 []string{ 221 "imgtest", 222 "init", 223 "--work-dir", goldctlWorkDir, 224 "--instance", "skia", 225 "--url", "https://gold.skia.org", 226 "--bucket", "skia-infra-gm", 227 "--git_hash", "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 228 "--crs", "gerrit", 229 "--cis", "buildbucket", 230 "--changelist", "changelist-id", 231 "--patchset", "1", 232 "--jobid", "tryjob-id", 233 "--key", "arch:arm64", 234 "--key", "model:Pixel5", 235 "--key", "os:Android", 236 }, 237 ) 238} 239 240func TestRun_BenchmarkTest_Success(t *testing.T) { 241 // Given that we have tests for common.UploadToPerf(), it suffices to test a couple of "happy" 242 // cases here. 243 244 test := func(name string, tdArgs taskDriverArgs, benchmarkInvocation []string) { 245 t.Run(name, func(t *testing.T) { 246 // Create directory with fake undeclared test outputs. 247 resultsJSONFileContents := `{"foo": "this test requires that this file exists; its contents do not matter"}` 248 testutils.PopulateDir(t, tdArgs.undeclaredOutputsDir, map[string]string{ 249 "results.json": resultsJSONFileContents, 250 "some-image.png": "fake PNG", 251 "some-plaintext.txt": "fake TXT", 252 }) 253 254 commandCollector := exec.CommandCollector{} 255 256 gcsClient := mocks.NewGCSClient(t) 257 gcsClient.On("Bucket").Return("skia-perf") 258 gcsClient.On( 259 "SetFileContents", 260 infra_testutils.AnyContext, 261 "nano-json-v1/2022/01/31/01/ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99/BazelTest-Foo-Bar/results_1234567890.json", 262 gcs.FILE_WRITE_OPTS_TEXT, 263 []byte(resultsJSONFileContents)). 264 Return(nil).Once() 265 tdArgs.gcsClient = gcsClient 266 267 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 268 // Make sure we use UTC instead of the system timezone. The GCS path reflects the fact that 269 // we convert from UTC+1 to UTC. 270 fakeNow := time.Date(2022, time.January, 31, 2, 2, 3, 0, time.FixedZone("UTC+1", 60*60)) 271 ctx = now.TimeTravelingContext(fakeNow).WithContext(ctx) 272 ctx = td.WithExecRunFn(ctx, commandCollector.Run) 273 274 err := run(ctx, tdArgs) 275 276 assert.NoError(t, err) 277 return err 278 }) 279 280 require.Empty(t, res.Errors) 281 require.Empty(t, res.Exceptions) 282 283 testutils.AssertStepNames(t, res, 284 strings.Join(benchmarkInvocation, " "), 285 "Stat "+tdArgs.undeclaredOutputsDir+"/results.json", 286 "Read "+tdArgs.undeclaredOutputsDir+"/results.json", 287 "Upload gs://skia-perf/nano-json-v1/2022/01/31/01/ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99/BazelTest-Foo-Bar/results_1234567890.json", 288 ) 289 290 assert.Equal(t, "/path/to/workdir", commandCollector.Commands()[0].Dir) 291 exec_testutils.AssertCommandsMatch(t, [][]string{benchmarkInvocation}, commandCollector.Commands()) 292 }) 293 } 294 295 test( 296 "post-submit task", 297 taskDriverArgs{ 298 BenchmarkInfo: common.BenchmarkInfo{ 299 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 300 TaskName: "BazelTest-Foo-Bar", 301 TaskID: "1234567890", 302 }, 303 commandPath: "/path/to/command", 304 commandWorkDir: "/path/to/workdir", 305 testKind: benchmarkTest, 306 deviceSpecificBazelConfigName: "Pixel5", 307 undeclaredOutputsDir: t.TempDir(), 308 }, 309 []string{ 310 "/path/to/command", 311 "--gitHash", "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 312 "--links", 313 "task", "https://task-scheduler.skia.org/task/1234567890", 314 "--device-specific-bazel-config", "Pixel5", 315 "--key", 316 "arch", "arm64", 317 "model", "Pixel5", 318 "os", "Android", 319 "--gpuName", "Adreno620", 320 }, 321 ) 322 323 test( 324 "CL task", 325 taskDriverArgs{ 326 BenchmarkInfo: common.BenchmarkInfo{ 327 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 328 TaskName: "BazelTest-Foo-Bar", 329 TaskID: "1234567890", 330 ChangelistID: "12345", 331 PatchsetOrder: "3", 332 }, 333 commandPath: "/path/to/command", 334 commandWorkDir: "/path/to/workdir", 335 testKind: benchmarkTest, 336 deviceSpecificBazelConfigName: "Pixel5", 337 undeclaredOutputsDir: t.TempDir(), 338 }, 339 []string{ 340 "/path/to/command", 341 "--gitHash", "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 342 "--issue", "12345", 343 "--patchset", "3", 344 "--links", 345 "task", "https://task-scheduler.skia.org/task/1234567890", 346 "changelist", "https://skia-review.googlesource.com/c/skia/+/12345/3", 347 "--device-specific-bazel-config", "Pixel5", 348 "--key", 349 "arch", "arm64", 350 "model", "Pixel5", 351 "os", "Android", 352 "--gpuName", "Adreno620", 353 }, 354 ) 355} 356