1// Copyright 2023 Google LLC 2// 3// Use of this source code is governed by a BSD-style license that can be 4// found in the LICENSE file. 5 6package common 7 8import ( 9 "context" 10 "os" 11 "path/filepath" 12 "testing" 13 14 exec_testutils "go.skia.org/infra/go/exec/testutils" 15 16 "github.com/stretchr/testify/assert" 17 "github.com/stretchr/testify/require" 18 "go.skia.org/infra/go/exec" 19 "go.skia.org/infra/task_driver/go/lib/os_steps" 20 "go.skia.org/infra/task_driver/go/td" 21 "go.skia.org/skia/infra/bots/task_drivers/testutils" 22) 23 24func TestUploadToGold_NoOutputsZIPOrDir_NoGoldctlInvocations(t *testing.T) { 25 test := func(name string, utgArgs UploadToGoldArgs) { 26 t.Run(name, func(t *testing.T) { 27 commandCollector := exec.CommandCollector{} 28 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 29 ctx = td.WithExecRunFn(ctx, commandCollector.Run) 30 31 err := UploadToGold(ctx, utgArgs, "/path/to/skia/bazel-testlogs/some/test/target/test.outputs/outputs.zip") 32 33 assert.NoError(t, err) 34 return err 35 }) 36 37 require.Empty(t, res.Errors) 38 require.Empty(t, res.Exceptions) 39 testutils.AssertStepNames(t, res, 40 "Test did not produce an undeclared test outputs ZIP file or directory; nothing to upload to Gold", 41 ) 42 43 assert.Empty(t, commandCollector.Commands()) 44 }) 45 } 46 47 test("post-submit task", UploadToGoldArgs{ 48 TestOnlyAllowAnyBazelLabel: true, 49 BazelLabel: "//some/test:target", 50 DeviceSpecificBazelConfig: "Pixel5", 51 GoldctlPath: "/path/to/goldctl", 52 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 53 }) 54 55 test("CL task", UploadToGoldArgs{ 56 TestOnlyAllowAnyBazelLabel: true, 57 BazelLabel: "//some/test:target", 58 DeviceSpecificBazelConfig: "Pixel5", 59 GoldctlPath: "/path/to/goldctl", 60 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 61 62 // These arguments are only used when there are images to upload, and are therefore 63 // ignored by the task driver under this test. 64 ChangelistID: "changelist-id", 65 PatchsetOrder: "1", 66 TryjobID: "tryjob-id", 67 }) 68} 69 70func TestUploadToGold_WithOutputsZIPOrDir_NoValidImages_NoGoldctlInvocations(t *testing.T) { 71 test := func(name string, zip bool, utgArgs UploadToGoldArgs) { 72 t.Run(name, func(t *testing.T) { 73 undeclaredTestOutputs := map[string]string{ 74 // The contents of PNG files does not matter for this test. 75 "image-with-invalid-json-file.png": "fake PNG", 76 "image-with-invalid-json-file.json": `{ 77 "invalid JSON file": "This JSON file should be ignored" 78 }`, 79 "image-with-no-json-file.png": "fake PNG", 80 "json-file-with-no-image.json": `{ 81 "md5": "00000000000000000000000000000000", 82 "keys": { 83 "name": "no-image", 84 "source_type": "no-corpus" 85 } 86 }`, 87 "not-an-image-nor-json-file.txt": "I'm neither a PNG nor a JSON file.", 88 } 89 90 // Write undeclared test outputs to disk. 91 outputsZIPOrDir := "" 92 if zip { 93 outputsZIPOrDir = filepath.Join(t.TempDir(), "bazel-testlogs", "some", "test", "target", "test.outputs", "outputs.zip") 94 require.NoError(t, os.MkdirAll(filepath.Dir(outputsZIPOrDir), 0700)) 95 testutils.MakeZIP(t, outputsZIPOrDir, undeclaredTestOutputs) 96 } else { 97 outputsZIPOrDir = t.TempDir() 98 testutils.PopulateDir(t, outputsZIPOrDir, undeclaredTestOutputs) 99 } 100 101 // Will be returned by the mocked os_steps.TempDir() when the task driver tries to create a 102 // directory in which to extract the undeclared outputs ZIP archive. 103 outputsZIPExtractionDir := t.TempDir() 104 105 commandCollector := exec.CommandCollector{} 106 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 107 ctx = td.WithExecRunFn(ctx, commandCollector.Run) 108 109 if zip { 110 // Mock os_steps.TempDir() only for the case where outpusZIPOrDir is a ZIP archive. We 111 // don't need to assert the exact number of times that os_steps.TempDir() is called 112 // because said function produces a "Creating TempDir" task driver step, and we check the 113 // exact set of steps produced. 114 ctx = context.WithValue(ctx, os_steps.TempDirContextKey, testutils.MakeTempDirMockFn(t, outputsZIPExtractionDir)) 115 } 116 117 err := UploadToGold(ctx, utgArgs, outputsZIPOrDir) 118 119 assert.NoError(t, err) 120 return err 121 }) 122 123 require.Empty(t, res.Errors) 124 require.Empty(t, res.Exceptions) 125 126 expectedSteps := []string{} 127 if zip { 128 expectedSteps = append(expectedSteps, 129 "Creating TempDir", 130 "Extract undeclared outputs archive "+outputsZIPOrDir+" into "+outputsZIPExtractionDir, 131 "Extracting file: image-with-invalid-json-file.json", 132 "Extracting file: image-with-invalid-json-file.png", 133 "Extracting file: image-with-no-json-file.png", 134 "Extracting file: json-file-with-no-image.json", 135 "Not extracting non-PNG / non-JSON file: not-an-image-nor-json-file.txt", 136 ) 137 } 138 expectedSteps = append(expectedSteps, 139 "Gather JSON and PNG files produced by GMs", 140 "Ignoring \"image-with-invalid-json-file.json\": field \"md5\" not found", 141 "Ignoring \"json-file-with-no-image.json\": file \"json-file-with-no-image.png\" not found", 142 "Undeclared test outputs ZIP file or directory contains no GM outputs; nothing to upload to Gold", 143 ) 144 testutils.AssertStepNames(t, res, expectedSteps...) 145 146 assert.Empty(t, commandCollector.Commands()) 147 }) 148 } 149 150 postSubmitTaskArgs := UploadToGoldArgs{ 151 TestOnlyAllowAnyBazelLabel: true, 152 BazelLabel: "//some/test:target", 153 DeviceSpecificBazelConfig: "Pixel5", 154 GoldctlPath: "/path/to/goldctl", 155 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 156 } 157 test("post-submit task, ZIP file", true /* =zip */, postSubmitTaskArgs) 158 test("post-submit task, directory", false /* =zip */, postSubmitTaskArgs) 159 160 clTaskArgs := UploadToGoldArgs{ 161 TestOnlyAllowAnyBazelLabel: true, 162 BazelLabel: "//some/test:target", 163 DeviceSpecificBazelConfig: "Pixel5", 164 GoldctlPath: "/path/to/goldctl", 165 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 166 167 // These arguments are only used when there are images to upload, and are therefore 168 // ignored by the task driver under this test. 169 ChangelistID: "changelist-id", 170 PatchsetOrder: "1", 171 TryjobID: "tryjob-id", 172 } 173 test("CL task, ZIP file", true /* =zip */, clTaskArgs) 174 test("CL task, directory", false /* =zip */, clTaskArgs) 175} 176 177func TestUploadToGold_WithOutputsZIPOrDir_ValidImages_ImagesUploadedToGold(t *testing.T) { 178 test := func(name string, zip bool, utgArgs UploadToGoldArgs, goldctlWorkDir, goldctlImgtestInitStepName string, goldctlImgtestInitArgs []string) { 179 t.Run(name, func(t *testing.T) { 180 undeclaredTestOutputs := map[string]string{ 181 // The contents of PNG files does not matter for this test. 182 "alfa.png": "fake PNG", 183 "alfa.json": `{ 184 "md5": "a01a01a01a01a01a01a01a01a01a01a0", 185 "keys": { 186 "build_system": "bazel", 187 "name": "alfa", 188 "source_type": "gm" 189 } 190 }`, 191 "beta.png": "fake PNG", 192 "beta.json": `{ 193 "md5": "b02b02b02b02b02b02b02b02b02b02b0", 194 "keys": { 195 "build_system": "bazel", 196 "name": "beta", 197 "source_type": "gm" 198 } 199 }`, 200 "image-with-invalid-json-file.png": "fake PNG", 201 "image-with-invalid-json-file.json": `{ 202 "invalid JSON file": "This JSON file should be ignored" 203 }`, 204 "image-with-no-json-file.png": "fake PNG", 205 "json-file-with-no-image.json": `{ 206 "md5": "00000000000000000000000000000000", 207 "keys": { 208 "name": "no-image", 209 "source_type": "no-corpus" 210 } 211 }`, 212 "not-an-image-nor-json-file.txt": "I'm neither a PNG nor a JSON file.", 213 } 214 215 // Write undeclared test outputs to disk. 216 outpusZIPOrDir := "" 217 if zip { 218 outpusZIPOrDir = filepath.Join(t.TempDir(), "bazel-testlogs", "some", "test", "target", "test.outputs", "outputs.zip") 219 require.NoError(t, os.MkdirAll(filepath.Dir(outpusZIPOrDir), 0700)) 220 testutils.MakeZIP(t, outpusZIPOrDir, undeclaredTestOutputs) 221 } else { 222 outpusZIPOrDir = t.TempDir() 223 testutils.PopulateDir(t, outpusZIPOrDir, undeclaredTestOutputs) 224 } 225 226 // Will be returned by the mocked os_steps.TempDir() when the task driver tries to create a 227 // directory in which to extract the undeclared outputs ZIP archive. 228 outputsZIPExtractionDir := t.TempDir() 229 230 commandCollector := exec.CommandCollector{} 231 res := td.RunTestSteps(t, false, func(ctx context.Context) error { 232 ctx = td.WithExecRunFn(ctx, commandCollector.Run) 233 234 tempDirMockFn := testutils.MakeTempDirMockFn(t, goldctlWorkDir) 235 if zip { 236 tempDirMockFn = testutils.MakeTempDirMockFn(t, outputsZIPExtractionDir, goldctlWorkDir) 237 } 238 // We don't need to assert the exact number of times that os_steps.TempDir() is called 239 // because said function produces a "Creating TempDir" task driver step, and we check the 240 // exact set of steps produced. 241 ctx = context.WithValue(ctx, os_steps.TempDirContextKey, tempDirMockFn) 242 243 err := UploadToGold(ctx, utgArgs, outpusZIPOrDir) 244 245 assert.NoError(t, err) 246 return err 247 }) 248 249 require.Empty(t, res.Errors) 250 require.Empty(t, res.Exceptions) 251 252 dirWithTestOutputs := outpusZIPOrDir 253 if zip { 254 dirWithTestOutputs = outputsZIPExtractionDir 255 } 256 257 expectedSteps := []string{} 258 if zip { 259 expectedSteps = append(expectedSteps, 260 "Creating TempDir", 261 "Extract undeclared outputs archive "+outpusZIPOrDir+" into "+outputsZIPExtractionDir, 262 "Extracting file: alfa.json", 263 "Extracting file: alfa.png", 264 "Extracting file: beta.json", 265 "Extracting file: beta.png", 266 "Extracting file: image-with-invalid-json-file.json", 267 "Extracting file: image-with-invalid-json-file.png", 268 "Extracting file: image-with-no-json-file.png", 269 "Extracting file: json-file-with-no-image.json", 270 "Not extracting non-PNG / non-JSON file: not-an-image-nor-json-file.txt", 271 ) 272 } 273 expectedSteps = append(expectedSteps, 274 "Gather JSON and PNG files produced by GMs", 275 "Gather \"alfa.png\"", 276 "Gather \"beta.png\"", 277 "Ignoring \"image-with-invalid-json-file.json\": field \"md5\" not found", 278 "Ignoring \"json-file-with-no-image.json\": file \"json-file-with-no-image.png\" not found", 279 "Upload GM outputs to Gold", 280 "Creating TempDir", 281 "/path/to/goldctl auth --work-dir "+goldctlWorkDir+" --luci", 282 goldctlImgtestInitStepName, 283 "/path/to/goldctl imgtest add --work-dir "+goldctlWorkDir+" --test-name alfa --png-file "+dirWithTestOutputs+"/alfa.png --png-digest a01a01a01a01a01a01a01a01a01a01a0 --add-test-key build_system:bazel --add-test-key name:alfa --add-test-key source_type:gm", 284 "/path/to/goldctl imgtest add --work-dir "+goldctlWorkDir+" --test-name beta --png-file "+dirWithTestOutputs+"/beta.png --png-digest b02b02b02b02b02b02b02b02b02b02b0 --add-test-key build_system:bazel --add-test-key name:beta --add-test-key source_type:gm", 285 "/path/to/goldctl imgtest finalize --work-dir "+goldctlWorkDir, 286 ) 287 testutils.AssertStepNames(t, res, expectedSteps...) 288 289 exec_testutils.AssertCommandsMatch(t, [][]string{ 290 { 291 "/path/to/goldctl", 292 "auth", 293 "--work-dir", goldctlWorkDir, 294 "--luci", 295 }, 296 append([]string{"/path/to/goldctl"}, goldctlImgtestInitArgs..., 297 ), 298 { 299 "/path/to/goldctl", 300 "imgtest", 301 "add", 302 "--work-dir", goldctlWorkDir, 303 "--test-name", "alfa", 304 "--png-file", dirWithTestOutputs + "/alfa.png", 305 "--png-digest", "a01a01a01a01a01a01a01a01a01a01a0", 306 "--add-test-key", "build_system:bazel", 307 "--add-test-key", "name:alfa", 308 "--add-test-key", "source_type:gm", 309 }, 310 { 311 "/path/to/goldctl", 312 "imgtest", 313 "add", 314 "--work-dir", goldctlWorkDir, 315 "--test-name", "beta", 316 "--png-file", dirWithTestOutputs + "/beta.png", 317 "--png-digest", "b02b02b02b02b02b02b02b02b02b02b0", 318 "--add-test-key", "build_system:bazel", 319 "--add-test-key", "name:beta", 320 "--add-test-key", "source_type:gm", 321 }, 322 { 323 "/path/to/goldctl", 324 "imgtest", 325 "finalize", 326 "--work-dir", goldctlWorkDir, 327 }, 328 }, commandCollector.Commands()) 329 330 }) 331 } 332 333 goldctlWorkDir := t.TempDir() 334 postSubmitTaskArgs := UploadToGoldArgs{ 335 TestOnlyAllowAnyBazelLabel: true, 336 BazelLabel: "//some/test:target", 337 DeviceSpecificBazelConfig: "Pixel5", 338 GoldctlPath: "/path/to/goldctl", 339 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 340 } 341 postSubmitTaskGoldctlImgtestInitStep := "/path/to/goldctl imgtest init --work-dir " + goldctlWorkDir + " --instance skia --url https://gold.skia.org --bucket skia-infra-gm --git_hash ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99 --key arch:arm64 --key model:Pixel5 --key os:Android" 342 postSubmitTaskGoldctlImgtestInitArgs := []string{ 343 "imgtest", 344 "init", 345 "--work-dir", goldctlWorkDir, 346 "--instance", "skia", 347 "--url", "https://gold.skia.org", 348 "--bucket", "skia-infra-gm", 349 "--git_hash", "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 350 "--key", "arch:arm64", 351 "--key", "model:Pixel5", 352 "--key", "os:Android", 353 } 354 test("post-submit task, ZIP file", true /* =zip */, postSubmitTaskArgs, goldctlWorkDir, postSubmitTaskGoldctlImgtestInitStep, postSubmitTaskGoldctlImgtestInitArgs) 355 test("post-submit task, directory", false /* =zip */, postSubmitTaskArgs, goldctlWorkDir, postSubmitTaskGoldctlImgtestInitStep, postSubmitTaskGoldctlImgtestInitArgs) 356 357 goldctlWorkDir = t.TempDir() 358 clTaskArgs := UploadToGoldArgs{ 359 TestOnlyAllowAnyBazelLabel: true, 360 BazelLabel: "//some/test:target", 361 DeviceSpecificBazelConfig: "Pixel5", 362 GoldctlPath: "/path/to/goldctl", 363 GitCommit: "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 364 ChangelistID: "changelist-id", 365 PatchsetOrder: "1", 366 TryjobID: "tryjob-id", 367 } 368 clTaskGoldctlImgtestInitStep := "/path/to/goldctl imgtest init --work-dir " + goldctlWorkDir + " --instance skia --url https://gold.skia.org --bucket skia-infra-gm --git_hash ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99 --crs gerrit --cis buildbucket --changelist changelist-id --patchset 1 --jobid tryjob-id --key arch:arm64 --key model:Pixel5 --key os:Android" 369 clTaskGoldctlImgtestInitArgs := []string{ 370 "imgtest", 371 "init", 372 "--work-dir", goldctlWorkDir, 373 "--instance", "skia", 374 "--url", "https://gold.skia.org", 375 "--bucket", "skia-infra-gm", 376 "--git_hash", "ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99", 377 "--crs", "gerrit", 378 "--cis", "buildbucket", 379 "--changelist", "changelist-id", 380 "--patchset", "1", 381 "--jobid", "tryjob-id", 382 "--key", "arch:arm64", 383 "--key", "model:Pixel5", 384 "--key", "os:Android", 385 } 386 test("CL task, ZIP file", true /* =zip */, clTaskArgs, goldctlWorkDir, clTaskGoldctlImgtestInitStep, clTaskGoldctlImgtestInitArgs) 387 test("CL task, directory", false /* =zip */, clTaskArgs, goldctlWorkDir, clTaskGoldctlImgtestInitStep, clTaskGoldctlImgtestInitArgs) 388} 389