1// Copyright 2022 Google LLC 2// 3// Use of this source code is governed by a BSD-style license that can be 4// found in the LICENSE file. 5// 6// This executable runs a Bazel(isk) build command for a single label using the provided 7// config (which is assumed to be in //bazel/buildrc) and any provided Bazel args. 8// This handles any setup needed to run Bazel on our CI machines before running the task, like 9// setting up logs and the Bazel cache. 10package main 11 12import ( 13 "context" 14 "flag" 15 "fmt" 16 "path/filepath" 17 "strings" 18 19 sk_exec "go.skia.org/infra/go/exec" 20 "go.skia.org/infra/task_driver/go/lib/bazel" 21 "go.skia.org/infra/task_driver/go/lib/os_steps" 22 "go.skia.org/infra/task_driver/go/td" 23 "go.skia.org/skia/infra/bots/task_drivers/common" 24) 25 26var ( 27 // Required properties for this task. 28 // We want the cache to be on a bigger disk than default. The root disk, where the home 29 // directory (and default Bazel cache) lives, is only 15 GB on our GCE VMs. 30 gitPath = flag.String("git_path", "", "Location of git binary to use for diffs.") 31 projectId = flag.String("project_id", "", "ID of the Google Cloud project.") 32 taskId = flag.String("task_id", "", "ID of this task.") 33 taskName = flag.String("task_name", "", "Name of the task.") 34 workdir = flag.String("workdir", ".", "Working directory, the root directory of a full Skia checkout") 35 // Optional flags. 36 local = flag.Bool("local", false, "True if running locally (as opposed to on the CI/CQ)") 37 output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.") 38) 39 40func main() { 41 bazelFlags := common.MakeBazelFlags(common.MakeBazelFlagsOpts{ 42 AdditionalArgs: true, 43 }) 44 45 // StartRun calls flag.Parse() 46 ctx := td.StartRun(projectId, taskId, taskName, output, local) 47 defer td.EndRun(ctx) 48 49 bazelFlags.Validate(ctx) 50 51 if *gitPath == "" { 52 td.Fatal(ctx, fmt.Errorf("--git_path is required")) 53 } 54 55 wd, err := os_steps.Abs(ctx, *workdir) 56 if err != nil { 57 td.Fatal(ctx, err) 58 } 59 absGit, err := os_steps.Abs(ctx, *gitPath) 60 if err != nil { 61 td.Fatal(ctx, err) 62 } 63 if _, err := os_steps.Stat(ctx, absGit); err != nil { 64 fmt.Printf("Cannot stat git binary %s\n", absGit) 65 td.Fatal(ctx, err) 66 } 67 skiaPath := filepath.Join(wd, "skia") 68 69 // When running on the CI, there is not a git checkout here, so we make a temp one. 70 if !*local { 71 if err := gitInit(ctx, absGit, skiaPath); err != nil { 72 td.Fatal(ctx, err) 73 } 74 } 75 76 opts := bazel.BazelOptions{ 77 CachePath: *bazelFlags.CacheDir, 78 } 79 if err := bazel.EnsureBazelRCFile(ctx, opts); err != nil { 80 td.Fatal(ctx, err) 81 } 82 83 if err := bazelRun(ctx, skiaPath, "//bazel/deps_parser", *bazelFlags.AdditionalArgs...); err != nil { 84 td.Fatal(ctx, err) 85 } 86 87 if err := bazelRun(ctx, skiaPath, "//tools/gpu/gl/interface:generate_gl_interfaces", *bazelFlags.AdditionalArgs...); err != nil { 88 td.Fatal(ctx, err) 89 } 90 91 skslFlags := append([]string{"--config=compile_sksl"}, *bazelFlags.AdditionalArgs...) 92 skslTests := []string{ 93 "compile_hlsl_tests", 94 "compile_glsl_tests", 95 "compile_glsl_nosettings_tests", 96 "compile_metal_tests", 97 "compile_skrp_tests", 98 "compile_stage_tests", 99 "compile_spirv_tests", 100 "compile_wgsl_tests", 101 } 102 for _, label := range skslTests { 103 if err := bazelRun(ctx, skiaPath, "//tools/skslc:"+label, skslFlags...); err != nil { 104 td.Fatal(ctx, err) 105 } 106 } 107 108 if err := bazelRun(ctx, skiaPath, "//tools:generate_workarounds", *bazelFlags.AdditionalArgs...); err != nil { 109 td.Fatal(ctx, err) 110 } 111 112 if err := bazelRun(ctx, skiaPath, "//tools/sksl-minify:minify_srcs", skslFlags...); err != nil { 113 td.Fatal(ctx, err) 114 } 115 116 if err := bazelRun(ctx, skiaPath, "//tools/sksl-minify:minify_tests", skslFlags...); err != nil { 117 td.Fatal(ctx, err) 118 } 119 120 if err := generateGNIFiles(ctx, skiaPath); err != nil { 121 td.Fatal(ctx, err) 122 } 123 124 if err := gazelle(ctx, skiaPath); err != nil { 125 td.Fatal(ctx, err) 126 } 127 128 if err := gazelleDeps(ctx, skiaPath); err != nil { 129 td.Fatal(ctx, err) 130 } 131 132 if err := bazelRun(ctx, skiaPath, "//:buildifier", *bazelFlags.AdditionalArgs...); err != nil { 133 td.Fatal(ctx, err) 134 } 135 136 if err := bazelRun(ctx, skiaPath, "//:go", append(*bazelFlags.AdditionalArgs, "--", "generate", "./...")...); err != nil { 137 td.Fatal(ctx, err) 138 } 139 140 if err := checkGitDiff(ctx, absGit, skiaPath); err != nil { 141 td.Fatal(ctx, err) 142 } 143 144 if !*local { 145 if err := common.BazelCleanIfLowDiskSpace(ctx, *bazelFlags.CacheDir, skiaPath, "bazelisk"); err != nil { 146 td.Fatal(ctx, err) 147 } 148 } 149} 150 151// bazelRun runs the given Bazel label from the Skia path with any given args using Bazelisk. 152func bazelRun(ctx context.Context, skiaPath, label string, args ...string) error { 153 return td.Do(ctx, td.Props("bazel run "+label), func(ctx context.Context) error { 154 runCmd := &sk_exec.Command{ 155 Name: "bazelisk", 156 Args: append([]string{"run", label}, args...), 157 InheritEnv: true, // Need to make sure bazelisk is on the path 158 Dir: skiaPath, 159 LogStdout: true, 160 LogStderr: true, 161 } 162 _, err := sk_exec.RunCommand(ctx, runCmd) 163 if err != nil { 164 return err 165 } 166 return nil 167 }) 168} 169 170// gitInit creates a temporary git repository with all files in the Skia path. This allows the later 171// git diff call to work properly. This is necessary because our current Swarming setup does not 172// include the .git folder when copying down files. 173func gitInit(ctx context.Context, gitPath, skiaPath string) error { 174 step := fmt.Sprintf("Setting git baseline in %s", skiaPath) 175 err := td.Do(ctx, td.Props(step), func(ctx context.Context) error { 176 initCmd := &sk_exec.Command{ 177 Name: gitPath, 178 Args: []string{"init"}, 179 InheritEnv: false, 180 Dir: skiaPath, 181 LogStdout: true, 182 LogStderr: true, 183 } 184 if _, err := sk_exec.RunCommand(ctx, initCmd); err != nil { 185 return err 186 } 187 addCmd := &sk_exec.Command{ 188 Name: gitPath, 189 Args: []string{"add", "."}, 190 InheritEnv: false, 191 Dir: skiaPath, 192 LogStdout: true, 193 LogStderr: true, 194 } 195 if _, err := sk_exec.RunCommand(ctx, addCmd); err != nil { 196 return err 197 } 198 commitCmd := &sk_exec.Command{ 199 Name: gitPath, 200 Args: []string{"commit", "-m", "baseline commit"}, 201 InheritEnv: false, 202 Dir: skiaPath, 203 LogStdout: true, 204 LogStderr: true, 205 } 206 if _, err := sk_exec.RunCommand(ctx, commitCmd); err != nil { 207 return err 208 } 209 return nil 210 }) 211 return err 212} 213 214// generateGNIFiles re-generates the .gni files from BUILD.bazel files, allowing interopt between 215// the new system (Bazel) and the old one GN. 216func generateGNIFiles(ctx context.Context, skiaPath string) error { 217 return td.Do(ctx, td.Props("Generate GNI files from BUILD.bazel ones"), func(ctx context.Context) error { 218 // Note: This is not done with bazel run ... because the exporter_tool calls Bazel, causing 219 // an apparent deadlock because there can only be one running bazel task at a time. 220 runCmd := &sk_exec.Command{ 221 Name: "make", 222 Args: []string{"-C", "bazel", "generate_gni_rbe"}, 223 InheritEnv: true, // Need to make sure bazelisk is on the path, 224 Dir: skiaPath, 225 LogStdout: true, 226 LogStderr: true, 227 } 228 if _, err := sk_exec.RunCommand(ctx, runCmd); err != nil { 229 return err 230 } 231 return nil 232 }) 233} 234 235// gazelle generates/updates Go Bazel targets in BUILD.bazel files. 236func gazelle(ctx context.Context, skiaPath string) error { 237 return td.Do(ctx, td.Props("Gazelle: Generate/update targets"), func(ctx context.Context) error { 238 runCmd := &sk_exec.Command{ 239 Name: "make", 240 Args: []string{"-C", "bazel", "generate_go"}, 241 InheritEnv: true, // Need to make sure bazelisk is on the path, 242 Dir: skiaPath, 243 LogStdout: true, 244 LogStderr: true, 245 } 246 if _, err := sk_exec.RunCommand(ctx, runCmd); err != nil { 247 return err 248 } 249 return nil 250 }) 251} 252 253// gazelleDeps updates //go_repositories.bzl with Gazelle based on the contents of //go.mod. 254func gazelleDeps(ctx context.Context, skiaPath string) error { 255 return td.Do(ctx, td.Props("Gazelle: Update Go dependencies"), func(ctx context.Context) error { 256 runCmd := &sk_exec.Command{ 257 Name: "make", 258 Args: []string{"-C", "bazel", "gazelle_update_repo"}, 259 InheritEnv: true, // Need to make sure bazelisk is on the path, 260 Dir: skiaPath, 261 LogStdout: true, 262 LogStderr: true, 263 } 264 if _, err := sk_exec.RunCommand(ctx, runCmd); err != nil { 265 return err 266 } 267 return nil 268 }) 269} 270 271// checkGitDiff runs git diff and returns error if the diff is non-empty. 272func checkGitDiff(ctx context.Context, gitPath, skiaPath string) error { 273 step := fmt.Sprintf("git diff %s", skiaPath) 274 return td.Do(ctx, td.Props(step), func(ctx context.Context) error { 275 runCmd := &sk_exec.Command{ 276 Name: gitPath, 277 Args: []string{"diff", "--no-ext-diff"}, 278 InheritEnv: false, 279 Dir: skiaPath, 280 LogStdout: true, 281 LogStderr: true, 282 } 283 rv, err := sk_exec.RunCommand(ctx, runCmd) 284 if err != nil { 285 return err 286 } 287 if strings.TrimSpace(rv) != "" { 288 return fmt.Errorf("Non-empty diff:\n" + rv) 289 } 290 return nil 291 }) 292} 293