1// Copyright 2021 Google Inc. 2// 3// Use of this source code is governed by a BSD-style license that can be 4// found in the LICENSE file. 5 6package main 7 8import ( 9 "context" 10 "flag" 11 "fmt" 12 "os" 13 "path/filepath" 14 "strings" 15 16 "go.skia.org/infra/go/depot_tools" 17 "go.skia.org/infra/go/exec" 18 "go.skia.org/infra/go/gerrit" 19 "go.skia.org/infra/go/git/git_common" 20 "go.skia.org/infra/go/sklog" 21 "go.skia.org/infra/promk/go/pushgateway" 22 "go.skia.org/infra/task_driver/go/lib/auth_steps" 23 "go.skia.org/infra/task_driver/go/lib/checkout" 24 "go.skia.org/infra/task_driver/go/lib/gerrit_steps" 25 "go.skia.org/infra/task_driver/go/lib/os_steps" 26 "go.skia.org/infra/task_driver/go/td" 27 "go.skia.org/infra/task_scheduler/go/types" 28) 29 30const ( 31 // Metric constants for pushgateway. 32 jobName = "recreate-skps" 33 buildFailureMetricName = "recreate_skps_build_failure" 34 creatingSKPsFailureMetricName = "recreate_skps_creation_failure" 35 metricValue_NoFailure = "0" 36 metricValue_Failure = "1" 37) 38 39func botUpdate(ctx context.Context, checkoutRoot, gitCacheDir, skiaRev, patchRef, depotToolsDir string, local bool) error { 40 return td.Do(ctx, td.Props("bot_update").Infra(), func(ctx context.Context) error { 41 tmp, err := os_steps.TempDir(ctx, "", "") 42 if err != nil { 43 return err 44 } 45 defer func() { 46 _ = os_steps.RemoveAll(ctx, tmp) 47 }() 48 ctx = td.WithEnv(ctx, []string{ 49 "DEPOT_TOOLS_COLLECT_METRICS", 50 "DEPOT_TOOLS_UPDATE=0", 51 "GIT_CONFIG_NOSYSTEM: 1", 52 "GIT_HTTP_LOW_SPEED_LIMIT: 102400", 53 "GIT_HTTP_LOW_SPEED_TIME: 1800", 54 "GIT_TERMINAL_PROMPT: 0", 55 "GIT_USER_AGENT=", 56 }) 57 if _, err := exec.RunCwd(ctx, ".", "which", "git"); err != nil { 58 return err 59 } 60 if _, err := exec.RunCwd(ctx, ".", "git", "--version"); err != nil { 61 return err 62 } 63 if !local { 64 if err := git_common.EnsureGitIsFromCIPD(ctx); err != nil { 65 return err 66 } 67 } 68 69 spec := ` 70solutions = [ 71 { "name" : 'src', 72 "url" : 'https://chromium.googlesource.com/chromium/src.git', 73 "deps_file" : 'DEPS', 74 "managed" : False, 75 "custom_deps" : { 76 }, 77 "custom_vars": {}, 78 }, 79] 80` 81 specPath := filepath.Join(checkoutRoot, ".gclient") 82 if err := os_steps.WriteFile(ctx, specPath, []byte(spec), os.ModePerm); err != nil { 83 return err 84 } 85 skiaRepoURL := "https://skia.googlesource.com/skia.git" 86 botUpdateScript := filepath.Join(depotToolsDir, "recipes", "recipe_modules", "bot_update", "resources", "bot_update.py") 87 mainRepoName := "src" 88 patchRoot := "src/third_party/skia" 89 revision := "origin/main" 90 // These are required for some reason, despite our not using them. 91 outputJson := filepath.Join(tmp, "bot_update.json") 92 revMapFile := filepath.Join(tmp, "revmap") 93 revMap := `{"got_revision": "src"}` 94 if err := os_steps.WriteFile(ctx, revMapFile, []byte(revMap), os.ModePerm); err != nil { 95 return err 96 } 97 cleanupDir := filepath.Join(tmp, "cleanup") 98 if err := os_steps.MkdirAll(ctx, cleanupDir); err != nil { 99 return err 100 } 101 102 cmd := []string{ 103 "vpython", "-u", botUpdateScript, 104 "--spec-path", specPath, 105 "--patch_root", patchRoot, 106 "--revision_mapping_file", revMapFile, 107 "--cleanup-dir", cleanupDir, 108 "--output_json", outputJson, 109 "--revision", fmt.Sprintf("%s@%s", mainRepoName, revision), 110 "--revision", fmt.Sprintf("%s@%s", "https://skia.googlesource.com/skia.git", skiaRev), 111 } 112 if gitCacheDir != "" { 113 cmd = append(cmd, "--git-cache-dir", gitCacheDir) 114 } 115 if patchRef != "" { 116 patchRepoURL := skiaRepoURL 117 patchBaseRev := skiaRev 118 cmd = append(cmd, "--patch_ref", fmt.Sprintf("%s@%s:%s", patchRepoURL, patchBaseRev, patchRef)) 119 } 120 if _, err := exec.RunCwd(ctx, checkoutRoot, cmd...); err != nil { 121 return err 122 } 123 124 if _, err := exec.RunCwd(ctx, checkoutRoot, "vpython", "-u", filepath.Join(depotToolsDir, "gclient.py"), "runhooks"); err != nil { 125 return err 126 } 127 128 return nil 129 }) 130} 131 132func main() { 133 var ( 134 projectId = flag.String("project_id", "", "ID of the Google Cloud project.") 135 taskId = flag.String("task_id", "", "ID of this task.") 136 taskName = flag.String("task_name", "", "Name of the task.") 137 output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.") 138 local = flag.Bool("local", true, "True if running locally (as opposed to on the bots)") 139 dryRun = flag.Bool("dry_run", false, "If set, generate SKPs but do not upload or commit them.") 140 skiaRev = flag.String("skia_revision", "origin/main", "Revision of Skia at which this task is running.") 141 patchRef = flag.String("patch_ref", "", "Patch ref, if any, associated with this task.") 142 skipSync = flag.Bool("skip-sync", false, "Skip sync. Helpful for running locally.") 143 skipBuild = flag.Bool("skip-build", false, "skip build. Helpful for running locally.") 144 gitCacheDirFlag = flag.String("git_cache", "", "Git cache directory.") 145 checkoutRootFlag = flag.String("checkout_root", "", "Directory to use for checkouts.") 146 dmPathFlag = flag.String("dm_path", "", "Path to the DM binary.") 147 ) 148 ctx := td.StartRun(projectId, taskId, taskName, output, local) 149 defer td.EndRun(ctx) 150 151 // Setup. 152 client, err := auth_steps.InitHttpClient(ctx, *local, gerrit.AuthScope) 153 if err != nil { 154 td.Fatal(ctx, err) 155 } 156 var g gerrit.GerritInterface 157 if !*dryRun { 158 g, err = gerrit.NewGerrit("https://skia-review.googlesource.com", client) 159 if err != nil { 160 td.Fatal(ctx, err) 161 } 162 } 163 cwd, err := os.Getwd() 164 if err != nil { 165 td.Fatal(ctx, err) 166 } 167 skiaDir := filepath.Join(cwd, "skia") 168 gitCacheDir := "" 169 if *gitCacheDirFlag != "" { 170 gitCacheDir = filepath.Join(cwd, *gitCacheDirFlag) 171 } 172 checkoutRoot := cwd 173 if *checkoutRootFlag != "" { 174 checkoutRoot = filepath.Join(cwd, *checkoutRootFlag) 175 } 176 if *dmPathFlag == "" { 177 td.Fatal(ctx, fmt.Errorf("Must specify --dm_path")) 178 } 179 dmPath := filepath.Join(cwd, *dmPathFlag) 180 181 // Fetch `sk` 182 if _, err := exec.RunCwd(ctx, skiaDir, "python3", filepath.Join("bin", "fetch-sk")); err != nil { 183 td.Fatal(ctx, err) 184 } 185 186 // Create a temporary directory. 187 tmp, err := os_steps.TempDir(ctx, "", "") 188 if err != nil { 189 td.Fatal(ctx, err) 190 } 191 defer func() { 192 _ = os_steps.RemoveAll(ctx, tmp) 193 }() 194 recipesCfgFile := filepath.Join(skiaDir, "infra", "config", "recipes.cfg") 195 196 // Check out depot_tools at the exact revision expected by tests (defined in recipes.cfg), and 197 // make it available to tests by by adding it to the PATH. 198 var depotToolsDir string 199 if err := td.Do(ctx, td.Props("Check out depot_tools"), func(ctx context.Context) error { 200 var err error 201 depotToolsDir, err = depot_tools.Sync(ctx, tmp, recipesCfgFile) 202 return err 203 }); err != nil { 204 td.Fatal(ctx, err) 205 } 206 ctx = td.WithEnv(ctx, []string{"PATH=%(PATH)s:" + depotToolsDir}) 207 208 // Sync Chrome. 209 if !*skipSync { 210 if err := botUpdate(ctx, checkoutRoot, gitCacheDir, *skiaRev, *patchRef, depotToolsDir, *local); err != nil { 211 td.Fatal(ctx, err) 212 } 213 } 214 215 // For updating metrics. 216 pg := pushgateway.New(client, jobName, pushgateway.DefaultPushgatewayURL) 217 218 chromiumDir := filepath.Join(checkoutRoot, "src") 219 outDir := filepath.Join(chromiumDir, "out", "Release") 220 221 // Build Chrome. 222 if !*skipBuild { 223 if err := td.Do(ctx, td.Props("Build Chrome"), func(ctx context.Context) error { 224 // Run "gn gen". This task driver only runs on Linux. 225 gn := filepath.Join(chromiumDir, "buildtools", "linux64", "gn") 226 if _, err := exec.RunCommand(ctx, &exec.Command{ 227 Name: gn, 228 Args: []string{"gen", outDir}, 229 Dir: chromiumDir, 230 Env: []string{ 231 "CPPFLAGS=-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1", 232 "GYP_GENERATORS=ninja", 233 }, 234 InheritEnv: true, 235 InheritPath: true, 236 }); err != nil { 237 return err 238 } 239 240 // Perform the build. 241 ninja := filepath.Join(depotToolsDir, "ninja") 242 if _, err := exec.RunCwd(ctx, chromiumDir, ninja, "-C", outDir, "chrome"); err != nil { 243 return err 244 } 245 return nil 246 }); err != nil { 247 // Report that the build failed. 248 pg.Push(ctx, buildFailureMetricName, metricValue_Failure) 249 td.Fatal(ctx, err) 250 } 251 // Report that the build was successful. 252 pg.Push(ctx, buildFailureMetricName, metricValue_NoFailure) 253 } 254 255 // Capture and upload the SKPs. 256 script := filepath.Join(skiaDir, "infra", "bots", "assets", "skp", "create_and_upload.py") 257 cmd := []string{ 258 "vpython3", "-u", script, 259 "--chrome_src_path", chromiumDir, 260 "--browser_executable", filepath.Join(outDir, "chrome"), 261 "--dm_path", dmPath, 262 } 263 if *dryRun { 264 cmd = append(cmd, "--dry_run") 265 } 266 if *local { 267 cmd = append(cmd, "--local") 268 } 269 command := &exec.Command{ 270 Name: filepath.Join(depotToolsDir, cmd[0]), 271 Args: cmd[1:], 272 Dir: skiaDir, 273 Env: []string{ 274 fmt.Sprintf("PATH=%s:%s", os.Getenv("PATH"), depotToolsDir), 275 }, 276 } 277 sklog.Infof("Running command: %s %s", command.Name, strings.Join(command.Args, " ")) 278 if err := exec.Run(ctx, command); err != nil { 279 // Creating SKP asset in RecreateSKPs failed. 280 pg.Push(ctx, creatingSKPsFailureMetricName, metricValue_Failure) 281 td.Fatal(ctx, err) 282 } 283 // Report that the asset creation was successful. 284 pg.Push(ctx, creatingSKPsFailureMetricName, metricValue_NoFailure) 285 if *dryRun { 286 return 287 } 288 289 // Retrieve the new SKP version. 290 versionFileSubPath := filepath.Join("infra", "bots", "assets", "skp", "VERSION") 291 skpVersion, err := os_steps.ReadFile(ctx, filepath.Join(skiaDir, versionFileSubPath)) 292 if err != nil { 293 td.Fatal(ctx, err) 294 } 295 296 // Sync a new checkout of Skia to create the CL. 297 tmpSkia := filepath.Join(tmp, "skia") 298 co, err := checkout.EnsureGitCheckout(ctx, tmpSkia, types.RepoState{ 299 Repo: "https://skia.googlesource.com/skia.git", 300 Revision: "origin/main", 301 }) 302 if err != nil { 303 td.Fatal(ctx, err) 304 } 305 baseRev, err := co.FullHash(ctx, "HEAD") 306 if err != nil { 307 td.Fatal(ctx, err) 308 } 309 310 // Write the new SKP version. 311 if err := os_steps.WriteFile(ctx, filepath.Join(co.Dir(), versionFileSubPath), skpVersion, os.ModePerm); err != nil { 312 td.Fatal(ctx, err) 313 } 314 315 // Regenerate tasks.json. 316 if _, err := exec.RunCwd(ctx, tmpSkia, "go", "run", "./infra/bots/gen_tasks.go"); err != nil { 317 td.Fatal(ctx, err) 318 } 319 320 // Upload a CL. 321 commitMsg := `Update SKP version 322 323Automatic commit by the RecreateSKPs bot. 324` 325 gerrit_steps.UploadCL(ctx, g, co, "skia", "main", baseRev, commitMsg, []string{"rmistry@google.com"}, false) 326} 327