1// Copyright 2021 Google Inc. 2// 3// Use of this source code is governed by a BSD-style license that can be 4// found in the LICENSE file. 5 6package main 7 8import ( 9 "context" 10 "flag" 11 "fmt" 12 "net/http" 13 "os" 14 "path/filepath" 15 "strings" 16 17 "go.skia.org/infra/go/depot_tools" 18 "go.skia.org/infra/go/exec" 19 "go.skia.org/infra/go/gerrit" 20 "go.skia.org/infra/go/git/git_common" 21 "go.skia.org/infra/go/sklog" 22 "go.skia.org/infra/task_driver/go/lib/auth_steps" 23 "go.skia.org/infra/task_driver/go/lib/checkout" 24 "go.skia.org/infra/task_driver/go/lib/gerrit_steps" 25 "go.skia.org/infra/task_driver/go/lib/os_steps" 26 "go.skia.org/infra/task_driver/go/td" 27 "go.skia.org/infra/task_scheduler/go/types" 28) 29 30func botUpdate(ctx context.Context, checkoutRoot, gitCacheDir, skiaRev, patchRef, depotToolsDir string, local bool) error { 31 return td.Do(ctx, td.Props("bot_update").Infra(), func(ctx context.Context) error { 32 tmp, err := os_steps.TempDir(ctx, "", "") 33 if err != nil { 34 return err 35 } 36 defer func() { 37 _ = os_steps.RemoveAll(ctx, tmp) 38 }() 39 ctx = td.WithEnv(ctx, []string{ 40 "DEPOT_TOOLS_COLLECT_METRICS", 41 "DEPOT_TOOLS_UPDATE=0", 42 "GIT_CONFIG_NOSYSTEM: 1", 43 "GIT_HTTP_LOW_SPEED_LIMIT: 102400", 44 "GIT_HTTP_LOW_SPEED_TIME: 1800", 45 "GIT_TERMINAL_PROMPT: 0", 46 "GIT_USER_AGENT=", 47 }) 48 if _, err := exec.RunCwd(ctx, ".", "which", "git"); err != nil { 49 return err 50 } 51 if _, err := exec.RunCwd(ctx, ".", "git", "--version"); err != nil { 52 return err 53 } 54 if !local { 55 if err := git_common.EnsureGitIsFromCIPD(ctx); err != nil { 56 return err 57 } 58 } 59 60 spec := ` 61solutions = [ 62 { "name" : 'src', 63 "url" : 'https://chromium.googlesource.com/chromium/src.git', 64 "deps_file" : 'DEPS', 65 "managed" : False, 66 "custom_deps" : { 67 }, 68 "custom_vars": {}, 69 }, 70] 71` 72 specPath := filepath.Join(checkoutRoot, ".gclient") 73 if err := os_steps.WriteFile(ctx, specPath, []byte(spec), os.ModePerm); err != nil { 74 return err 75 } 76 skiaRepoURL := "https://skia.googlesource.com/skia.git" 77 botUpdateScript := filepath.Join(depotToolsDir, "recipes", "recipe_modules", "bot_update", "resources", "bot_update.py") 78 mainRepoName := "src" 79 patchRoot := "src/third_party/skia" 80 revision := "origin/main" 81 // These are required for some reason, despite our not using them. 82 outputJson := filepath.Join(tmp, "bot_update.json") 83 revMapFile := filepath.Join(tmp, "revmap") 84 revMap := `{"got_revision": "src"}` 85 if err := os_steps.WriteFile(ctx, revMapFile, []byte(revMap), os.ModePerm); err != nil { 86 return err 87 } 88 cleanupDir := filepath.Join(tmp, "cleanup") 89 if err := os_steps.MkdirAll(ctx, cleanupDir); err != nil { 90 return err 91 } 92 93 cmd := []string{ 94 "vpython", "-u", botUpdateScript, 95 "--spec-path", specPath, 96 "--patch_root", patchRoot, 97 "--revision_mapping_file", revMapFile, 98 "--cleanup-dir", cleanupDir, 99 "--output_json", outputJson, 100 "--revision", fmt.Sprintf("%s@%s", mainRepoName, revision), 101 "--revision", fmt.Sprintf("%s@%s", "https://skia.googlesource.com/skia.git", skiaRev), 102 } 103 if gitCacheDir != "" { 104 cmd = append(cmd, "--git-cache-dir", gitCacheDir) 105 } 106 if patchRef != "" { 107 patchRepoURL := skiaRepoURL 108 patchBaseRev := skiaRev 109 cmd = append(cmd, "--patch_ref", fmt.Sprintf("%s@%s:%s", patchRepoURL, patchBaseRev, patchRef)) 110 } 111 if _, err := exec.RunCwd(ctx, checkoutRoot, cmd...); err != nil { 112 return err 113 } 114 115 if _, err := exec.RunCwd(ctx, checkoutRoot, "vpython", "-u", filepath.Join(depotToolsDir, "gclient.py"), "runhooks"); err != nil { 116 return err 117 } 118 119 return nil 120 }) 121} 122 123func main() { 124 var ( 125 projectId = flag.String("project_id", "", "ID of the Google Cloud project.") 126 taskId = flag.String("task_id", "", "ID of this task.") 127 taskName = flag.String("task_name", "", "Name of the task.") 128 output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.") 129 local = flag.Bool("local", true, "True if running locally (as opposed to on the bots)") 130 dryRun = flag.Bool("dry_run", false, "If set, generate SKPs but do not upload or commit them.") 131 skiaRev = flag.String("skia_revision", "origin/main", "Revision of Skia at which this task is running.") 132 patchRef = flag.String("patch_ref", "", "Patch ref, if any, associated with this task.") 133 skipSync = flag.Bool("skip-sync", false, "Skip sync. Helpful for running locally.") 134 skipBuild = flag.Bool("skip-build", false, "skip build. Helpful for running locally.") 135 gitCacheDirFlag = flag.String("git_cache", "", "Git cache directory.") 136 checkoutRootFlag = flag.String("checkout_root", "", "Directory to use for checkouts.") 137 dmPathFlag = flag.String("dm_path", "", "Path to the DM binary.") 138 ) 139 ctx := td.StartRun(projectId, taskId, taskName, output, local) 140 defer td.EndRun(ctx) 141 142 // Setup. 143 var client *http.Client 144 var g gerrit.GerritInterface 145 if !*dryRun { 146 var err error 147 client, err = auth_steps.InitHttpClient(ctx, *local, gerrit.AuthScope) 148 if err != nil { 149 td.Fatal(ctx, err) 150 } 151 g, err = gerrit.NewGerrit("https://skia-review.googlesource.com", client) 152 if err != nil { 153 td.Fatal(ctx, err) 154 } 155 } 156 cwd, err := os.Getwd() 157 if err != nil { 158 td.Fatal(ctx, err) 159 } 160 skiaDir := filepath.Join(cwd, "skia") 161 gitCacheDir := "" 162 if *gitCacheDirFlag != "" { 163 gitCacheDir = filepath.Join(cwd, *gitCacheDirFlag) 164 } 165 checkoutRoot := cwd 166 if *checkoutRootFlag != "" { 167 checkoutRoot = filepath.Join(cwd, *checkoutRootFlag) 168 } 169 if *dmPathFlag == "" { 170 td.Fatal(ctx, fmt.Errorf("Must specify --dm_path")) 171 } 172 dmPath := filepath.Join(cwd, *dmPathFlag) 173 174 // Fetch `sk` 175 if _, err := exec.RunCwd(ctx, skiaDir, "python3", filepath.Join("bin", "fetch-sk")); err != nil { 176 td.Fatal(ctx, err) 177 } 178 179 // Create a temporary directory. 180 tmp, err := os_steps.TempDir(ctx, "", "") 181 if err != nil { 182 td.Fatal(ctx, err) 183 } 184 defer func() { 185 _ = os_steps.RemoveAll(ctx, tmp) 186 }() 187 recipesCfgFile := filepath.Join(skiaDir, "infra", "config", "recipes.cfg") 188 189 // Check out depot_tools at the exact revision expected by tests (defined in recipes.cfg), and 190 // make it available to tests by by adding it to the PATH. 191 var depotToolsDir string 192 if err := td.Do(ctx, td.Props("Check out depot_tools"), func(ctx context.Context) error { 193 var err error 194 depotToolsDir, err = depot_tools.Sync(ctx, tmp, recipesCfgFile) 195 return err 196 }); err != nil { 197 td.Fatal(ctx, err) 198 } 199 ctx = td.WithEnv(ctx, []string{"PATH=%(PATH)s:" + depotToolsDir}) 200 201 // Sync Chrome. 202 if !*skipSync { 203 if err := botUpdate(ctx, checkoutRoot, gitCacheDir, *skiaRev, *patchRef, depotToolsDir, *local); err != nil { 204 td.Fatal(ctx, err) 205 } 206 } 207 208 chromiumDir := filepath.Join(checkoutRoot, "src") 209 outDir := filepath.Join(chromiumDir, "out", "Release") 210 211 // Build Chrome. 212 if !*skipBuild { 213 if err := td.Do(ctx, td.Props("Build Chrome"), func(ctx context.Context) error { 214 // Run "gn gen". This task driver only runs on Linux. 215 gn := filepath.Join(chromiumDir, "buildtools", "linux64", "gn") 216 if _, err := exec.RunCommand(ctx, &exec.Command{ 217 Name: gn, 218 Args: []string{"gen", outDir}, 219 Dir: chromiumDir, 220 Env: []string{ 221 "CPPFLAGS=-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1", 222 "GYP_GENERATORS=ninja", 223 }, 224 InheritEnv: true, 225 InheritPath: true, 226 }); err != nil { 227 return err 228 } 229 230 // Perform the build. 231 ninja := filepath.Join(depotToolsDir, "ninja") 232 if _, err := exec.RunCwd(ctx, chromiumDir, ninja, "-C", outDir, "chrome"); err != nil { 233 return err 234 } 235 return nil 236 }); err != nil { 237 td.Fatal(ctx, err) 238 } 239 } 240 241 // Capture and upload the SKPs. 242 script := filepath.Join(skiaDir, "infra", "bots", "assets", "skp", "create_and_upload.py") 243 cmd := []string{ 244 "vpython3", "-u", script, 245 "--chrome_src_path", chromiumDir, 246 "--browser_executable", filepath.Join(outDir, "chrome"), 247 "--dm_path", dmPath, 248 } 249 if *dryRun { 250 cmd = append(cmd, "--dry_run") 251 } 252 if *local { 253 cmd = append(cmd, "--local") 254 } 255 command := &exec.Command{ 256 Name: filepath.Join(depotToolsDir, cmd[0]), 257 Args: cmd[1:], 258 Dir: skiaDir, 259 Env: []string{ 260 fmt.Sprintf("PATH=%s:%s", os.Getenv("PATH"), depotToolsDir), 261 }, 262 } 263 sklog.Infof("Running command: %s %s", command.Name, strings.Join(command.Args, " ")) 264 if err := exec.Run(ctx, command); err != nil { 265 td.Fatal(ctx, err) 266 } 267 if *dryRun { 268 return 269 } 270 271 // Retrieve the new SKP version. 272 versionFileSubPath := filepath.Join("infra", "bots", "assets", "skp", "VERSION") 273 skpVersion, err := os_steps.ReadFile(ctx, filepath.Join(skiaDir, versionFileSubPath)) 274 if err != nil { 275 td.Fatal(ctx, err) 276 } 277 278 // Sync a new checkout of Skia to create the CL. 279 tmpSkia := filepath.Join(tmp, "skia") 280 co, err := checkout.EnsureGitCheckout(ctx, tmpSkia, types.RepoState{ 281 Repo: "https://skia.googlesource.com/skia.git", 282 Revision: "origin/main", 283 }) 284 if err != nil { 285 td.Fatal(ctx, err) 286 } 287 baseRev, err := co.FullHash(ctx, "HEAD") 288 if err != nil { 289 td.Fatal(ctx, err) 290 } 291 292 // Write the new SKP version. 293 if err := os_steps.WriteFile(ctx, filepath.Join(co.Dir(), versionFileSubPath), skpVersion, os.ModePerm); err != nil { 294 td.Fatal(ctx, err) 295 } 296 297 // Regenerate tasks.json. 298 if _, err := exec.RunCwd(ctx, tmpSkia, "go", "run", "./infra/bots/gen_tasks.go"); err != nil { 299 td.Fatal(ctx, err) 300 } 301 302 // Upload a CL. 303 commitMsg := `Update SKP version 304 305Automatic commit by the RecreateSKPs bot. 306` 307 gerrit_steps.UploadCL(ctx, g, co, "skia", "main", baseRev, commitMsg, []string{"rmistry@google.com"}, false) 308} 309