// Copyright 2021 Google Inc. // // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. package main import ( "context" "flag" "fmt" "net/http" "os" "path/filepath" "strings" "go.skia.org/infra/go/depot_tools" "go.skia.org/infra/go/exec" "go.skia.org/infra/go/gerrit" "go.skia.org/infra/go/git/git_common" "go.skia.org/infra/go/sklog" "go.skia.org/infra/task_driver/go/lib/auth_steps" "go.skia.org/infra/task_driver/go/lib/checkout" "go.skia.org/infra/task_driver/go/lib/gerrit_steps" "go.skia.org/infra/task_driver/go/lib/os_steps" "go.skia.org/infra/task_driver/go/td" "go.skia.org/infra/task_scheduler/go/types" ) func botUpdate(ctx context.Context, checkoutRoot, gitCacheDir, skiaRev, patchRef, depotToolsDir string, local bool) error { return td.Do(ctx, td.Props("bot_update").Infra(), func(ctx context.Context) error { tmp, err := os_steps.TempDir(ctx, "", "") if err != nil { return err } defer func() { _ = os_steps.RemoveAll(ctx, tmp) }() ctx = td.WithEnv(ctx, []string{ "DEPOT_TOOLS_COLLECT_METRICS", "DEPOT_TOOLS_UPDATE=0", "GIT_CONFIG_NOSYSTEM: 1", "GIT_HTTP_LOW_SPEED_LIMIT: 102400", "GIT_HTTP_LOW_SPEED_TIME: 1800", "GIT_TERMINAL_PROMPT: 0", "GIT_USER_AGENT=", }) if _, err := exec.RunCwd(ctx, ".", "which", "git"); err != nil { return err } if _, err := exec.RunCwd(ctx, ".", "git", "--version"); err != nil { return err } if !local { if err := git_common.EnsureGitIsFromCIPD(ctx); err != nil { return err } } spec := ` solutions = [ { "name" : 'src', "url" : 'https://chromium.googlesource.com/chromium/src.git', "deps_file" : 'DEPS', "managed" : False, "custom_deps" : { }, "custom_vars": {}, }, ] ` specPath := filepath.Join(checkoutRoot, ".gclient") if err := os_steps.WriteFile(ctx, specPath, []byte(spec), os.ModePerm); err != nil { return err } skiaRepoURL := "https://skia.googlesource.com/skia.git" botUpdateScript := filepath.Join(depotToolsDir, "recipes", "recipe_modules", "bot_update", "resources", "bot_update.py") mainRepoName := "src" patchRoot := "src/third_party/skia" revision := "origin/main" // These are required for some reason, despite our not using them. outputJson := filepath.Join(tmp, "bot_update.json") revMapFile := filepath.Join(tmp, "revmap") revMap := `{"got_revision": "src"}` if err := os_steps.WriteFile(ctx, revMapFile, []byte(revMap), os.ModePerm); err != nil { return err } cleanupDir := filepath.Join(tmp, "cleanup") if err := os_steps.MkdirAll(ctx, cleanupDir); err != nil { return err } cmd := []string{ "vpython", "-u", botUpdateScript, "--spec-path", specPath, "--patch_root", patchRoot, "--revision_mapping_file", revMapFile, "--cleanup-dir", cleanupDir, "--output_json", outputJson, "--revision", fmt.Sprintf("%s@%s", mainRepoName, revision), "--revision", fmt.Sprintf("%s@%s", "https://skia.googlesource.com/skia.git", skiaRev), } if gitCacheDir != "" { cmd = append(cmd, "--git-cache-dir", gitCacheDir) } if patchRef != "" { patchRepoURL := skiaRepoURL patchBaseRev := skiaRev cmd = append(cmd, "--patch_ref", fmt.Sprintf("%s@%s:%s", patchRepoURL, patchBaseRev, patchRef)) } if _, err := exec.RunCwd(ctx, checkoutRoot, cmd...); err != nil { return err } if _, err := exec.RunCwd(ctx, checkoutRoot, "vpython", "-u", filepath.Join(depotToolsDir, "gclient.py"), "runhooks"); err != nil { return err } return nil }) } func main() { var ( projectId = flag.String("project_id", "", "ID of the Google Cloud project.") taskId = flag.String("task_id", "", "ID of this task.") taskName = flag.String("task_name", "", "Name of the task.") output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.") local = flag.Bool("local", true, "True if running locally (as opposed to on the bots)") dryRun = flag.Bool("dry_run", false, "If set, generate SKPs but do not upload or commit them.") skiaRev = flag.String("skia_revision", "origin/main", "Revision of Skia at which this task is running.") patchRef = flag.String("patch_ref", "", "Patch ref, if any, associated with this task.") skipSync = flag.Bool("skip-sync", false, "Skip sync. Helpful for running locally.") skipBuild = flag.Bool("skip-build", false, "skip build. Helpful for running locally.") gitCacheDirFlag = flag.String("git_cache", "", "Git cache directory.") checkoutRootFlag = flag.String("checkout_root", "", "Directory to use for checkouts.") dmPathFlag = flag.String("dm_path", "", "Path to the DM binary.") ) ctx := td.StartRun(projectId, taskId, taskName, output, local) defer td.EndRun(ctx) // Setup. var client *http.Client var g gerrit.GerritInterface if !*dryRun { var err error client, err = auth_steps.InitHttpClient(ctx, *local, gerrit.AuthScope) if err != nil { td.Fatal(ctx, err) } g, err = gerrit.NewGerrit("https://skia-review.googlesource.com", client) if err != nil { td.Fatal(ctx, err) } } cwd, err := os.Getwd() if err != nil { td.Fatal(ctx, err) } skiaDir := filepath.Join(cwd, "skia") gitCacheDir := "" if *gitCacheDirFlag != "" { gitCacheDir = filepath.Join(cwd, *gitCacheDirFlag) } checkoutRoot := cwd if *checkoutRootFlag != "" { checkoutRoot = filepath.Join(cwd, *checkoutRootFlag) } if *dmPathFlag == "" { td.Fatal(ctx, fmt.Errorf("Must specify --dm_path")) } dmPath := filepath.Join(cwd, *dmPathFlag) // Fetch `sk` if _, err := exec.RunCwd(ctx, skiaDir, "python3", filepath.Join("bin", "fetch-sk")); err != nil { td.Fatal(ctx, err) } // Create a temporary directory. tmp, err := os_steps.TempDir(ctx, "", "") if err != nil { td.Fatal(ctx, err) } defer func() { _ = os_steps.RemoveAll(ctx, tmp) }() recipesCfgFile := filepath.Join(skiaDir, "infra", "config", "recipes.cfg") // Check out depot_tools at the exact revision expected by tests (defined in recipes.cfg), and // make it available to tests by by adding it to the PATH. var depotToolsDir string if err := td.Do(ctx, td.Props("Check out depot_tools"), func(ctx context.Context) error { var err error depotToolsDir, err = depot_tools.Sync(ctx, tmp, recipesCfgFile) return err }); err != nil { td.Fatal(ctx, err) } ctx = td.WithEnv(ctx, []string{"PATH=%(PATH)s:" + depotToolsDir}) // Sync Chrome. if !*skipSync { if err := botUpdate(ctx, checkoutRoot, gitCacheDir, *skiaRev, *patchRef, depotToolsDir, *local); err != nil { td.Fatal(ctx, err) } } chromiumDir := filepath.Join(checkoutRoot, "src") outDir := filepath.Join(chromiumDir, "out", "Release") // Build Chrome. if !*skipBuild { if err := td.Do(ctx, td.Props("Build Chrome"), func(ctx context.Context) error { // Run "gn gen". This task driver only runs on Linux. gn := filepath.Join(chromiumDir, "buildtools", "linux64", "gn") if _, err := exec.RunCommand(ctx, &exec.Command{ Name: gn, Args: []string{"gen", outDir}, Dir: chromiumDir, Env: []string{ "CPPFLAGS=-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1", "GYP_GENERATORS=ninja", }, InheritEnv: true, InheritPath: true, }); err != nil { return err } // Perform the build. ninja := filepath.Join(depotToolsDir, "ninja") if _, err := exec.RunCwd(ctx, chromiumDir, ninja, "-C", outDir, "chrome"); err != nil { return err } return nil }); err != nil { td.Fatal(ctx, err) } } // Capture and upload the SKPs. script := filepath.Join(skiaDir, "infra", "bots", "assets", "skp", "create_and_upload.py") cmd := []string{ "vpython3", "-u", script, "--chrome_src_path", chromiumDir, "--browser_executable", filepath.Join(outDir, "chrome"), "--dm_path", dmPath, } if *dryRun { cmd = append(cmd, "--dry_run") } if *local { cmd = append(cmd, "--local") } command := &exec.Command{ Name: filepath.Join(depotToolsDir, cmd[0]), Args: cmd[1:], Dir: skiaDir, Env: []string{ fmt.Sprintf("PATH=%s:%s", os.Getenv("PATH"), depotToolsDir), }, } sklog.Infof("Running command: %s %s", command.Name, strings.Join(command.Args, " ")) if err := exec.Run(ctx, command); err != nil { td.Fatal(ctx, err) } if *dryRun { return } // Retrieve the new SKP version. versionFileSubPath := filepath.Join("infra", "bots", "assets", "skp", "VERSION") skpVersion, err := os_steps.ReadFile(ctx, filepath.Join(skiaDir, versionFileSubPath)) if err != nil { td.Fatal(ctx, err) } // Sync a new checkout of Skia to create the CL. tmpSkia := filepath.Join(tmp, "skia") co, err := checkout.EnsureGitCheckout(ctx, tmpSkia, types.RepoState{ Repo: "https://skia.googlesource.com/skia.git", Revision: "origin/main", }) if err != nil { td.Fatal(ctx, err) } baseRev, err := co.FullHash(ctx, "HEAD") if err != nil { td.Fatal(ctx, err) } // Write the new SKP version. if err := os_steps.WriteFile(ctx, filepath.Join(co.Dir(), versionFileSubPath), skpVersion, os.ModePerm); err != nil { td.Fatal(ctx, err) } // Regenerate tasks.json. if _, err := exec.RunCwd(ctx, tmpSkia, "go", "run", "./infra/bots/gen_tasks.go"); err != nil { td.Fatal(ctx, err) } // Upload a CL. commitMsg := `Update SKP version Automatic commit by the RecreateSKPs bot. ` gerrit_steps.UploadCL(ctx, g, co, "skia", "main", baseRev, commitMsg, []string{"rmistry@google.com"}, false) }