1// Copyright 2021 Google Inc. 2// 3// Use of this source code is governed by a BSD-style license that can be 4// found in the LICENSE file. 5// 6// This executable builds the Docker images based off the Skia executables in the 7// gcr.io/skia-public/skia-release image. It then issues a PubSub notification to have those apps 8// tagged and deployed by docker_pushes_watcher. 9// See //docker_pushes_watcher/README.md in the infra repo for more. 10package main 11 12import ( 13 "context" 14 "flag" 15 "fmt" 16 "os" 17 "path" 18 "path/filepath" 19 20 "cloud.google.com/go/pubsub" 21 "google.golang.org/api/option" 22 23 "go.skia.org/infra/go/auth" 24 "go.skia.org/infra/go/common" 25 docker_pubsub "go.skia.org/infra/go/docker/build/pubsub" 26 sk_exec "go.skia.org/infra/go/exec" 27 "go.skia.org/infra/task_driver/go/lib/auth_steps" 28 "go.skia.org/infra/task_driver/go/lib/bazel" 29 "go.skia.org/infra/task_driver/go/lib/checkout" 30 "go.skia.org/infra/task_driver/go/lib/docker" 31 "go.skia.org/infra/task_driver/go/lib/golang" 32 "go.skia.org/infra/task_driver/go/lib/os_steps" 33 "go.skia.org/infra/task_driver/go/td" 34 "go.skia.org/infra/task_scheduler/go/types" 35) 36 37var ( 38 // Required properties for this task. 39 projectId = flag.String("project_id", "", "ID of the Google Cloud project.") 40 taskId = flag.String("task_id", "", "ID of this task.") 41 taskName = flag.String("task_name", "", "Name of the task.") 42 workdir = flag.String("workdir", ".", "Working directory") 43 infraRevision = flag.String("infra_revision", "origin/main", "Specifies which revision of the infra repo the images should be built off") 44 45 checkoutFlags = checkout.SetupFlags(nil) 46 47 // Optional flags. 48 local = flag.Bool("local", false, "True if running locally (as opposed to on the bots)") 49 output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.") 50) 51 52const ( 53 fiddlerImageName = "fiddler" 54 apiImageName = "api" 55) 56 57func buildPushFiddlerImage(ctx context.Context, dkr *docker.Docker, tag, infraCheckoutDir string, topic *pubsub.Topic) error { 58 // Run skia-release image and extract products out of /tmp/skia/skia. See 59 // https://skia.googlesource.com/skia/+/0e845dc8b05cb2d40d1c880184e33dd76081283a/docker/skia-release/Dockerfile#33 60 productsDir, err := os_steps.TempDir(ctx, "", "") 61 if err != nil { 62 return err 63 } 64 volumes := []string{ 65 fmt.Sprintf("%s:/OUT", productsDir), 66 } 67 skiaCopyCmd := []string{"/bin/sh", "-c", "cd /tmp; tar cvzf skia.tar.gz --directory=/tmp/skia skia; cp /tmp/skia.tar.gz /OUT/"} 68 releaseImg := fmt.Sprintf("gcr.io/skia-public/skia-release:%s", tag) 69 if err := dkr.Run(ctx, releaseImg, skiaCopyCmd, volumes, nil); err != nil { 70 return err 71 } 72 73 err = td.Do(ctx, td.Props("Build "+fiddlerImageName+" image").Infra(), func(ctx context.Context) error { 74 runCmd := &sk_exec.Command{ 75 Name: "make", 76 Args: []string{"release-fiddler-ci"}, 77 InheritEnv: true, 78 Env: []string{ 79 "COPY_FROM_DIR=" + productsDir, 80 "STABLE_DOCKER_TAG=" + tag, 81 }, 82 Dir: filepath.Join(infraCheckoutDir, "fiddlek"), 83 LogStdout: true, 84 LogStderr: true, 85 } 86 _, err := sk_exec.RunCommand(ctx, runCmd) 87 if err != nil { 88 return err 89 } 90 return nil 91 }) 92 if err != nil { 93 return err 94 } 95 if err := docker.PublishToTopic(ctx, "gcr.io/skia-public/"+fiddlerImageName, tag, common.REPO_SKIA, topic); err != nil { 96 return err 97 } 98 99 return cleanupTempFiles(ctx, dkr, releaseImg, volumes) 100} 101 102func cleanupTempFiles(ctx context.Context, dkr *docker.Docker, image string, volumes []string) error { 103 // Remove all temporary files from the host machine. Swarming gets upset if there are root-owned 104 // files it cannot clean up. 105 cleanupCmd := []string{"/bin/sh", "-c", "rm -rf /OUT/*"} 106 return dkr.Run(ctx, image, cleanupCmd, volumes, nil) 107} 108 109func buildPushApiImage(ctx context.Context, dkr *docker.Docker, tag, checkoutDir, infraCheckoutDir string, topic *pubsub.Topic) error { 110 tempDir, err := os_steps.TempDir(ctx, "", "") 111 if err != nil { 112 return err 113 } 114 // Change perms of the directory for doxygen to be able to write to it. 115 if err := os.Chmod(tempDir, 0777); err != nil { 116 return err 117 } 118 // Run Doxygen pointing to the location of the checkout and the out dir. 119 volumes := []string{ 120 fmt.Sprintf("%s:/OUT", tempDir), 121 fmt.Sprintf("%s:/CHECKOUT", checkoutDir), 122 } 123 env := []string{ 124 "OUTPUT_DIRECTORY=/OUT", 125 } 126 doxygenCmd := []string{"/bin/sh", "-c", "cd /CHECKOUT/tools/doxygen && doxygen ProdDoxyfile"} 127 doxygenImg := "gcr.io/skia-public/doxygen:testing-slim" 128 // Make sure we have the latest doxygen image. 129 if err := dkr.Pull(ctx, doxygenImg); err != nil { 130 return err 131 } 132 if err := dkr.Run(ctx, doxygenImg, doxygenCmd, volumes, env); err != nil { 133 return err 134 } 135 136 err = td.Do(ctx, td.Props("Build "+apiImageName+" image").Infra(), func(ctx context.Context) error { 137 runCmd := &sk_exec.Command{ 138 Name: "make", 139 Args: []string{"release-api-ci"}, 140 InheritEnv: true, 141 Env: []string{ 142 "COPY_FROM_DIR=" + filepath.Join(tempDir, "html"), 143 "STABLE_DOCKER_TAG=" + tag, 144 }, 145 Dir: filepath.Join(infraCheckoutDir, "api"), 146 LogStdout: true, 147 LogStderr: true, 148 } 149 _, err := sk_exec.RunCommand(ctx, runCmd) 150 if err != nil { 151 return err 152 } 153 return nil 154 }) 155 if err != nil { 156 return err 157 } 158 if err := docker.PublishToTopic(ctx, "gcr.io/skia-public/"+apiImageName, tag, common.REPO_SKIA, topic); err != nil { 159 return err 160 } 161 162 return cleanupTempFiles(ctx, dkr, doxygenImg, volumes) 163} 164 165func main() { 166 // Setup. 167 ctx := td.StartRun(projectId, taskId, taskName, output, local) 168 defer td.EndRun(ctx) 169 170 if *infraRevision == "" { 171 td.Fatalf(ctx, "Must specify --infra_revision") 172 } 173 174 rs, err := checkout.GetRepoState(checkoutFlags) 175 if err != nil { 176 td.Fatal(ctx, err) 177 } 178 wd, err := os_steps.Abs(ctx, *workdir) 179 if err != nil { 180 td.Fatal(ctx, err) 181 } 182 // Check out the Skia repo code. 183 co, err := checkout.EnsureGitCheckout(ctx, path.Join(wd, "repo"), rs) 184 if err != nil { 185 td.Fatal(ctx, err) 186 } 187 skiaCheckoutDir := co.Dir() 188 189 // Checkout out the Skia infra repo at the specified commit. 190 infraRS := types.RepoState{ 191 Repo: common.REPO_SKIA_INFRA, 192 Revision: *infraRevision, 193 } 194 infraCheckoutDir := filepath.Join("infra_repo") 195 if _, err := checkout.EnsureGitCheckout(ctx, infraCheckoutDir, infraRS); err != nil { 196 td.Fatal(ctx, err) 197 } 198 199 // Setup go. 200 ctx = golang.WithEnv(ctx, wd) 201 202 // Ensure that the bazel cache is setup. 203 opts := bazel.BazelOptions{ 204 CachePath: "/mnt/pd0/bazel_cache", 205 } 206 if err := bazel.EnsureBazelRCFile(ctx, opts); err != nil { 207 td.Fatal(ctx, err) 208 } 209 210 // Create token source with scope for cloud registry (storage) and pubsub. 211 ts, err := auth_steps.Init(ctx, *local, auth.ScopeUserinfoEmail, auth.ScopeFullControl, pubsub.ScopePubSub) 212 if err != nil { 213 td.Fatal(ctx, err) 214 } 215 216 // Create pubsub client. 217 client, err := pubsub.NewClient(ctx, docker_pubsub.TOPIC_PROJECT_ID, option.WithTokenSource(ts)) 218 if err != nil { 219 td.Fatal(ctx, err) 220 } 221 topic := client.Topic(docker_pubsub.TOPIC) 222 223 // Figure out which tag to use for docker build and push. 224 tag := rs.Revision 225 if rs.Issue != "" && rs.Patchset != "" { 226 tag = fmt.Sprintf("%s_%s", rs.Issue, rs.Patchset) 227 } 228 229 // Instantiate docker. 230 dkr, err := docker.New(ctx, ts) 231 if err != nil { 232 td.Fatal(ctx, err) 233 } 234 235 // Build and push all apps of interest below. 236 if err := buildPushApiImage(ctx, dkr, tag, skiaCheckoutDir, infraCheckoutDir, topic); err != nil { 237 td.Fatal(ctx, err) 238 } 239 if err := buildPushFiddlerImage(ctx, dkr, tag, infraCheckoutDir, topic); err != nil { 240 td.Fatal(ctx, err) 241 } 242} 243