1// Copyright 2022 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5// This task driver takes a binary (e.g. "dm") built by a Build-* task (e.g. 6// "Build-Debian10-Clang-x86_64-Release"), runs Bloaty against the binary, and uploads the resulting 7// code size statistics to the GCS bucket belonging to the https://codesize.skia.org service. 8package main 9 10import ( 11 "context" 12 "encoding/json" 13 "flag" 14 "fmt" 15 "os" 16 "strconv" 17 "time" 18 19 "cloud.google.com/go/storage" 20 "google.golang.org/api/option" 21 22 "go.skia.org/infra/go/auth" 23 "go.skia.org/infra/go/exec" 24 "go.skia.org/infra/go/gcs" 25 "go.skia.org/infra/go/gcs/gcsclient" 26 "go.skia.org/infra/go/gerrit" 27 "go.skia.org/infra/go/gitiles" 28 "go.skia.org/infra/go/now" 29 "go.skia.org/infra/go/skerr" 30 "go.skia.org/infra/task_driver/go/lib/auth_steps" 31 "go.skia.org/infra/task_driver/go/lib/checkout" 32 "go.skia.org/infra/task_driver/go/td" 33 "go.skia.org/infra/task_scheduler/go/types" 34) 35 36const gcsBucketName = "skia-codesize" 37 38// BloatyOutputMetadata contains the Bloaty version and command-line arguments used, and metadata 39// about the task where Bloaty was invoked. This struct is serialized into a JSON file that is 40// uploaded to GCS alongside the Bloaty output file. 41// 42// TODO(lovisolo): Move this struct to the buildbot repository. 43type BloatyOutputMetadata struct { 44 Version int `json:"version"` // Schema version of this file, starting at 1. 45 Timestamp string `json:"timestamp"` 46 47 SwarmingTaskID string `json:"swarming_task_id"` 48 SwarmingServer string `json:"swarming_server"` 49 50 TaskID string `json:"task_id"` 51 TaskName string `json:"task_name"` 52 CompileTaskName string `json:"compile_task_name"` 53 BinaryName string `json:"binary_name"` 54 55 BloatyCipdVersion string `json:"bloaty_cipd_version"` 56 BloatyArgs []string `json:"bloaty_args"` 57 58 PatchIssue string `json:"patch_issue"` 59 PatchServer string `json:"patch_server"` 60 PatchSet string `json:"patch_set"` 61 Repo string `json:"repo"` 62 Revision string `json:"revision"` 63 64 CommitTimestamp string `json:"commit_timestamp"` 65 Author string `json:"author"` 66 Subject string `json:"subject"` 67} 68 69func main() { 70 var ( 71 projectID = flag.String("project_id", "", "ID of the Google Cloud project.") 72 taskID = flag.String("task_id", "", "ID of this task.") 73 taskName = flag.String("task_name", "", "Name of the task.") 74 compileTaskName = flag.String("compile_task_name", "", "Name of the compile task that produced the binary to analyze.") 75 binaryName = flag.String("binary_name", "", "Name of the binary to analyze (e.g. \"dm\").") 76 bloatyCIPDVersion = flag.String("bloaty_cipd_version", "", "Version of the \"bloaty\" CIPD package used.") 77 output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.") 78 local = flag.Bool("local", true, "True if running locally (as opposed to on the bots).") 79 80 checkoutFlags = checkout.SetupFlags(nil) 81 ) 82 ctx := td.StartRun(projectID, taskID, taskName, output, local) 83 defer td.EndRun(ctx) 84 85 // The repository state contains the commit hash and patch/patchset if available. 86 repoState, err := checkout.GetRepoState(checkoutFlags) 87 if err != nil { 88 td.Fatal(ctx, skerr.Wrap(err)) 89 } 90 91 // Make an HTTP client with the required permissions to hit GCS, Gerrit and Gitiles. 92 httpClient, err := auth_steps.InitHttpClient(ctx, *local, auth.ScopeReadWrite, gerrit.AuthScope, auth.ScopeUserinfoEmail) 93 if err != nil { 94 td.Fatal(ctx, skerr.Wrap(err)) 95 } 96 97 // Make a GCS client with the required permissions to upload to the codesize.skia.org GCS bucket. 98 store, err := storage.NewClient(ctx, option.WithHTTPClient(httpClient)) 99 if err != nil { 100 td.Fatal(ctx, skerr.Wrap(err)) 101 } 102 gcsClient := gcsclient.New(store, gcsBucketName) 103 104 // Make a Gerrit client. 105 gerrit, err := gerrit.NewGerrit(repoState.Server, httpClient) 106 if err != nil { 107 td.Fatal(ctx, skerr.Wrap(err)) 108 } 109 110 // Make a Gitiles client. 111 gitilesRepo := gitiles.NewRepo(repoState.Repo, httpClient) 112 113 args := runStepsArgs{ 114 repoState: repoState, 115 gerrit: gerrit, 116 gitilesRepo: gitilesRepo, 117 gcsClient: gcsClient, 118 swarmingTaskID: os.Getenv("SWARMING_TASK_ID"), 119 swarmingServer: os.Getenv("SWARMING_SERVER"), 120 taskID: *taskID, 121 taskName: *taskName, 122 compileTaskName: *compileTaskName, 123 binaryName: *binaryName, 124 bloatyCIPDVersion: *bloatyCIPDVersion, 125 } 126 127 if err := runSteps(ctx, args); err != nil { 128 td.Fatal(ctx, skerr.Wrap(err)) 129 } 130} 131 132// runStepsArgs contains the input arguments to the runSteps function. 133type runStepsArgs struct { 134 repoState types.RepoState 135 gerrit *gerrit.Gerrit 136 gitilesRepo gitiles.GitilesRepo 137 gcsClient gcs.GCSClient 138 swarmingTaskID string 139 swarmingServer string 140 taskID string 141 taskName string 142 compileTaskName string 143 binaryName string 144 bloatyCIPDVersion string 145} 146 147// runSteps runs the main steps of this task driver. 148func runSteps(ctx context.Context, args runStepsArgs) error { 149 var ( 150 author string 151 subject string 152 commitTimestamp string 153 ) 154 155 // Read the CL subject, author and timestamp. We talk to Gerrit when running as a tryjob, or to 156 // Gitiles when running as a post-submit task. 157 if args.repoState.IsTryJob() { 158 issue, err := strconv.ParseInt(args.repoState.Issue, 10, 64) 159 if err != nil { 160 return skerr.Wrap(err) 161 } 162 patchset, err := strconv.ParseInt(args.repoState.Patchset, 10, 64) 163 if err != nil { 164 return skerr.Wrap(err) 165 } 166 changeInfo, err := args.gerrit.GetIssueProperties(ctx, issue) 167 if err != nil { 168 return skerr.Wrap(err) 169 } 170 // This matches the format of the author field returned by Gitiles. 171 author = fmt.Sprintf("%s (%s)", changeInfo.Owner.Name, changeInfo.Owner.Email) 172 subject = changeInfo.Subject 173 for _, revision := range changeInfo.Revisions { 174 if revision.Number == patchset { 175 commitTimestamp = revision.CreatedString 176 break 177 } 178 } 179 } else { 180 longCommit, err := args.gitilesRepo.Details(ctx, args.repoState.Revision) 181 if err != nil { 182 return skerr.Wrap(err) 183 } 184 author = longCommit.Author 185 subject = longCommit.Subject 186 commitTimestamp = longCommit.Timestamp.Format(time.RFC3339) 187 } 188 189 // Run Bloaty and capture its output. 190 bloatyOutput, bloatyArgs, err := runBloaty(ctx, args.binaryName) 191 if err != nil { 192 return skerr.Wrap(err) 193 } 194 195 // Build metadata structure. 196 metadata := &BloatyOutputMetadata{ 197 Version: 1, 198 Timestamp: now.Now(ctx).Format(time.RFC3339), 199 SwarmingTaskID: args.swarmingTaskID, 200 SwarmingServer: args.swarmingServer, 201 TaskID: args.taskID, 202 TaskName: args.taskName, 203 CompileTaskName: args.compileTaskName, 204 BinaryName: args.binaryName, 205 BloatyCipdVersion: args.bloatyCIPDVersion, 206 BloatyArgs: bloatyArgs, 207 PatchIssue: args.repoState.Issue, 208 PatchServer: args.repoState.Server, 209 PatchSet: args.repoState.Patchset, 210 Repo: args.repoState.Repo, 211 Revision: args.repoState.Revision, 212 CommitTimestamp: commitTimestamp, 213 Author: author, 214 Subject: subject, 215 } 216 217 gcsDir := computeTargetGCSDirectory(ctx, args.repoState, args.taskID, args.compileTaskName) 218 219 // Upload Bloaty output TSV file to GCS. 220 if err = uploadFileToGCS(ctx, args.gcsClient, fmt.Sprintf("%s/%s.tsv", gcsDir, args.binaryName), []byte(bloatyOutput)); err != nil { 221 return skerr.Wrap(err) 222 } 223 224 // Upload pretty-printed JSON metadata file to GCS. 225 jsonMetadata, err := json.MarshalIndent(metadata, "", " ") 226 if err != nil { 227 return skerr.Wrap(err) 228 } 229 if err = uploadFileToGCS(ctx, args.gcsClient, fmt.Sprintf("%s/%s.json", gcsDir, args.binaryName), jsonMetadata); err != nil { 230 return skerr.Wrap(err) 231 } 232 233 return nil 234} 235 236// runBloaty runs Bloaty against the given binary and returns the Bloaty output in TSV format and 237// the Bloaty command-line arguments used. 238func runBloaty(ctx context.Context, binaryName string) (string, []string, error) { 239 err := td.Do(ctx, td.Props("List files under $PWD/build (for debugging purposes)"), func(ctx context.Context) error { 240 runCmd := &exec.Command{ 241 Name: "ls", 242 Args: []string{"build"}, 243 InheritEnv: true, 244 LogStdout: true, 245 LogStderr: true, 246 } 247 _, err := exec.RunCommand(ctx, runCmd) 248 return err 249 }) 250 if err != nil { 251 return "", []string{}, skerr.Wrap(err) 252 } 253 254 runCmd := &exec.Command{ 255 Name: "bloaty/bloaty", 256 Args: []string{ 257 "build/" + binaryName, 258 "-d", 259 "compileunits,symbols", 260 "-n", 261 "0", 262 "--tsv", 263 }, 264 InheritEnv: true, 265 LogStdout: true, 266 LogStderr: true, 267 } 268 269 var bloatyOutput string 270 271 if err := td.Do(ctx, td.Props(fmt.Sprintf("Run Bloaty against binary %q", binaryName)), func(ctx context.Context) error { 272 bloatyOutput, err = exec.RunCommand(ctx, runCmd) 273 return err 274 }); err != nil { 275 return "", []string{}, skerr.Wrap(err) 276 } 277 278 return bloatyOutput, runCmd.Args, nil 279} 280 281// computeTargetGCSDirectory computs the target GCS directory where to upload the Bloaty output file 282// and JSON metadata file. 283func computeTargetGCSDirectory(ctx context.Context, repoState types.RepoState, taskID, compileTaskName string) string { 284 yearMonthDate := now.Now(ctx).Format("2006/01/02") // YYYY/MM/DD. 285 if repoState.IsTryJob() { 286 // Example: 2022/01/31/tryjob/12345/3/CkPp9ElAaEXyYWNHpXHU/Build-Debian10-Clang-x86_64-Release 287 return fmt.Sprintf("%s/tryjob/%s/%s/%s/%s", yearMonthDate, repoState.Patch.Issue, repoState.Patch.Patchset, taskID, compileTaskName) 288 } else { 289 // Example: 2022/01/31/033ccea12c0949d0f712471bfcb4ed6daf69aaff/Build-Debian10-Clang-x86_64-Release 290 return fmt.Sprintf("%s/%s/%s", yearMonthDate, repoState.Revision, compileTaskName) 291 } 292} 293 294// uploadFileToGCS uploads a file to the codesize.skia.org GCS bucket. 295func uploadFileToGCS(ctx context.Context, gcsClient gcs.GCSClient, path string, contents []byte) error { 296 gcsURL := fmt.Sprintf("gs://%s/%s", gcsBucketName, path) 297 return td.Do(ctx, td.Props(fmt.Sprintf("Upload %s", gcsURL)), func(ctx context.Context) error { 298 if err := gcsClient.SetFileContents(ctx, path, gcs.FILE_WRITE_OPTS_TEXT, contents); err != nil { 299 return fmt.Errorf("Could not write task to %s: %s", gcsURL, err) 300 } 301 return nil 302 }) 303} 304