1// Copyright 2020 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package build 16 17// This file contains the functionality to upload data from one location to 18// another. 19 20import ( 21 "io/ioutil" 22 "os" 23 "path/filepath" 24 "time" 25 26 "android/soong/ui/metrics" 27 28 "google.golang.org/protobuf/proto" 29 30 upload_proto "android/soong/ui/metrics/upload_proto" 31) 32 33const ( 34 // Used to generate a raw protobuf file that contains information 35 // of the list of metrics files from host to destination storage. 36 uploadPbFilename = ".uploader.pb" 37) 38 39var ( 40 // For testing purpose. 41 tmpDir = ioutil.TempDir 42) 43 44// pruneMetricsFiles iterates the list of paths, checking if a path exist. 45// If a path is a file, it is added to the return list. If the path is a 46// directory, a recursive call is made to add the children files of the 47// path. 48func pruneMetricsFiles(paths []string) []string { 49 var metricsFiles []string 50 for _, p := range paths { 51 fi, err := os.Stat(p) 52 // Some paths passed may not exist. For example, build errors protobuf 53 // file may not exist since the build was successful. 54 if err != nil { 55 continue 56 } 57 58 if fi.IsDir() { 59 if l, err := ioutil.ReadDir(p); err == nil { 60 files := make([]string, 0, len(l)) 61 for _, fi := range l { 62 files = append(files, filepath.Join(p, fi.Name())) 63 } 64 metricsFiles = append(metricsFiles, pruneMetricsFiles(files)...) 65 } 66 } else { 67 metricsFiles = append(metricsFiles, p) 68 } 69 } 70 return metricsFiles 71} 72 73// UploadMetrics uploads a set of metrics files to a server for analysis. 74// The metrics files are first copied to a temporary directory 75// and the uploader is then executed in the background to allow the user/system 76// to continue working. Soong communicates to the uploader through the 77// upload_proto raw protobuf file. 78func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, paths ...string) { 79 ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics") 80 defer ctx.EndTrace() 81 82 uploader := config.MetricsUploaderApp() 83 if uploader == "" { 84 // If the uploader path was not specified, no metrics shall be uploaded. 85 return 86 } 87 88 // Several of the files might be directories. 89 metricsFiles := pruneMetricsFiles(paths) 90 if len(metricsFiles) == 0 { 91 return 92 } 93 94 // The temporary directory cannot be deleted as the metrics uploader is started 95 // in the background and requires to exist until the operation is done. The 96 // uploader can delete the directory as it is specified in the upload proto. 97 tmpDir, err := tmpDir("", "upload_metrics") 98 if err != nil { 99 ctx.Fatalf("failed to create a temporary directory to store the list of metrics files: %v\n", err) 100 } 101 102 for i, src := range metricsFiles { 103 dst := filepath.Join(tmpDir, filepath.Base(src)) 104 if _, err := copyFile(src, dst); err != nil { 105 ctx.Fatalf("failed to copy %q to %q: %v\n", src, dst, err) 106 } 107 metricsFiles[i] = dst 108 } 109 110 // For platform builds, the branch and target name is hardcoded to specific 111 // values for later extraction of the metrics in the data metrics pipeline. 112 data, err := proto.Marshal(&upload_proto.Upload{ 113 CreationTimestampMs: proto.Uint64(uint64(buildStarted.UnixNano() / int64(time.Millisecond))), 114 CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))), 115 BranchName: proto.String("developer-metrics"), 116 TargetName: proto.String("platform-build-systems-metrics"), 117 MetricsFiles: metricsFiles, 118 DirectoriesToDelete: []string{tmpDir}, 119 }) 120 if err != nil { 121 ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err) 122 } 123 124 pbFile := filepath.Join(tmpDir, uploadPbFilename) 125 if err := ioutil.WriteFile(pbFile, data, 0644); err != nil { 126 ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err) 127 } 128 129 // Start the uploader in the background as it takes several milliseconds to start the uploader 130 // and prepare the metrics for upload. This affects small shell commands like "lunch". 131 cmd := Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile) 132 if simpleOutput { 133 cmd.RunOrFatal() 134 } else { 135 cmd.RunAndStreamOrFatal() 136 } 137} 138