1// Copyright 2024 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15// Package execution_metrics represents the metrics system for Android Platform Build Systems. 16package execution_metrics 17 18// This is the main heart of the metrics system for Android Platform Build Systems. 19// The starting of the soong_ui (cmd/soong_ui/main.go), the metrics system is 20// initialized by the invocation of New and is then stored in the context 21// (ui/build/context.go) to be used throughout the system. During the build 22// initialization phase, several functions in this file are invoked to store 23// information such as the environment, build configuration and build metadata. 24// There are several scoped code that has Begin() and defer End() functions 25// that captures the metrics and is them added as a perfInfo into the set 26// of the collected metrics. Finally, when soong_ui has finished the build, 27// the defer Dump function is invoked to store the collected metrics to the 28// raw protobuf file in the $OUT directory and this raw protobuf file will be 29// uploaded to the destination. See ui/build/upload.go for more details. The 30// filename of the raw protobuf file and the list of files to be uploaded is 31// defined in cmd/soong_ui/main.go. See ui/metrics/event.go for the explanation 32// of what an event is and how the metrics system is a stack based system. 33 34import ( 35 "context" 36 "io/fs" 37 "maps" 38 "os" 39 "path/filepath" 40 "slices" 41 "sync" 42 43 "android/soong/ui/logger" 44 45 fid_proto "android/soong/cmd/find_input_delta/find_input_delta_proto" 46 "android/soong/ui/metrics" 47 soong_execution_proto "android/soong/ui/metrics/execution_metrics_proto" 48 soong_metrics_proto "android/soong/ui/metrics/metrics_proto" 49 "google.golang.org/protobuf/encoding/protowire" 50 "google.golang.org/protobuf/proto" 51) 52 53type ExecutionMetrics struct { 54 MetricsAggregationDir string 55 ctx context.Context 56 logger logger.Logger 57 waitGroup sync.WaitGroup 58 fileList *fileList 59} 60 61type fileList struct { 62 totalChanges uint32 63 changes fileChanges 64 seenFiles map[string]bool 65} 66 67type fileChanges struct { 68 additions changeInfo 69 deletions changeInfo 70 modifications changeInfo 71} 72 73type fileChangeCounts struct { 74 additions uint32 75 deletions uint32 76 modifications uint32 77} 78 79type changeInfo struct { 80 total uint32 81 list []string 82 byExtension map[string]uint32 83} 84 85var MAXIMUM_FILES uint32 = 50 86 87// Setup the handler for SoongExecutionMetrics. 88func NewExecutionMetrics(log logger.Logger) *ExecutionMetrics { 89 return &ExecutionMetrics{ 90 logger: log, 91 fileList: &fileList{seenFiles: make(map[string]bool)}, 92 } 93} 94 95// Save the path for ExecutionMetrics communications. 96func (c *ExecutionMetrics) SetDir(path string) { 97 c.MetricsAggregationDir = path 98} 99 100// Start collecting SoongExecutionMetrics. 101func (c *ExecutionMetrics) Start() { 102 if c.MetricsAggregationDir == "" { 103 return 104 } 105 106 tmpDir := c.MetricsAggregationDir + ".rm" 107 if _, err := fs.Stat(os.DirFS("."), c.MetricsAggregationDir); err == nil { 108 if err = os.RemoveAll(tmpDir); err != nil { 109 c.logger.Fatalf("Failed to remove %s: %v", tmpDir, err) 110 } 111 if err = os.Rename(c.MetricsAggregationDir, tmpDir); err != nil { 112 c.logger.Fatalf("Failed to rename %s to %s: %v", c.MetricsAggregationDir, tmpDir) 113 } 114 } 115 if err := os.MkdirAll(c.MetricsAggregationDir, 0777); err != nil { 116 c.logger.Fatalf("Failed to create %s: %v", c.MetricsAggregationDir) 117 } 118 119 c.waitGroup.Add(1) 120 go func(d string) { 121 defer c.waitGroup.Done() 122 os.RemoveAll(d) 123 }(tmpDir) 124 125 c.logger.Verbosef("ExecutionMetrics running\n") 126} 127 128type hasTrace interface { 129 BeginTrace(name, desc string) 130 EndTrace() 131} 132 133// Aggregate any execution metrics. 134func (c *ExecutionMetrics) Finish(ctx hasTrace) { 135 ctx.BeginTrace(metrics.RunSoong, "execution_metrics.Finish") 136 defer ctx.EndTrace() 137 if c.MetricsAggregationDir == "" { 138 return 139 } 140 c.waitGroup.Wait() 141 142 // Find and process all of the metrics files. 143 aggFs := os.DirFS(c.MetricsAggregationDir) 144 fs.WalkDir(aggFs, ".", func(path string, d fs.DirEntry, err error) error { 145 if err != nil { 146 c.logger.Fatalf("ExecutionMetrics.Finish: Error walking %s: %v", c.MetricsAggregationDir, err) 147 } 148 if d.IsDir() { 149 return nil 150 } 151 path = filepath.Join(c.MetricsAggregationDir, path) 152 r, err := os.ReadFile(path) 153 if err != nil { 154 c.logger.Fatalf("ExecutionMetrics.Finish: Failed to read %s: %v", path, err) 155 } 156 msg := &soong_execution_proto.SoongExecutionMetrics{} 157 err = proto.Unmarshal(r, msg) 158 if err != nil { 159 c.logger.Verbosef("ExecutionMetrics.Finish: Error unmarshalling SoongExecutionMetrics message: %v\n", err) 160 return nil 161 } 162 switch { 163 case msg.GetFileList() != nil: 164 if err := c.fileList.aggregateFileList(msg.GetFileList()); err != nil { 165 c.logger.Verbosef("ExecutionMetrics.Finish: Error processing SoongExecutionMetrics message: %v\n", err) 166 } 167 // Status update for all others. 168 default: 169 tag, _ := protowire.ConsumeVarint(r) 170 id, _ := protowire.DecodeTag(tag) 171 c.logger.Verbosef("ExecutionMetrics.Finish: Unexpected SoongExecutionMetrics submessage id=%d\n", id) 172 } 173 return nil 174 }) 175} 176 177func (fl *fileList) aggregateFileList(msg *fid_proto.FileList) error { 178 fl.updateChangeInfo(msg.GetAdditions(), &fl.changes.additions) 179 fl.updateChangeInfo(msg.GetDeletions(), &fl.changes.deletions) 180 fl.updateChangeInfo(msg.GetChanges(), &fl.changes.modifications) 181 return nil 182} 183 184func (fl *fileList) updateChangeInfo(list []string, info *changeInfo) { 185 for _, filename := range list { 186 if fl.seenFiles[filename] { 187 continue 188 } 189 fl.seenFiles[filename] = true 190 if info.total < MAXIMUM_FILES { 191 info.list = append(info.list, filename) 192 } 193 ext := filepath.Ext(filename) 194 if info.byExtension == nil { 195 info.byExtension = make(map[string]uint32) 196 } 197 info.byExtension[ext] += 1 198 info.total += 1 199 fl.totalChanges += 1 200 } 201} 202 203func (c *ExecutionMetrics) Dump(path string, args []string) error { 204 if c.MetricsAggregationDir == "" { 205 return nil 206 } 207 msg := c.GetMetrics(args) 208 209 if _, err := os.Stat(filepath.Dir(path)); err != nil { 210 if err = os.MkdirAll(filepath.Dir(path), 0775); err != nil { 211 return err 212 } 213 } 214 data, err := proto.Marshal(msg) 215 if err != nil { 216 return err 217 } 218 return os.WriteFile(path, data, 0644) 219} 220 221func (c *ExecutionMetrics) GetMetrics(args []string) *soong_metrics_proto.ExecutionMetrics { 222 return &soong_metrics_proto.ExecutionMetrics{ 223 CommandArgs: args, 224 ChangedFiles: c.getChangedFiles(), 225 } 226} 227 228func (c *ExecutionMetrics) getChangedFiles() *soong_metrics_proto.AggregatedFileList { 229 fl := c.fileList 230 if fl == nil { 231 return nil 232 } 233 var count uint32 234 fileCounts := make(map[string]*soong_metrics_proto.FileCount) 235 ret := &soong_metrics_proto.AggregatedFileList{TotalDelta: proto.Uint32(c.fileList.totalChanges)} 236 237 // MAXIMUM_FILES is the upper bound on total file names reported. 238 if limit := min(MAXIMUM_FILES-min(MAXIMUM_FILES, count), fl.changes.additions.total); limit > 0 { 239 ret.Additions = fl.changes.additions.list[:limit] 240 count += limit 241 } 242 if limit := min(MAXIMUM_FILES-min(MAXIMUM_FILES, count), fl.changes.modifications.total); limit > 0 { 243 ret.Changes = fl.changes.modifications.list[:limit] 244 count += limit 245 } 246 if limit := min(MAXIMUM_FILES-min(MAXIMUM_FILES, count), fl.changes.deletions.total); limit > 0 { 247 ret.Deletions = fl.changes.deletions.list[:limit] 248 count += limit 249 } 250 251 addExt := func(key string) *soong_metrics_proto.FileCount { 252 // Create the fileCounts map entry if needed, and return the address to the caller. 253 if _, ok := fileCounts[key]; !ok { 254 fileCounts[key] = &soong_metrics_proto.FileCount{Extension: proto.String(key)} 255 } 256 return fileCounts[key] 257 } 258 addCount := func(loc **uint32, count uint32) { 259 if *loc == nil { 260 *loc = proto.Uint32(0) 261 } 262 **loc += count 263 } 264 for k, v := range fl.changes.additions.byExtension { 265 addCount(&addExt(k).Additions, v) 266 } 267 for k, v := range fl.changes.modifications.byExtension { 268 addCount(&addExt(k).Modifications, v) 269 } 270 for k, v := range fl.changes.deletions.byExtension { 271 addCount(&addExt(k).Deletions, v) 272 } 273 274 keys := slices.Sorted(maps.Keys(fileCounts)) 275 for _, k := range keys { 276 ret.Counts = append(ret.Counts, fileCounts[k]) 277 } 278 return ret 279} 280