1// Copyright 2024 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package find_input_delta_lib 16 17import ( 18 "fmt" 19 "hash/fnv" 20 "io" 21 "os" 22 "path/filepath" 23 "strings" 24 "text/template" 25 26 fid_exp "android/soong/cmd/find_input_delta/find_input_delta_proto" 27 "google.golang.org/protobuf/encoding/protowire" 28 "google.golang.org/protobuf/proto" 29) 30 31var DefaultTemplate = ` 32 {{- define "contents"}} 33 {{- range .Deletions}}-{{.}} {{end}} 34 {{- range .Additions}}+{{.}} {{end}} 35 {{- range .Changes}}+{{- .Name}} {{end}} 36 {{- range .Changes}} 37 {{- if or .Additions .Deletions .Changes}}--file {{.Name}} {{template "contents" .}}--endfile {{end}} 38 {{- end}} 39 {{- end}} 40 {{- template "contents" .}}` 41 42type FileList struct { 43 // The name of the parent for the list of file differences. 44 // For the outermost FileList, this is the name of the ninja target. 45 // Under `Changes`, it is the name of the changed file. 46 Name string 47 48 // The added files 49 Additions []string 50 51 // The deleted files 52 Deletions []string 53 54 // The modified files 55 Changes []FileList 56 57 // Map of file_extension:counts 58 ExtCountMap map[string]*FileCounts 59 60 // Total number of added/changed/deleted files. 61 TotalDelta uint32 62} 63 64// The maximum number of files that will be recorded by name. 65var MaxFilesRecorded uint32 = 50 66 67type FileCounts struct { 68 Additions uint32 69 Deletions uint32 70 Changes uint32 71} 72 73func FileListFactory(name string) *FileList { 74 return &FileList{ 75 Name: name, 76 ExtCountMap: make(map[string]*FileCounts), 77 } 78} 79 80func (fl *FileList) addFile(name string) { 81 fl.Additions = append(fl.Additions, name) 82 fl.TotalDelta += 1 83 ext := filepath.Ext(name) 84 if _, ok := fl.ExtCountMap[ext]; !ok { 85 fl.ExtCountMap[ext] = &FileCounts{} 86 } 87 fl.ExtCountMap[ext].Additions += 1 88} 89 90func (fl *FileList) deleteFile(name string) { 91 fl.Deletions = append(fl.Deletions, name) 92 fl.TotalDelta += 1 93 ext := filepath.Ext(name) 94 if _, ok := fl.ExtCountMap[ext]; !ok { 95 fl.ExtCountMap[ext] = &FileCounts{} 96 } 97 fl.ExtCountMap[ext].Deletions += 1 98} 99 100func (fl *FileList) changeFile(name string, ch *FileList) { 101 fl.Changes = append(fl.Changes, *ch) 102 fl.TotalDelta += 1 103 ext := filepath.Ext(name) 104 if _, ok := fl.ExtCountMap[ext]; !ok { 105 fl.ExtCountMap[ext] = &FileCounts{} 106 } 107 fl.ExtCountMap[ext].Changes += 1 108} 109 110// Write a SoongExecutionMetrics FileList proto to `dir`. 111// 112// Path 113// Prune any paths that 114// begin with `pruneDir` (usually ${OUT_DIR}). The file is only written if any 115// non-pruned changes are present. 116func (fl *FileList) WriteMetrics(dir, pruneDir string) (err error) { 117 if dir == "" { 118 return fmt.Errorf("No directory given") 119 } 120 var needed bool 121 122 if !strings.HasSuffix(pruneDir, "/") { 123 pruneDir += "/" 124 } 125 126 // Hash the dir and `fl.Name` to simplify scanning the metrics 127 // aggregation directory. 128 h := fnv.New128() 129 h.Write([]byte(dir + " " + fl.Name + ".FileList")) 130 path := fmt.Sprintf("%x.pb", h.Sum([]byte{})) 131 path = filepath.Join(dir, path[0:2], path[2:]) 132 133 var msg = &fid_exp.FileList{Name: proto.String(fl.Name)} 134 for _, a := range fl.Additions { 135 if strings.HasPrefix(a, pruneDir) { 136 continue 137 } 138 msg.Additions = append(msg.Additions, a) 139 needed = true 140 } 141 for _, ch := range fl.Changes { 142 if strings.HasPrefix(ch.Name, pruneDir) { 143 continue 144 } 145 msg.Changes = append(msg.Changes, ch.Name) 146 needed = true 147 } 148 for _, d := range fl.Deletions { 149 if strings.HasPrefix(d, pruneDir) { 150 continue 151 } 152 msg.Deletions = append(msg.Deletions, d) 153 needed = true 154 } 155 if !needed { 156 return nil 157 } 158 data := protowire.AppendVarint( 159 []byte{}, 160 protowire.EncodeTag( 161 protowire.Number(fid_exp.FieldNumbers_FIELD_NUMBERS_FILE_LIST), 162 protowire.BytesType)) 163 size := uint64(proto.Size(msg)) 164 data = protowire.AppendVarint(data, size) 165 data, err = proto.MarshalOptions{UseCachedSize: true}.MarshalAppend(data, msg) 166 if err != nil { 167 return err 168 } 169 170 err = os.MkdirAll(filepath.Dir(path), 0777) 171 if err != nil { 172 return err 173 } 174 175 return os.WriteFile(path, data, 0644) 176} 177 178func (fl *FileList) Format(wr io.Writer, format string) error { 179 tmpl, err := template.New("filelist").Parse(format) 180 if err != nil { 181 return err 182 } 183 return tmpl.Execute(wr, fl) 184} 185