1// Copyright 2015 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15// bpglob is the command line tool that checks if the list of files matching a glob has 16// changed, and only updates the output file list if it has changed. It is used to optimize 17// out build.ninja regenerations when non-matching files are added. See 18// github.com/google/blueprint/bootstrap/glob.go for a longer description. 19package main 20 21import ( 22 "bytes" 23 "errors" 24 "flag" 25 "fmt" 26 "io/ioutil" 27 "os" 28 "strconv" 29 "time" 30 31 "github.com/google/blueprint/deptools" 32 "github.com/google/blueprint/pathtools" 33) 34 35var ( 36 // flagSet is a flag.FlagSet with flag.ContinueOnError so that we can handle the versionMismatchError 37 // error from versionArg. 38 flagSet = flag.NewFlagSet("bpglob", flag.ContinueOnError) 39 40 out = flagSet.String("o", "", "file to write list of files that match glob") 41 42 versionMatch versionArg 43 globs []globArg 44) 45 46func init() { 47 flagSet.Var(&versionMatch, "v", "version number the command line was generated for") 48 flagSet.Var((*patternsArgs)(&globs), "p", "pattern to include in results") 49 flagSet.Var((*excludeArgs)(&globs), "e", "pattern to exclude from results from the most recent pattern") 50} 51 52// bpglob is executed through the rules in build-globs.ninja to determine whether soong_build 53// needs to rerun. That means when the arguments accepted by bpglob change it will be called 54// with the old arguments, then soong_build will rerun and update build-globs.ninja with the new 55// arguments. 56// 57// To avoid having to maintain backwards compatibility with old arguments across the transition, 58// a version argument is used to detect the transition in order to stop parsing arguments, touch the 59// output file and exit immediately. Aborting parsing arguments is necessary to handle parsing 60// errors that would be fatal, for example the removal of a flag. The version number in 61// pathtools.BPGlobArgumentVersion should be manually incremented when the bpglob argument format 62// changes. 63// 64// If the version argument is not passed then a version mismatch is assumed. 65 66// versionArg checks the argument against pathtools.BPGlobArgumentVersion, returning a 67// versionMismatchError error if it does not match. 68type versionArg bool 69 70var versionMismatchError = errors.New("version mismatch") 71 72func (v *versionArg) String() string { return "" } 73 74func (v *versionArg) Set(s string) error { 75 vers, err := strconv.Atoi(s) 76 if err != nil { 77 return fmt.Errorf("error parsing version argument: %w", err) 78 } 79 80 // Force the -o argument to come before the -v argument so that the output file can be 81 // updated on error. 82 if *out == "" { 83 return fmt.Errorf("-o argument must be passed before -v") 84 } 85 86 if vers != pathtools.BPGlobArgumentVersion { 87 return versionMismatchError 88 } 89 90 *v = true 91 92 return nil 93} 94 95// A glob arg holds a single -p argument with zero or more following -e arguments. 96type globArg struct { 97 pattern string 98 excludes []string 99} 100 101// patternsArgs implements flag.Value to handle -p arguments by adding a new globArg to the list. 102type patternsArgs []globArg 103 104func (p *patternsArgs) String() string { return `""` } 105 106func (p *patternsArgs) Set(s string) error { 107 globs = append(globs, globArg{ 108 pattern: s, 109 }) 110 return nil 111} 112 113// excludeArgs implements flag.Value to handle -e arguments by adding to the last globArg in the 114// list. 115type excludeArgs []globArg 116 117func (e *excludeArgs) String() string { return `""` } 118 119func (e *excludeArgs) Set(s string) error { 120 if len(*e) == 0 { 121 return fmt.Errorf("-p argument is required before the first -e argument") 122 } 123 124 glob := &(*e)[len(*e)-1] 125 glob.excludes = append(glob.excludes, s) 126 return nil 127} 128 129func usage() { 130 fmt.Fprintln(os.Stderr, "usage: bpglob -o out -v version -p glob [-e excludes ...] [-p glob ...]") 131 flagSet.PrintDefaults() 132 os.Exit(2) 133} 134 135func main() { 136 // Save the command line flag error output to a buffer, the flag package unconditionally 137 // writes an error message to the output on error, and we want to hide the error for the 138 // version mismatch case. 139 flagErrorBuffer := &bytes.Buffer{} 140 flagSet.SetOutput(flagErrorBuffer) 141 142 err := flagSet.Parse(os.Args[1:]) 143 144 if !versionMatch { 145 // A version mismatch error occurs when the arguments written into build-globs.ninja 146 // don't match the format expected by the bpglob binary. This happens during the 147 // first incremental build after bpglob is changed. Handle this case by aborting 148 // argument parsing and updating the output file with something that will always cause 149 // the primary builder to rerun. 150 // This can happen when there is no -v argument or if the -v argument doesn't match 151 // pathtools.BPGlobArgumentVersion. 152 writeErrorOutput(*out, versionMismatchError) 153 os.Exit(0) 154 } 155 156 if err != nil { 157 os.Stderr.Write(flagErrorBuffer.Bytes()) 158 fmt.Fprintln(os.Stderr, "error:", err.Error()) 159 usage() 160 } 161 162 if *out == "" { 163 fmt.Fprintln(os.Stderr, "error: -o is required") 164 usage() 165 } 166 167 if flagSet.NArg() > 0 { 168 usage() 169 } 170 171 err = globsWithDepFile(*out, *out+".d", globs) 172 if err != nil { 173 // Globs here were already run in the primary builder without error. The only errors here should be if the glob 174 // pattern was made invalid by a change in the pathtools glob implementation, in which case the primary builder 175 // needs to be rerun anyways. Update the output file with something that will always cause the primary builder 176 // to rerun. 177 writeErrorOutput(*out, err) 178 } 179} 180 181// writeErrorOutput writes an error to the output file with a timestamp to ensure that it is 182// considered dirty by ninja. 183func writeErrorOutput(path string, globErr error) { 184 s := fmt.Sprintf("%s: error: %s\n", time.Now().Format(time.StampNano), globErr.Error()) 185 err := ioutil.WriteFile(path, []byte(s), 0666) 186 if err != nil { 187 fmt.Fprintf(os.Stderr, "error: %s\n", err.Error()) 188 os.Exit(1) 189 } 190} 191 192// globsWithDepFile finds all files and directories that match glob. Directories 193// will have a trailing '/'. It compares the list of matches against the 194// contents of fileListFile, and rewrites fileListFile if it has changed. It 195// also writes all of the directories it traversed as dependencies on fileListFile 196// to depFile. 197// 198// The format of glob is either path/*.ext for a single directory glob, or 199// path/**/*.ext for a recursive glob. 200func globsWithDepFile(fileListFile, depFile string, globs []globArg) error { 201 var results pathtools.MultipleGlobResults 202 for _, glob := range globs { 203 result, err := pathtools.Glob(glob.pattern, glob.excludes, pathtools.FollowSymlinks) 204 if err != nil { 205 return err 206 } 207 results = append(results, result) 208 } 209 210 // Only write the output file if it has changed. 211 err := pathtools.WriteFileIfChanged(fileListFile, results.FileList(), 0666) 212 if err != nil { 213 return fmt.Errorf("failed to write file list to %q: %w", fileListFile, err) 214 } 215 216 // The depfile can be written unconditionally as its timestamp doesn't affect ninja's restat 217 // feature. 218 err = deptools.WriteDepFile(depFile, fileListFile, results.Deps()) 219 if err != nil { 220 return fmt.Errorf("failed to write dep file to %q: %w", depFile, err) 221 } 222 223 return nil 224} 225