1// Copyright 2014 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package pathtools 16 17import ( 18 "encoding/json" 19 "errors" 20 "fmt" 21 "io/ioutil" 22 "os" 23 "path/filepath" 24 "strings" 25) 26 27// BPGlobArgumentVersion is used to abort argument parsing early when the bpglob argument format 28// has changed but soong_build hasn't had a chance to rerun yet to update build-globs.ninja. 29// Increment it manually when changing the bpglob argument format. It is located here because 30// pathtools is the only package that is shared between bpglob and bootstrap. 31const BPGlobArgumentVersion = 2 32 33var GlobMultipleRecursiveErr = errors.New("pattern contains multiple '**'") 34var GlobLastRecursiveErr = errors.New("pattern has '**' as last path element") 35var GlobInvalidRecursiveErr = errors.New("pattern contains other characters between '**' and path separator") 36 37// GlobResult is a container holding the results of a call to Glob. 38type GlobResult struct { 39 // Pattern is the pattern that was passed to Glob. 40 Pattern string 41 // Excludes is the list of excludes that were passed to Glob. 42 Excludes []string 43 44 // Matches is the list of files or directories that matched the pattern but not the excludes. 45 Matches []string 46 47 // Deps is the list of files or directories that must be depended on to regenerate the glob. 48 Deps []string 49} 50 51// FileList returns the list of files matched by a glob for writing to an output file. 52func (result GlobResult) FileList() []byte { 53 return []byte(strings.Join(result.Matches, "\n") + "\n") 54} 55 56// MultipleGlobResults is a list of GlobResult structs. 57type MultipleGlobResults []GlobResult 58 59// FileList returns the list of files matched by a list of multiple globs for writing to an output file. 60func (results MultipleGlobResults) FileList() []byte { 61 multipleMatches := make([][]string, len(results)) 62 for i, result := range results { 63 multipleMatches[i] = result.Matches 64 } 65 buf, err := json.Marshal(multipleMatches) 66 if err != nil { 67 panic(fmt.Errorf("failed to marshal glob results to json: %w", err)) 68 } 69 return buf 70} 71 72// Deps returns the deps from all of the GlobResults. 73func (results MultipleGlobResults) Deps() []string { 74 var deps []string 75 for _, result := range results { 76 deps = append(deps, result.Deps...) 77 } 78 return deps 79} 80 81// Glob returns the list of files and directories that match the given pattern 82// but do not match the given exclude patterns, along with the list of 83// directories and other dependencies that were searched to construct the file 84// list. The supported glob and exclude patterns are equivalent to 85// filepath.Glob, with an extension that recursive glob (** matching zero or 86// more complete path entries) is supported. Any directories in the matches 87// list will have a '/' suffix. 88// 89// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps 90// should be used instead, as they will automatically set up dependencies 91// to rerun the primary builder when the list of matching files changes. 92func Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (GlobResult, error) { 93 return startGlob(OsFs, pattern, excludes, follow) 94} 95 96func startGlob(fs FileSystem, pattern string, excludes []string, 97 follow ShouldFollowSymlinks) (GlobResult, error) { 98 99 if filepath.Base(pattern) == "**" { 100 return GlobResult{}, GlobLastRecursiveErr 101 } 102 103 matches, deps, err := glob(fs, pattern, false, follow) 104 105 if err != nil { 106 return GlobResult{}, err 107 } 108 109 matches, err = filterExcludes(matches, excludes) 110 if err != nil { 111 return GlobResult{}, err 112 } 113 114 // If the pattern has wildcards, we added dependencies on the 115 // containing directories to know about changes. 116 // 117 // If the pattern didn't have wildcards, and didn't find matches, the 118 // most specific found directories were added. 119 // 120 // But if it didn't have wildcards, and did find a match, no 121 // dependencies were added, so add the match itself to detect when it 122 // is removed. 123 if !isWild(pattern) { 124 deps = append(deps, matches...) 125 } 126 127 for i, match := range matches { 128 var info os.FileInfo 129 if follow == DontFollowSymlinks { 130 info, err = fs.Lstat(match) 131 } else { 132 info, err = fs.Stat(match) 133 } 134 if err != nil { 135 return GlobResult{}, err 136 } 137 138 if info.IsDir() { 139 matches[i] = match + "/" 140 } 141 } 142 143 return GlobResult{ 144 Pattern: pattern, 145 Excludes: excludes, 146 Matches: matches, 147 Deps: deps, 148 }, nil 149} 150 151// glob is a recursive helper function to handle globbing each level of the pattern individually, 152// allowing searched directories to be tracked. Also handles the recursive glob pattern, **. 153func glob(fs FileSystem, pattern string, hasRecursive bool, 154 follow ShouldFollowSymlinks) (matches, dirs []string, err error) { 155 156 if !isWild(pattern) { 157 // If there are no wilds in the pattern, check whether the file exists or not. 158 // Uses filepath.Glob instead of manually statting to get consistent results. 159 pattern = filepath.Clean(pattern) 160 matches, err = fs.glob(pattern) 161 if err != nil { 162 return matches, dirs, err 163 } 164 165 if len(matches) == 0 { 166 // Some part of the non-wild pattern didn't exist. Add the last existing directory 167 // as a dependency. 168 var matchDirs []string 169 for len(matchDirs) == 0 { 170 pattern = filepath.Dir(pattern) 171 matchDirs, err = fs.glob(pattern) 172 if err != nil { 173 return matches, dirs, err 174 } 175 } 176 dirs = append(dirs, matchDirs...) 177 } 178 return matches, dirs, err 179 } 180 181 dir, file := saneSplit(pattern) 182 183 if file == "**" { 184 if hasRecursive { 185 return matches, dirs, GlobMultipleRecursiveErr 186 } 187 hasRecursive = true 188 } else if strings.Contains(file, "**") { 189 return matches, dirs, GlobInvalidRecursiveErr 190 } 191 192 dirMatches, dirs, err := glob(fs, dir, hasRecursive, follow) 193 if err != nil { 194 return nil, nil, err 195 } 196 197 for _, m := range dirMatches { 198 isDir, err := fs.IsDir(m) 199 if os.IsNotExist(err) { 200 if isSymlink, _ := fs.IsSymlink(m); isSymlink { 201 return nil, nil, fmt.Errorf("dangling symlink: %s", m) 202 } 203 } 204 if err != nil { 205 return nil, nil, fmt.Errorf("unexpected error after glob: %s", err) 206 } 207 208 if isDir { 209 if file == "**" { 210 recurseDirs, err := fs.ListDirsRecursive(m, follow) 211 if err != nil { 212 return nil, nil, err 213 } 214 matches = append(matches, recurseDirs...) 215 } else { 216 dirs = append(dirs, m) 217 newMatches, err := fs.glob(filepath.Join(MatchEscape(m), file)) 218 if err != nil { 219 return nil, nil, err 220 } 221 if file[0] != '.' { 222 newMatches = filterDotFiles(newMatches) 223 } 224 matches = append(matches, newMatches...) 225 } 226 } 227 } 228 229 return matches, dirs, nil 230} 231 232// Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations 233// Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is 234// not "/". Returns ".", "" if path is "." 235func saneSplit(path string) (dir, file string) { 236 if path == "." { 237 return ".", "" 238 } 239 dir, file = filepath.Split(path) 240 switch dir { 241 case "": 242 dir = "." 243 case "/": 244 // Nothing 245 default: 246 dir = dir[:len(dir)-1] 247 } 248 return dir, file 249} 250 251func isWild(pattern string) bool { 252 return strings.ContainsAny(pattern, "*?[") 253} 254 255// Filters the strings in matches based on the glob patterns in excludes. Hierarchical (a/*) and 256// recursive (**) glob patterns are supported. 257func filterExcludes(matches []string, excludes []string) ([]string, error) { 258 if len(excludes) == 0 { 259 return matches, nil 260 } 261 262 var ret []string 263matchLoop: 264 for _, m := range matches { 265 for _, e := range excludes { 266 exclude, err := Match(e, m) 267 if err != nil { 268 return nil, err 269 } 270 if exclude { 271 continue matchLoop 272 } 273 } 274 ret = append(ret, m) 275 } 276 277 return ret, nil 278} 279 280// filterDotFiles filters out files that start with '.' 281func filterDotFiles(matches []string) []string { 282 ret := make([]string, 0, len(matches)) 283 284 for _, match := range matches { 285 _, name := filepath.Split(match) 286 if name[0] == '.' { 287 continue 288 } 289 ret = append(ret, match) 290 } 291 292 return ret 293} 294 295// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting 296// recursive globs (**). 297func Match(pattern, name string) (bool, error) { 298 if filepath.Base(pattern) == "**" { 299 return false, GlobLastRecursiveErr 300 } 301 302 patternDir := pattern[len(pattern)-1] == '/' 303 nameDir := name[len(name)-1] == '/' 304 305 if patternDir != nameDir { 306 return false, nil 307 } 308 309 if nameDir { 310 name = name[:len(name)-1] 311 pattern = pattern[:len(pattern)-1] 312 } 313 314 for { 315 var patternFile, nameFile string 316 pattern, patternFile = filepath.Dir(pattern), filepath.Base(pattern) 317 318 if patternFile == "**" { 319 if strings.Contains(pattern, "**") { 320 return false, GlobMultipleRecursiveErr 321 } 322 // Test if the any prefix of name matches the part of the pattern before ** 323 for { 324 if name == "." || name == "/" { 325 return name == pattern, nil 326 } 327 if match, err := filepath.Match(pattern, name); err != nil { 328 return false, err 329 } else if match { 330 return true, nil 331 } 332 name = filepath.Dir(name) 333 } 334 } else if strings.Contains(patternFile, "**") { 335 return false, GlobInvalidRecursiveErr 336 } 337 338 name, nameFile = filepath.Dir(name), filepath.Base(name) 339 340 if nameFile == "." && patternFile == "." { 341 return true, nil 342 } else if nameFile == "/" && patternFile == "/" { 343 return true, nil 344 } else if nameFile == "." || patternFile == "." || nameFile == "/" || patternFile == "/" { 345 return false, nil 346 } 347 348 match, err := filepath.Match(patternFile, nameFile) 349 if err != nil || !match { 350 return match, err 351 } 352 } 353} 354 355// IsGlob returns true if the pattern contains any glob characters (*, ?, or [). 356func IsGlob(pattern string) bool { 357 return strings.IndexAny(pattern, "*?[") >= 0 358} 359 360// HasGlob returns true if any string in the list contains any glob characters (*, ?, or [). 361func HasGlob(in []string) bool { 362 for _, s := range in { 363 if IsGlob(s) { 364 return true 365 } 366 } 367 368 return false 369} 370 371// WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if 372// the files does not already exist with identical contents. This can be used 373// along with ninja restat rules to skip rebuilding downstream rules if no 374// changes were made by a rule. 375func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error { 376 var isChanged bool 377 378 dir := filepath.Dir(filename) 379 err := os.MkdirAll(dir, 0777) 380 if err != nil { 381 return err 382 } 383 384 info, err := os.Stat(filename) 385 if err != nil { 386 if os.IsNotExist(err) { 387 // The file does not exist yet. 388 isChanged = true 389 } else { 390 return err 391 } 392 } else { 393 if info.Size() != int64(len(data)) { 394 isChanged = true 395 } else { 396 oldData, err := ioutil.ReadFile(filename) 397 if err != nil { 398 return err 399 } 400 401 if len(oldData) != len(data) { 402 isChanged = true 403 } else { 404 for i := range data { 405 if oldData[i] != data[i] { 406 isChanged = true 407 break 408 } 409 } 410 } 411 } 412 } 413 414 if isChanged { 415 err = ioutil.WriteFile(filename, data, perm) 416 if err != nil { 417 return err 418 } 419 } 420 421 return nil 422} 423 424var matchEscaper = strings.NewReplacer( 425 `*`, `\*`, 426 `?`, `\?`, 427 `[`, `\[`, 428 `]`, `\]`, 429) 430 431// MatchEscape returns its inputs with characters that would be interpreted by 432func MatchEscape(s string) string { 433 return matchEscaper.Replace(s) 434} 435