1// Copyright 2014 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package pathtools 16 17import ( 18 "errors" 19 "fmt" 20 "io/ioutil" 21 "os" 22 "path/filepath" 23 "strings" 24 25 "github.com/google/blueprint/deptools" 26) 27 28var GlobMultipleRecursiveErr = errors.New("pattern contains multiple **") 29var GlobLastRecursiveErr = errors.New("pattern ** as last path element") 30 31// Glob returns the list of files and directories that match the given pattern 32// but do not match the given exclude patterns, along with the list of 33// directories and other dependencies that were searched to construct the file 34// list. The supported glob and exclude patterns are equivalent to 35// filepath.Glob, with an extension that recursive glob (** matching zero or 36// more complete path entries) is supported. Any directories in the matches 37// list will have a '/' suffix. 38// 39// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps 40// should be used instead, as they will automatically set up dependencies 41// to rerun the primary builder when the list of matching files changes. 42func Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (matches, deps []string, err error) { 43 return startGlob(OsFs, pattern, excludes, follow) 44} 45 46func startGlob(fs FileSystem, pattern string, excludes []string, 47 follow ShouldFollowSymlinks) (matches, deps []string, err error) { 48 49 if filepath.Base(pattern) == "**" { 50 return nil, nil, GlobLastRecursiveErr 51 } else { 52 matches, deps, err = glob(fs, pattern, false, follow) 53 } 54 55 if err != nil { 56 return nil, nil, err 57 } 58 59 matches, err = filterExcludes(matches, excludes) 60 if err != nil { 61 return nil, nil, err 62 } 63 64 // If the pattern has wildcards, we added dependencies on the 65 // containing directories to know about changes. 66 // 67 // If the pattern didn't have wildcards, and didn't find matches, the 68 // most specific found directories were added. 69 // 70 // But if it didn't have wildcards, and did find a match, no 71 // dependencies were added, so add the match itself to detect when it 72 // is removed. 73 if !isWild(pattern) { 74 deps = append(deps, matches...) 75 } 76 77 for i, match := range matches { 78 isSymlink, err := fs.IsSymlink(match) 79 if err != nil { 80 return nil, nil, err 81 } 82 if !(isSymlink && follow == DontFollowSymlinks) { 83 isDir, err := fs.IsDir(match) 84 if os.IsNotExist(err) { 85 if isSymlink { 86 return nil, nil, fmt.Errorf("%s: dangling symlink", match) 87 } 88 } 89 if err != nil { 90 return nil, nil, fmt.Errorf("%s: %s", match, err.Error()) 91 } 92 93 if isDir { 94 matches[i] = match + "/" 95 } 96 } 97 } 98 99 return matches, deps, nil 100} 101 102// glob is a recursive helper function to handle globbing each level of the pattern individually, 103// allowing searched directories to be tracked. Also handles the recursive glob pattern, **. 104func glob(fs FileSystem, pattern string, hasRecursive bool, 105 follow ShouldFollowSymlinks) (matches, dirs []string, err error) { 106 107 if !isWild(pattern) { 108 // If there are no wilds in the pattern, check whether the file exists or not. 109 // Uses filepath.Glob instead of manually statting to get consistent results. 110 pattern = filepath.Clean(pattern) 111 matches, err = fs.glob(pattern) 112 if err != nil { 113 return matches, dirs, err 114 } 115 116 if len(matches) == 0 { 117 // Some part of the non-wild pattern didn't exist. Add the last existing directory 118 // as a dependency. 119 var matchDirs []string 120 for len(matchDirs) == 0 { 121 pattern, _ = saneSplit(pattern) 122 matchDirs, err = fs.glob(pattern) 123 if err != nil { 124 return matches, dirs, err 125 } 126 } 127 dirs = append(dirs, matchDirs...) 128 } 129 return matches, dirs, err 130 } 131 132 dir, file := saneSplit(pattern) 133 134 if file == "**" { 135 if hasRecursive { 136 return matches, dirs, GlobMultipleRecursiveErr 137 } 138 hasRecursive = true 139 } 140 141 dirMatches, dirs, err := glob(fs, dir, hasRecursive, follow) 142 if err != nil { 143 return nil, nil, err 144 } 145 146 for _, m := range dirMatches { 147 isDir, err := fs.IsDir(m) 148 if os.IsNotExist(err) { 149 if isSymlink, _ := fs.IsSymlink(m); isSymlink { 150 return nil, nil, fmt.Errorf("dangling symlink: %s", m) 151 } 152 } 153 if err != nil { 154 return nil, nil, fmt.Errorf("unexpected error after glob: %s", err) 155 } 156 157 if isDir { 158 if file == "**" { 159 recurseDirs, err := fs.ListDirsRecursive(m, follow) 160 if err != nil { 161 return nil, nil, err 162 } 163 matches = append(matches, recurseDirs...) 164 } else { 165 dirs = append(dirs, m) 166 newMatches, err := fs.glob(filepath.Join(MatchEscape(m), file)) 167 if err != nil { 168 return nil, nil, err 169 } 170 if file[0] != '.' { 171 newMatches = filterDotFiles(newMatches) 172 } 173 matches = append(matches, newMatches...) 174 } 175 } 176 } 177 178 return matches, dirs, nil 179} 180 181// Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations 182// Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is 183// not "/". Returns ".", "" if path is "." 184func saneSplit(path string) (dir, file string) { 185 if path == "." { 186 return ".", "" 187 } 188 dir, file = filepath.Split(path) 189 switch dir { 190 case "": 191 dir = "." 192 case "/": 193 // Nothing 194 default: 195 dir = dir[:len(dir)-1] 196 } 197 return dir, file 198} 199 200func isWild(pattern string) bool { 201 return strings.ContainsAny(pattern, "*?[") 202} 203 204// Filters the strings in matches based on the glob patterns in excludes. Hierarchical (a/*) and 205// recursive (**) glob patterns are supported. 206func filterExcludes(matches []string, excludes []string) ([]string, error) { 207 if len(excludes) == 0 { 208 return matches, nil 209 } 210 211 var ret []string 212matchLoop: 213 for _, m := range matches { 214 for _, e := range excludes { 215 exclude, err := Match(e, m) 216 if err != nil { 217 return nil, err 218 } 219 if exclude { 220 continue matchLoop 221 } 222 } 223 ret = append(ret, m) 224 } 225 226 return ret, nil 227} 228 229// filterDotFiles filters out files that start with '.' 230func filterDotFiles(matches []string) []string { 231 ret := make([]string, 0, len(matches)) 232 233 for _, match := range matches { 234 _, name := filepath.Split(match) 235 if name[0] == '.' { 236 continue 237 } 238 ret = append(ret, match) 239 } 240 241 return ret 242} 243 244// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting 245// hierarchical patterns (a/*) and recursive globs (**). 246func Match(pattern, name string) (bool, error) { 247 if filepath.Base(pattern) == "**" { 248 return false, GlobLastRecursiveErr 249 } 250 251 patternDir := pattern[len(pattern)-1] == '/' 252 nameDir := name[len(name)-1] == '/' 253 254 if patternDir != nameDir { 255 return false, nil 256 } 257 258 if nameDir { 259 name = name[:len(name)-1] 260 pattern = pattern[:len(pattern)-1] 261 } 262 263 for { 264 var patternFile, nameFile string 265 pattern, patternFile = saneSplit(pattern) 266 name, nameFile = saneSplit(name) 267 268 if patternFile == "**" { 269 return matchPrefix(pattern, filepath.Join(name, nameFile)) 270 } 271 272 if nameFile == "" && patternFile == "" { 273 return true, nil 274 } else if nameFile == "" || patternFile == "" { 275 return false, nil 276 } 277 278 match, err := filepath.Match(patternFile, nameFile) 279 if err != nil || !match { 280 return match, err 281 } 282 } 283} 284 285// matchPrefix returns true if the beginning of name matches pattern using the same rules as 286// filepath.Match, but supporting hierarchical patterns (a/*). Recursive globs (**) are not 287// supported, they should have been handled in Match(). 288func matchPrefix(pattern, name string) (bool, error) { 289 if len(pattern) > 0 && pattern[0] == '/' { 290 if len(name) > 0 && name[0] == '/' { 291 pattern = pattern[1:] 292 name = name[1:] 293 } else { 294 return false, nil 295 } 296 } 297 298 for { 299 var patternElem, nameElem string 300 patternElem, pattern = saneSplitFirst(pattern) 301 nameElem, name = saneSplitFirst(name) 302 303 if patternElem == "." { 304 patternElem = "" 305 } 306 if nameElem == "." { 307 nameElem = "" 308 } 309 310 if patternElem == "**" { 311 return false, GlobMultipleRecursiveErr 312 } 313 314 if patternElem == "" { 315 return true, nil 316 } else if nameElem == "" { 317 return false, nil 318 } 319 320 match, err := filepath.Match(patternElem, nameElem) 321 if err != nil || !match { 322 return match, err 323 } 324 } 325} 326 327func saneSplitFirst(path string) (string, string) { 328 i := strings.IndexRune(path, filepath.Separator) 329 if i < 0 { 330 return path, "" 331 } 332 return path[:i], path[i+1:] 333} 334 335func GlobPatternList(patterns []string, prefix string) (globedList []string, depDirs []string, err error) { 336 var ( 337 matches []string 338 deps []string 339 ) 340 341 globedList = make([]string, 0) 342 depDirs = make([]string, 0) 343 344 for _, pattern := range patterns { 345 if isWild(pattern) { 346 matches, deps, err = Glob(filepath.Join(prefix, pattern), nil, FollowSymlinks) 347 if err != nil { 348 return nil, nil, err 349 } 350 globedList = append(globedList, matches...) 351 depDirs = append(depDirs, deps...) 352 } else { 353 globedList = append(globedList, filepath.Join(prefix, pattern)) 354 } 355 } 356 return globedList, depDirs, nil 357} 358 359// IsGlob returns true if the pattern contains any glob characters (*, ?, or [). 360func IsGlob(pattern string) bool { 361 return strings.IndexAny(pattern, "*?[") >= 0 362} 363 364// HasGlob returns true if any string in the list contains any glob characters (*, ?, or [). 365func HasGlob(in []string) bool { 366 for _, s := range in { 367 if IsGlob(s) { 368 return true 369 } 370 } 371 372 return false 373} 374 375// GlobWithDepFile finds all files and directories that match glob. Directories 376// will have a trailing '/'. It compares the list of matches against the 377// contents of fileListFile, and rewrites fileListFile if it has changed. It 378// also writes all of the the directories it traversed as dependencies on 379// fileListFile to depFile. 380// 381// The format of glob is either path/*.ext for a single directory glob, or 382// path/**/*.ext for a recursive glob. 383// 384// Returns a list of file paths, and an error. 385// 386// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps 387// should be used instead, as they will automatically set up dependencies 388// to rerun the primary builder when the list of matching files changes. 389func GlobWithDepFile(glob, fileListFile, depFile string, excludes []string) (files []string, err error) { 390 files, deps, err := Glob(glob, excludes, FollowSymlinks) 391 if err != nil { 392 return nil, err 393 } 394 395 fileList := strings.Join(files, "\n") + "\n" 396 397 WriteFileIfChanged(fileListFile, []byte(fileList), 0666) 398 deptools.WriteDepFile(depFile, fileListFile, deps) 399 400 return 401} 402 403// WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if 404// the files does not already exist with identical contents. This can be used 405// along with ninja restat rules to skip rebuilding downstream rules if no 406// changes were made by a rule. 407func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error { 408 var isChanged bool 409 410 dir := filepath.Dir(filename) 411 err := os.MkdirAll(dir, 0777) 412 if err != nil { 413 return err 414 } 415 416 info, err := os.Stat(filename) 417 if err != nil { 418 if os.IsNotExist(err) { 419 // The file does not exist yet. 420 isChanged = true 421 } else { 422 return err 423 } 424 } else { 425 if info.Size() != int64(len(data)) { 426 isChanged = true 427 } else { 428 oldData, err := ioutil.ReadFile(filename) 429 if err != nil { 430 return err 431 } 432 433 if len(oldData) != len(data) { 434 isChanged = true 435 } else { 436 for i := range data { 437 if oldData[i] != data[i] { 438 isChanged = true 439 break 440 } 441 } 442 } 443 } 444 } 445 446 if isChanged { 447 err = ioutil.WriteFile(filename, data, perm) 448 if err != nil { 449 return err 450 } 451 } 452 453 return nil 454} 455 456var matchEscaper = strings.NewReplacer( 457 `*`, `\*`, 458 `?`, `\?`, 459 `[`, `\[`, 460 `]`, `\]`, 461) 462 463// MatchEscape returns its inputs with characters that would be interpreted by 464func MatchEscape(s string) string { 465 return matchEscaper.Replace(s) 466} 467