1// Copyright 2014 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package pathtools 16 17import ( 18 "encoding/json" 19 "errors" 20 "fmt" 21 "io/ioutil" 22 "os" 23 "path/filepath" 24 "strings" 25) 26 27var GlobMultipleRecursiveErr = errors.New("pattern contains multiple '**'") 28var GlobLastRecursiveErr = errors.New("pattern has '**' as last path element") 29var GlobInvalidRecursiveErr = errors.New("pattern contains other characters between '**' and path separator") 30 31// GlobResult is a container holding the results of a call to Glob. 32type GlobResult struct { 33 // Pattern is the pattern that was passed to Glob. 34 Pattern string 35 // Excludes is the list of excludes that were passed to Glob. 36 Excludes []string 37 38 // Matches is the list of files or directories that matched the pattern but not the excludes. 39 Matches []string 40 41 // Deps is the list of files or directories that must be depended on to regenerate the glob. 42 Deps []string 43} 44 45// FileList returns the list of files matched by a glob for writing to an output file. 46func (result GlobResult) FileList() []byte { 47 return []byte(strings.Join(result.Matches, "\n") + "\n") 48} 49 50// MultipleGlobResults is a list of GlobResult structs. 51type MultipleGlobResults []GlobResult 52 53// FileList returns the list of files matched by a list of multiple globs for writing to an output file. 54func (results MultipleGlobResults) FileList() []byte { 55 multipleMatches := make([][]string, len(results)) 56 for i, result := range results { 57 multipleMatches[i] = result.Matches 58 } 59 buf, err := json.Marshal(multipleMatches) 60 if err != nil { 61 panic(fmt.Errorf("failed to marshal glob results to json: %w", err)) 62 } 63 return buf 64} 65 66// Deps returns the deps from all of the GlobResults. 67func (results MultipleGlobResults) Deps() []string { 68 var deps []string 69 for _, result := range results { 70 deps = append(deps, result.Deps...) 71 } 72 return deps 73} 74 75// Glob returns the list of files and directories that match the given pattern 76// but do not match the given exclude patterns, along with the list of 77// directories and other dependencies that were searched to construct the file 78// list. The supported glob and exclude patterns are equivalent to 79// filepath.Glob, with an extension that recursive glob (** matching zero or 80// more complete path entries) is supported. Any directories in the matches 81// list will have a '/' suffix. 82// 83// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps 84// should be used instead, as they will automatically set up dependencies 85// to rerun the primary builder when the list of matching files changes. 86func Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (GlobResult, error) { 87 return startGlob(OsFs, pattern, excludes, follow) 88} 89 90func startGlob(fs FileSystem, pattern string, excludes []string, 91 follow ShouldFollowSymlinks) (GlobResult, error) { 92 93 if filepath.Base(pattern) == "**" { 94 return GlobResult{}, GlobLastRecursiveErr 95 } 96 97 matches, deps, err := glob(fs, pattern, false, follow) 98 99 if err != nil { 100 return GlobResult{}, err 101 } 102 103 matches, err = filterExcludes(matches, excludes) 104 if err != nil { 105 return GlobResult{}, err 106 } 107 108 // If the pattern has wildcards, we added dependencies on the 109 // containing directories to know about changes. 110 // 111 // If the pattern didn't have wildcards, and didn't find matches, the 112 // most specific found directories were added. 113 // 114 // But if it didn't have wildcards, and did find a match, no 115 // dependencies were added, so add the match itself to detect when it 116 // is removed. 117 if !isWild(pattern) { 118 deps = append(deps, matches...) 119 } 120 121 for i, match := range matches { 122 var info os.FileInfo 123 if follow == DontFollowSymlinks { 124 info, err = fs.Lstat(match) 125 } else { 126 info, err = fs.Stat(match) 127 if err != nil && os.IsNotExist(err) { 128 // ErrNotExist from Stat may be due to a dangling symlink, retry with lstat. 129 info, err = fs.Lstat(match) 130 } 131 } 132 if err != nil { 133 return GlobResult{}, err 134 } 135 136 if info.IsDir() { 137 matches[i] = match + "/" 138 } 139 } 140 141 return GlobResult{ 142 Pattern: pattern, 143 Excludes: excludes, 144 Matches: matches, 145 Deps: deps, 146 }, nil 147} 148 149// glob is a recursive helper function to handle globbing each level of the pattern individually, 150// allowing searched directories to be tracked. Also handles the recursive glob pattern, **. 151func glob(fs FileSystem, pattern string, hasRecursive bool, 152 follow ShouldFollowSymlinks) (matches, dirs []string, err error) { 153 154 if !isWild(pattern) { 155 // If there are no wilds in the pattern, check whether the file exists or not. 156 // Uses filepath.Glob instead of manually statting to get consistent results. 157 pattern = filepath.Clean(pattern) 158 matches, err = fs.glob(pattern) 159 if err != nil { 160 return matches, dirs, err 161 } 162 163 if len(matches) == 0 { 164 // Some part of the non-wild pattern didn't exist. Add the last existing directory 165 // as a dependency. 166 var matchDirs []string 167 for len(matchDirs) == 0 { 168 pattern = filepath.Dir(pattern) 169 matchDirs, err = fs.glob(pattern) 170 if err != nil { 171 return matches, dirs, err 172 } 173 } 174 dirs = append(dirs, matchDirs...) 175 } 176 return matches, dirs, err 177 } 178 179 dir, file := quickSplit(pattern) 180 181 if file == "**" { 182 if hasRecursive { 183 return matches, dirs, GlobMultipleRecursiveErr 184 } 185 hasRecursive = true 186 } else if strings.Contains(file, "**") { 187 return matches, dirs, GlobInvalidRecursiveErr 188 } 189 190 dirMatches, dirs, err := glob(fs, dir, hasRecursive, follow) 191 if err != nil { 192 return nil, nil, err 193 } 194 195 for _, m := range dirMatches { 196 isDir, err := fs.IsDir(m) 197 if os.IsNotExist(err) { 198 if isSymlink, _ := fs.IsSymlink(m); isSymlink { 199 return nil, nil, fmt.Errorf("dangling symlink: %s", m) 200 } 201 } 202 if err != nil { 203 return nil, nil, fmt.Errorf("unexpected error after glob: %s", err) 204 } 205 206 if isDir { 207 if file == "**" { 208 recurseDirs, err := fs.ListDirsRecursive(m, follow) 209 if err != nil { 210 return nil, nil, err 211 } 212 matches = append(matches, recurseDirs...) 213 } else { 214 dirs = append(dirs, m) 215 newMatches, err := fs.glob(filepath.Join(MatchEscape(m), file)) 216 if err != nil { 217 return nil, nil, err 218 } 219 if file[0] != '.' { 220 newMatches = filterDotFiles(newMatches) 221 } 222 matches = append(matches, newMatches...) 223 } 224 } 225 } 226 227 return matches, dirs, nil 228} 229 230// Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations 231// Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is 232// not "/". Returns ".", "" if path is "." 233func quickSplit(path string) (dir, file string) { 234 if path == "." { 235 return ".", "" 236 } 237 dir, file = filepath.Split(path) 238 switch dir { 239 case "": 240 dir = "." 241 case "/": 242 // Nothing 243 default: 244 dir = dir[:len(dir)-1] 245 } 246 return dir, file 247} 248 249func isWild(pattern string) bool { 250 return strings.ContainsAny(pattern, "*?[") 251} 252 253// Filters the strings in matches based on the glob patterns in excludes. Hierarchical (a/*) and 254// recursive (**) glob patterns are supported. 255func filterExcludes(matches []string, excludes []string) ([]string, error) { 256 if len(excludes) == 0 { 257 return matches, nil 258 } 259 260 var ret []string 261matchLoop: 262 for _, m := range matches { 263 for _, e := range excludes { 264 exclude, err := Match(e, m) 265 if err != nil { 266 return nil, err 267 } 268 if exclude { 269 continue matchLoop 270 } 271 } 272 ret = append(ret, m) 273 } 274 275 return ret, nil 276} 277 278// filterDotFiles filters out files that start with '.' 279func filterDotFiles(matches []string) []string { 280 ret := make([]string, 0, len(matches)) 281 282 for _, match := range matches { 283 _, name := filepath.Split(match) 284 if name[0] == '.' { 285 continue 286 } 287 ret = append(ret, match) 288 } 289 290 return ret 291} 292 293// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting 294// recursive globs (**). 295func Match(pattern, name string) (bool, error) { 296 if filepath.Base(pattern) == "**" { 297 return false, GlobLastRecursiveErr 298 } 299 300 patternDir := pattern[len(pattern)-1] == '/' 301 nameDir := name[len(name)-1] == '/' 302 303 if patternDir != nameDir { 304 return false, nil 305 } 306 307 if nameDir { 308 name = name[:len(name)-1] 309 pattern = pattern[:len(pattern)-1] 310 } 311 312 for { 313 var patternFile, nameFile string 314 pattern, patternFile = filepath.Dir(pattern), filepath.Base(pattern) 315 316 if patternFile == "**" { 317 if strings.Contains(pattern, "**") { 318 return false, GlobMultipleRecursiveErr 319 } 320 // Test if the any prefix of name matches the part of the pattern before ** 321 for { 322 if name == "." || name == "/" { 323 return name == pattern, nil 324 } 325 if match, err := filepath.Match(pattern, name); err != nil { 326 return false, err 327 } else if match { 328 return true, nil 329 } 330 name = filepath.Dir(name) 331 } 332 } else if strings.Contains(patternFile, "**") { 333 return false, GlobInvalidRecursiveErr 334 } 335 336 name, nameFile = filepath.Dir(name), filepath.Base(name) 337 338 if nameFile == "." && patternFile == "." { 339 return true, nil 340 } else if nameFile == "/" && patternFile == "/" { 341 return true, nil 342 } else if nameFile == "." || patternFile == "." || nameFile == "/" || patternFile == "/" { 343 return false, nil 344 } 345 346 match, err := filepath.Match(patternFile, nameFile) 347 if err != nil || !match { 348 return match, err 349 } 350 } 351} 352 353// IsGlob returns true if the pattern contains any glob characters (*, ?, or [). 354func IsGlob(pattern string) bool { 355 return strings.IndexAny(pattern, "*?[") >= 0 356} 357 358// HasGlob returns true if any string in the list contains any glob characters (*, ?, or [). 359func HasGlob(in []string) bool { 360 for _, s := range in { 361 if IsGlob(s) { 362 return true 363 } 364 } 365 366 return false 367} 368 369// WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if 370// the files does not already exist with identical contents. This can be used 371// along with ninja restat rules to skip rebuilding downstream rules if no 372// changes were made by a rule. 373func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error { 374 var isChanged bool 375 376 dir := filepath.Dir(filename) 377 err := os.MkdirAll(dir, 0777) 378 if err != nil { 379 return err 380 } 381 382 info, err := os.Stat(filename) 383 if err != nil { 384 if os.IsNotExist(err) { 385 // The file does not exist yet. 386 isChanged = true 387 } else { 388 return err 389 } 390 } else { 391 if info.Size() != int64(len(data)) { 392 isChanged = true 393 } else { 394 oldData, err := ioutil.ReadFile(filename) 395 if err != nil { 396 return err 397 } 398 399 if len(oldData) != len(data) { 400 isChanged = true 401 } else { 402 for i := range data { 403 if oldData[i] != data[i] { 404 isChanged = true 405 break 406 } 407 } 408 } 409 } 410 } 411 412 if isChanged { 413 err = ioutil.WriteFile(filename, data, perm) 414 if err != nil { 415 return err 416 } 417 } 418 419 return nil 420} 421 422var matchEscaper = strings.NewReplacer( 423 `*`, `\*`, 424 `?`, `\?`, 425 `[`, `\[`, 426 `]`, `\]`, 427) 428 429// MatchEscape returns its inputs with characters that would be interpreted by 430func MatchEscape(s string) string { 431 return matchEscaper.Replace(s) 432} 433