• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright 2014 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package pathtools
16
17import (
18	"errors"
19	"fmt"
20	"io/ioutil"
21	"os"
22	"path/filepath"
23	"strings"
24
25	"github.com/google/blueprint/deptools"
26)
27
28var GlobMultipleRecursiveErr = errors.New("pattern contains multiple **")
29var GlobLastRecursiveErr = errors.New("pattern ** as last path element")
30
31// Glob returns the list of files and directories that match the given pattern
32// but do not match the given exclude patterns, along with the list of
33// directories and other dependencies that were searched to construct the file
34// list.  The supported glob and exclude patterns are equivalent to
35// filepath.Glob, with an extension that recursive glob (** matching zero or
36// more complete path entries) is supported. Any directories in the matches
37// list will have a '/' suffix.
38//
39// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
40// should be used instead, as they will automatically set up dependencies
41// to rerun the primary builder when the list of matching files changes.
42func Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (matches, deps []string, err error) {
43	return startGlob(OsFs, pattern, excludes, follow)
44}
45
46func startGlob(fs FileSystem, pattern string, excludes []string,
47	follow ShouldFollowSymlinks) (matches, deps []string, err error) {
48
49	if filepath.Base(pattern) == "**" {
50		return nil, nil, GlobLastRecursiveErr
51	} else {
52		matches, deps, err = glob(fs, pattern, false, follow)
53	}
54
55	if err != nil {
56		return nil, nil, err
57	}
58
59	matches, err = filterExcludes(matches, excludes)
60	if err != nil {
61		return nil, nil, err
62	}
63
64	// If the pattern has wildcards, we added dependencies on the
65	// containing directories to know about changes.
66	//
67	// If the pattern didn't have wildcards, and didn't find matches, the
68	// most specific found directories were added.
69	//
70	// But if it didn't have wildcards, and did find a match, no
71	// dependencies were added, so add the match itself to detect when it
72	// is removed.
73	if !isWild(pattern) {
74		deps = append(deps, matches...)
75	}
76
77	for i, match := range matches {
78		isSymlink, err := fs.IsSymlink(match)
79		if err != nil {
80			return nil, nil, err
81		}
82		if !(isSymlink && follow == DontFollowSymlinks) {
83			isDir, err := fs.IsDir(match)
84			if os.IsNotExist(err) {
85				if isSymlink {
86					return nil, nil, fmt.Errorf("%s: dangling symlink", match)
87				}
88			}
89			if err != nil {
90				return nil, nil, fmt.Errorf("%s: %s", match, err.Error())
91			}
92
93			if isDir {
94				matches[i] = match + "/"
95			}
96		}
97	}
98
99	return matches, deps, nil
100}
101
102// glob is a recursive helper function to handle globbing each level of the pattern individually,
103// allowing searched directories to be tracked.  Also handles the recursive glob pattern, **.
104func glob(fs FileSystem, pattern string, hasRecursive bool,
105	follow ShouldFollowSymlinks) (matches, dirs []string, err error) {
106
107	if !isWild(pattern) {
108		// If there are no wilds in the pattern, check whether the file exists or not.
109		// Uses filepath.Glob instead of manually statting to get consistent results.
110		pattern = filepath.Clean(pattern)
111		matches, err = fs.glob(pattern)
112		if err != nil {
113			return matches, dirs, err
114		}
115
116		if len(matches) == 0 {
117			// Some part of the non-wild pattern didn't exist.  Add the last existing directory
118			// as a dependency.
119			var matchDirs []string
120			for len(matchDirs) == 0 {
121				pattern, _ = saneSplit(pattern)
122				matchDirs, err = fs.glob(pattern)
123				if err != nil {
124					return matches, dirs, err
125				}
126			}
127			dirs = append(dirs, matchDirs...)
128		}
129		return matches, dirs, err
130	}
131
132	dir, file := saneSplit(pattern)
133
134	if file == "**" {
135		if hasRecursive {
136			return matches, dirs, GlobMultipleRecursiveErr
137		}
138		hasRecursive = true
139	}
140
141	dirMatches, dirs, err := glob(fs, dir, hasRecursive, follow)
142	if err != nil {
143		return nil, nil, err
144	}
145
146	for _, m := range dirMatches {
147		isDir, err := fs.IsDir(m)
148		if os.IsNotExist(err) {
149			if isSymlink, _ := fs.IsSymlink(m); isSymlink {
150				return nil, nil, fmt.Errorf("dangling symlink: %s", m)
151			}
152		}
153		if err != nil {
154			return nil, nil, fmt.Errorf("unexpected error after glob: %s", err)
155		}
156
157		if isDir {
158			if file == "**" {
159				recurseDirs, err := fs.ListDirsRecursive(m, follow)
160				if err != nil {
161					return nil, nil, err
162				}
163				matches = append(matches, recurseDirs...)
164			} else {
165				dirs = append(dirs, m)
166				newMatches, err := fs.glob(filepath.Join(MatchEscape(m), file))
167				if err != nil {
168					return nil, nil, err
169				}
170				if file[0] != '.' {
171					newMatches = filterDotFiles(newMatches)
172				}
173				matches = append(matches, newMatches...)
174			}
175		}
176	}
177
178	return matches, dirs, nil
179}
180
181// Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations
182// Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is
183// not "/".  Returns ".", "" if path is "."
184func saneSplit(path string) (dir, file string) {
185	if path == "." {
186		return ".", ""
187	}
188	dir, file = filepath.Split(path)
189	switch dir {
190	case "":
191		dir = "."
192	case "/":
193		// Nothing
194	default:
195		dir = dir[:len(dir)-1]
196	}
197	return dir, file
198}
199
200func isWild(pattern string) bool {
201	return strings.ContainsAny(pattern, "*?[")
202}
203
204// Filters the strings in matches based on the glob patterns in excludes.  Hierarchical (a/*) and
205// recursive (**) glob patterns are supported.
206func filterExcludes(matches []string, excludes []string) ([]string, error) {
207	if len(excludes) == 0 {
208		return matches, nil
209	}
210
211	var ret []string
212matchLoop:
213	for _, m := range matches {
214		for _, e := range excludes {
215			exclude, err := Match(e, m)
216			if err != nil {
217				return nil, err
218			}
219			if exclude {
220				continue matchLoop
221			}
222		}
223		ret = append(ret, m)
224	}
225
226	return ret, nil
227}
228
229// filterDotFiles filters out files that start with '.'
230func filterDotFiles(matches []string) []string {
231	ret := make([]string, 0, len(matches))
232
233	for _, match := range matches {
234		_, name := filepath.Split(match)
235		if name[0] == '.' {
236			continue
237		}
238		ret = append(ret, match)
239	}
240
241	return ret
242}
243
244// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting
245// hierarchical patterns (a/*) and recursive globs (**).
246func Match(pattern, name string) (bool, error) {
247	if filepath.Base(pattern) == "**" {
248		return false, GlobLastRecursiveErr
249	}
250
251	patternDir := pattern[len(pattern)-1] == '/'
252	nameDir := name[len(name)-1] == '/'
253
254	if patternDir != nameDir {
255		return false, nil
256	}
257
258	if nameDir {
259		name = name[:len(name)-1]
260		pattern = pattern[:len(pattern)-1]
261	}
262
263	for {
264		var patternFile, nameFile string
265		pattern, patternFile = saneSplit(pattern)
266		name, nameFile = saneSplit(name)
267
268		if patternFile == "**" {
269			return matchPrefix(pattern, filepath.Join(name, nameFile))
270		}
271
272		if nameFile == "" && patternFile == "" {
273			return true, nil
274		} else if nameFile == "" || patternFile == "" {
275			return false, nil
276		}
277
278		match, err := filepath.Match(patternFile, nameFile)
279		if err != nil || !match {
280			return match, err
281		}
282	}
283}
284
285// matchPrefix returns true if the beginning of name matches pattern using the same rules as
286// filepath.Match, but supporting hierarchical patterns (a/*).  Recursive globs (**) are not
287// supported, they should have been handled in Match().
288func matchPrefix(pattern, name string) (bool, error) {
289	if len(pattern) > 0 && pattern[0] == '/' {
290		if len(name) > 0 && name[0] == '/' {
291			pattern = pattern[1:]
292			name = name[1:]
293		} else {
294			return false, nil
295		}
296	}
297
298	for {
299		var patternElem, nameElem string
300		patternElem, pattern = saneSplitFirst(pattern)
301		nameElem, name = saneSplitFirst(name)
302
303		if patternElem == "." {
304			patternElem = ""
305		}
306		if nameElem == "." {
307			nameElem = ""
308		}
309
310		if patternElem == "**" {
311			return false, GlobMultipleRecursiveErr
312		}
313
314		if patternElem == "" {
315			return true, nil
316		} else if nameElem == "" {
317			return false, nil
318		}
319
320		match, err := filepath.Match(patternElem, nameElem)
321		if err != nil || !match {
322			return match, err
323		}
324	}
325}
326
327func saneSplitFirst(path string) (string, string) {
328	i := strings.IndexRune(path, filepath.Separator)
329	if i < 0 {
330		return path, ""
331	}
332	return path[:i], path[i+1:]
333}
334
335func GlobPatternList(patterns []string, prefix string) (globedList []string, depDirs []string, err error) {
336	var (
337		matches []string
338		deps    []string
339	)
340
341	globedList = make([]string, 0)
342	depDirs = make([]string, 0)
343
344	for _, pattern := range patterns {
345		if isWild(pattern) {
346			matches, deps, err = Glob(filepath.Join(prefix, pattern), nil, FollowSymlinks)
347			if err != nil {
348				return nil, nil, err
349			}
350			globedList = append(globedList, matches...)
351			depDirs = append(depDirs, deps...)
352		} else {
353			globedList = append(globedList, filepath.Join(prefix, pattern))
354		}
355	}
356	return globedList, depDirs, nil
357}
358
359// IsGlob returns true if the pattern contains any glob characters (*, ?, or [).
360func IsGlob(pattern string) bool {
361	return strings.IndexAny(pattern, "*?[") >= 0
362}
363
364// HasGlob returns true if any string in the list contains any glob characters (*, ?, or [).
365func HasGlob(in []string) bool {
366	for _, s := range in {
367		if IsGlob(s) {
368			return true
369		}
370	}
371
372	return false
373}
374
375// GlobWithDepFile finds all files and directories that match glob.  Directories
376// will have a trailing '/'.  It compares the list of matches against the
377// contents of fileListFile, and rewrites fileListFile if it has changed.  It
378// also writes all of the the directories it traversed as dependencies on
379// fileListFile to depFile.
380//
381// The format of glob is either path/*.ext for a single directory glob, or
382// path/**/*.ext for a recursive glob.
383//
384// Returns a list of file paths, and an error.
385//
386// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
387// should be used instead, as they will automatically set up dependencies
388// to rerun the primary builder when the list of matching files changes.
389func GlobWithDepFile(glob, fileListFile, depFile string, excludes []string) (files []string, err error) {
390	files, deps, err := Glob(glob, excludes, FollowSymlinks)
391	if err != nil {
392		return nil, err
393	}
394
395	fileList := strings.Join(files, "\n") + "\n"
396
397	WriteFileIfChanged(fileListFile, []byte(fileList), 0666)
398	deptools.WriteDepFile(depFile, fileListFile, deps)
399
400	return
401}
402
403// WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if
404// the files does not already exist with identical contents.  This can be used
405// along with ninja restat rules to skip rebuilding downstream rules if no
406// changes were made by a rule.
407func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error {
408	var isChanged bool
409
410	dir := filepath.Dir(filename)
411	err := os.MkdirAll(dir, 0777)
412	if err != nil {
413		return err
414	}
415
416	info, err := os.Stat(filename)
417	if err != nil {
418		if os.IsNotExist(err) {
419			// The file does not exist yet.
420			isChanged = true
421		} else {
422			return err
423		}
424	} else {
425		if info.Size() != int64(len(data)) {
426			isChanged = true
427		} else {
428			oldData, err := ioutil.ReadFile(filename)
429			if err != nil {
430				return err
431			}
432
433			if len(oldData) != len(data) {
434				isChanged = true
435			} else {
436				for i := range data {
437					if oldData[i] != data[i] {
438						isChanged = true
439						break
440					}
441				}
442			}
443		}
444	}
445
446	if isChanged {
447		err = ioutil.WriteFile(filename, data, perm)
448		if err != nil {
449			return err
450		}
451	}
452
453	return nil
454}
455
456var matchEscaper = strings.NewReplacer(
457	`*`, `\*`,
458	`?`, `\?`,
459	`[`, `\[`,
460	`]`, `\]`,
461)
462
463// MatchEscape returns its inputs with characters that would be interpreted by
464func MatchEscape(s string) string {
465	return matchEscaper.Replace(s)
466}
467