• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright 2014 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package pathtools
16
17import (
18	"encoding/json"
19	"errors"
20	"fmt"
21	"io/ioutil"
22	"os"
23	"path/filepath"
24	"strings"
25)
26
27var GlobMultipleRecursiveErr = errors.New("pattern contains multiple '**'")
28var GlobLastRecursiveErr = errors.New("pattern has '**' as last path element")
29var GlobInvalidRecursiveErr = errors.New("pattern contains other characters between '**' and path separator")
30
31// GlobResult is a container holding the results of a call to Glob.
32type GlobResult struct {
33	// Pattern is the pattern that was passed to Glob.
34	Pattern string
35	// Excludes is the list of excludes that were passed to Glob.
36	Excludes []string
37
38	// Matches is the list of files or directories that matched the pattern but not the excludes.
39	Matches []string
40
41	// Deps is the list of files or directories that must be depended on to regenerate the glob.
42	Deps []string
43}
44
45// FileList returns the list of files matched by a glob for writing to an output file.
46func (result GlobResult) FileList() []byte {
47	return []byte(strings.Join(result.Matches, "\n") + "\n")
48}
49
50// MultipleGlobResults is a list of GlobResult structs.
51type MultipleGlobResults []GlobResult
52
53// FileList returns the list of files matched by a list of multiple globs for writing to an output file.
54func (results MultipleGlobResults) FileList() []byte {
55	multipleMatches := make([][]string, len(results))
56	for i, result := range results {
57		multipleMatches[i] = result.Matches
58	}
59	buf, err := json.Marshal(multipleMatches)
60	if err != nil {
61		panic(fmt.Errorf("failed to marshal glob results to json: %w", err))
62	}
63	return buf
64}
65
66// Deps returns the deps from all of the GlobResults.
67func (results MultipleGlobResults) Deps() []string {
68	var deps []string
69	for _, result := range results {
70		deps = append(deps, result.Deps...)
71	}
72	return deps
73}
74
75// Glob returns the list of files and directories that match the given pattern
76// but do not match the given exclude patterns, along with the list of
77// directories and other dependencies that were searched to construct the file
78// list.  The supported glob and exclude patterns are equivalent to
79// filepath.Glob, with an extension that recursive glob (** matching zero or
80// more complete path entries) is supported. Any directories in the matches
81// list will have a '/' suffix.
82//
83// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
84// should be used instead, as they will automatically set up dependencies
85// to rerun the primary builder when the list of matching files changes.
86func Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (GlobResult, error) {
87	return startGlob(OsFs, pattern, excludes, follow)
88}
89
90func startGlob(fs FileSystem, pattern string, excludes []string,
91	follow ShouldFollowSymlinks) (GlobResult, error) {
92
93	if filepath.Base(pattern) == "**" {
94		return GlobResult{}, GlobLastRecursiveErr
95	}
96
97	matches, deps, err := glob(fs, pattern, false, follow)
98
99	if err != nil {
100		return GlobResult{}, err
101	}
102
103	matches, err = filterExcludes(matches, excludes)
104	if err != nil {
105		return GlobResult{}, err
106	}
107
108	// If the pattern has wildcards, we added dependencies on the
109	// containing directories to know about changes.
110	//
111	// If the pattern didn't have wildcards, and didn't find matches, the
112	// most specific found directories were added.
113	//
114	// But if it didn't have wildcards, and did find a match, no
115	// dependencies were added, so add the match itself to detect when it
116	// is removed.
117	if !isWild(pattern) {
118		deps = append(deps, matches...)
119	}
120
121	for i, match := range matches {
122		var info os.FileInfo
123		if follow == DontFollowSymlinks {
124			info, err = fs.Lstat(match)
125		} else {
126			info, err = fs.Stat(match)
127			if err != nil && os.IsNotExist(err) {
128				// ErrNotExist from Stat may be due to a dangling symlink, retry with lstat.
129				info, err = fs.Lstat(match)
130			}
131		}
132		if err != nil {
133			return GlobResult{}, err
134		}
135
136		if info.IsDir() {
137			matches[i] = match + "/"
138		}
139	}
140
141	return GlobResult{
142		Pattern:  pattern,
143		Excludes: excludes,
144		Matches:  matches,
145		Deps:     deps,
146	}, nil
147}
148
149// glob is a recursive helper function to handle globbing each level of the pattern individually,
150// allowing searched directories to be tracked.  Also handles the recursive glob pattern, **.
151func glob(fs FileSystem, pattern string, hasRecursive bool,
152	follow ShouldFollowSymlinks) (matches, dirs []string, err error) {
153
154	if !isWild(pattern) {
155		// If there are no wilds in the pattern, check whether the file exists or not.
156		// Uses filepath.Glob instead of manually statting to get consistent results.
157		pattern = filepath.Clean(pattern)
158		matches, err = fs.glob(pattern)
159		if err != nil {
160			return matches, dirs, err
161		}
162
163		if len(matches) == 0 {
164			// Some part of the non-wild pattern didn't exist.  Add the last existing directory
165			// as a dependency.
166			var matchDirs []string
167			for len(matchDirs) == 0 {
168				pattern = filepath.Dir(pattern)
169				matchDirs, err = fs.glob(pattern)
170				if err != nil {
171					return matches, dirs, err
172				}
173			}
174			dirs = append(dirs, matchDirs...)
175		}
176		return matches, dirs, err
177	}
178
179	dir, file := quickSplit(pattern)
180
181	if file == "**" {
182		if hasRecursive {
183			return matches, dirs, GlobMultipleRecursiveErr
184		}
185		hasRecursive = true
186	} else if strings.Contains(file, "**") {
187		return matches, dirs, GlobInvalidRecursiveErr
188	}
189
190	dirMatches, dirs, err := glob(fs, dir, hasRecursive, follow)
191	if err != nil {
192		return nil, nil, err
193	}
194
195	for _, m := range dirMatches {
196		isDir, err := fs.IsDir(m)
197		if os.IsNotExist(err) {
198			if isSymlink, _ := fs.IsSymlink(m); isSymlink {
199				return nil, nil, fmt.Errorf("dangling symlink: %s", m)
200			}
201		}
202		if err != nil {
203			return nil, nil, fmt.Errorf("unexpected error after glob: %s", err)
204		}
205
206		if isDir {
207			if file == "**" {
208				recurseDirs, err := fs.ListDirsRecursive(m, follow)
209				if err != nil {
210					return nil, nil, err
211				}
212				matches = append(matches, recurseDirs...)
213			} else {
214				dirs = append(dirs, m)
215				newMatches, err := fs.glob(filepath.Join(MatchEscape(m), file))
216				if err != nil {
217					return nil, nil, err
218				}
219				if file[0] != '.' {
220					newMatches = filterDotFiles(newMatches)
221				}
222				matches = append(matches, newMatches...)
223			}
224		}
225	}
226
227	return matches, dirs, nil
228}
229
230// Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations
231// Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is
232// not "/".  Returns ".", "" if path is "."
233func quickSplit(path string) (dir, file string) {
234	if path == "." {
235		return ".", ""
236	}
237	dir, file = filepath.Split(path)
238	switch dir {
239	case "":
240		dir = "."
241	case "/":
242		// Nothing
243	default:
244		dir = dir[:len(dir)-1]
245	}
246	return dir, file
247}
248
249func isWild(pattern string) bool {
250	return strings.ContainsAny(pattern, "*?[")
251}
252
253// Filters the strings in matches based on the glob patterns in excludes.  Hierarchical (a/*) and
254// recursive (**) glob patterns are supported.
255func filterExcludes(matches []string, excludes []string) ([]string, error) {
256	if len(excludes) == 0 {
257		return matches, nil
258	}
259
260	var ret []string
261matchLoop:
262	for _, m := range matches {
263		for _, e := range excludes {
264			exclude, err := Match(e, m)
265			if err != nil {
266				return nil, err
267			}
268			if exclude {
269				continue matchLoop
270			}
271		}
272		ret = append(ret, m)
273	}
274
275	return ret, nil
276}
277
278// filterDotFiles filters out files that start with '.'
279func filterDotFiles(matches []string) []string {
280	ret := make([]string, 0, len(matches))
281
282	for _, match := range matches {
283		_, name := filepath.Split(match)
284		if name[0] == '.' {
285			continue
286		}
287		ret = append(ret, match)
288	}
289
290	return ret
291}
292
293// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting
294// recursive globs (**).
295func Match(pattern, name string) (bool, error) {
296	if filepath.Base(pattern) == "**" {
297		return false, GlobLastRecursiveErr
298	}
299
300	patternDir := pattern[len(pattern)-1] == '/'
301	nameDir := name[len(name)-1] == '/'
302
303	if patternDir != nameDir {
304		return false, nil
305	}
306
307	if nameDir {
308		name = name[:len(name)-1]
309		pattern = pattern[:len(pattern)-1]
310	}
311
312	for {
313		var patternFile, nameFile string
314		pattern, patternFile = filepath.Dir(pattern), filepath.Base(pattern)
315
316		if patternFile == "**" {
317			if strings.Contains(pattern, "**") {
318				return false, GlobMultipleRecursiveErr
319			}
320			// Test if the any prefix of name matches the part of the pattern before **
321			for {
322				if name == "." || name == "/" {
323					return name == pattern, nil
324				}
325				if match, err := filepath.Match(pattern, name); err != nil {
326					return false, err
327				} else if match {
328					return true, nil
329				}
330				name = filepath.Dir(name)
331			}
332		} else if strings.Contains(patternFile, "**") {
333			return false, GlobInvalidRecursiveErr
334		}
335
336		name, nameFile = filepath.Dir(name), filepath.Base(name)
337
338		if nameFile == "." && patternFile == "." {
339			return true, nil
340		} else if nameFile == "/" && patternFile == "/" {
341			return true, nil
342		} else if nameFile == "." || patternFile == "." || nameFile == "/" || patternFile == "/" {
343			return false, nil
344		}
345
346		match, err := filepath.Match(patternFile, nameFile)
347		if err != nil || !match {
348			return match, err
349		}
350	}
351}
352
353// IsGlob returns true if the pattern contains any glob characters (*, ?, or [).
354func IsGlob(pattern string) bool {
355	return strings.IndexAny(pattern, "*?[") >= 0
356}
357
358// HasGlob returns true if any string in the list contains any glob characters (*, ?, or [).
359func HasGlob(in []string) bool {
360	for _, s := range in {
361		if IsGlob(s) {
362			return true
363		}
364	}
365
366	return false
367}
368
369// WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if
370// the files does not already exist with identical contents.  This can be used
371// along with ninja restat rules to skip rebuilding downstream rules if no
372// changes were made by a rule.
373func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error {
374	var isChanged bool
375
376	dir := filepath.Dir(filename)
377	err := os.MkdirAll(dir, 0777)
378	if err != nil {
379		return err
380	}
381
382	info, err := os.Stat(filename)
383	if err != nil {
384		if os.IsNotExist(err) {
385			// The file does not exist yet.
386			isChanged = true
387		} else {
388			return err
389		}
390	} else {
391		if info.Size() != int64(len(data)) {
392			isChanged = true
393		} else {
394			oldData, err := ioutil.ReadFile(filename)
395			if err != nil {
396				return err
397			}
398
399			if len(oldData) != len(data) {
400				isChanged = true
401			} else {
402				for i := range data {
403					if oldData[i] != data[i] {
404						isChanged = true
405						break
406					}
407				}
408			}
409		}
410	}
411
412	if isChanged {
413		err = ioutil.WriteFile(filename, data, perm)
414		if err != nil {
415			return err
416		}
417	}
418
419	return nil
420}
421
422var matchEscaper = strings.NewReplacer(
423	`*`, `\*`,
424	`?`, `\?`,
425	`[`, `\[`,
426	`]`, `\]`,
427)
428
429// MatchEscape returns its inputs with characters that would be interpreted by
430func MatchEscape(s string) string {
431	return matchEscaper.Replace(s)
432}
433