• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright 2014 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package pathtools
16
17import (
18	"encoding/json"
19	"errors"
20	"fmt"
21	"io/ioutil"
22	"os"
23	"path/filepath"
24	"strings"
25)
26
27// BPGlobArgumentVersion is used to abort argument parsing early when the bpglob argument format
28// has changed but soong_build hasn't had a chance to rerun yet to update build-globs.ninja.
29// Increment it manually when changing the bpglob argument format.  It is located here because
30// pathtools is the only package that is shared between bpglob and bootstrap.
31const BPGlobArgumentVersion = 2
32
33var GlobMultipleRecursiveErr = errors.New("pattern contains multiple '**'")
34var GlobLastRecursiveErr = errors.New("pattern has '**' as last path element")
35var GlobInvalidRecursiveErr = errors.New("pattern contains other characters between '**' and path separator")
36
37// GlobResult is a container holding the results of a call to Glob.
38type GlobResult struct {
39	// Pattern is the pattern that was passed to Glob.
40	Pattern string
41	// Excludes is the list of excludes that were passed to Glob.
42	Excludes []string
43
44	// Matches is the list of files or directories that matched the pattern but not the excludes.
45	Matches []string
46
47	// Deps is the list of files or directories that must be depended on to regenerate the glob.
48	Deps []string
49}
50
51// FileList returns the list of files matched by a glob for writing to an output file.
52func (result GlobResult) FileList() []byte {
53	return []byte(strings.Join(result.Matches, "\n") + "\n")
54}
55
56// MultipleGlobResults is a list of GlobResult structs.
57type MultipleGlobResults []GlobResult
58
59// FileList returns the list of files matched by a list of multiple globs for writing to an output file.
60func (results MultipleGlobResults) FileList() []byte {
61	multipleMatches := make([][]string, len(results))
62	for i, result := range results {
63		multipleMatches[i] = result.Matches
64	}
65	buf, err := json.Marshal(multipleMatches)
66	if err != nil {
67		panic(fmt.Errorf("failed to marshal glob results to json: %w", err))
68	}
69	return buf
70}
71
72// Deps returns the deps from all of the GlobResults.
73func (results MultipleGlobResults) Deps() []string {
74	var deps []string
75	for _, result := range results {
76		deps = append(deps, result.Deps...)
77	}
78	return deps
79}
80
81// Glob returns the list of files and directories that match the given pattern
82// but do not match the given exclude patterns, along with the list of
83// directories and other dependencies that were searched to construct the file
84// list.  The supported glob and exclude patterns are equivalent to
85// filepath.Glob, with an extension that recursive glob (** matching zero or
86// more complete path entries) is supported. Any directories in the matches
87// list will have a '/' suffix.
88//
89// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
90// should be used instead, as they will automatically set up dependencies
91// to rerun the primary builder when the list of matching files changes.
92func Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (GlobResult, error) {
93	return startGlob(OsFs, pattern, excludes, follow)
94}
95
96func startGlob(fs FileSystem, pattern string, excludes []string,
97	follow ShouldFollowSymlinks) (GlobResult, error) {
98
99	if filepath.Base(pattern) == "**" {
100		return GlobResult{}, GlobLastRecursiveErr
101	}
102
103	matches, deps, err := glob(fs, pattern, false, follow)
104
105	if err != nil {
106		return GlobResult{}, err
107	}
108
109	matches, err = filterExcludes(matches, excludes)
110	if err != nil {
111		return GlobResult{}, err
112	}
113
114	// If the pattern has wildcards, we added dependencies on the
115	// containing directories to know about changes.
116	//
117	// If the pattern didn't have wildcards, and didn't find matches, the
118	// most specific found directories were added.
119	//
120	// But if it didn't have wildcards, and did find a match, no
121	// dependencies were added, so add the match itself to detect when it
122	// is removed.
123	if !isWild(pattern) {
124		deps = append(deps, matches...)
125	}
126
127	for i, match := range matches {
128		var info os.FileInfo
129		if follow == DontFollowSymlinks {
130			info, err = fs.Lstat(match)
131		} else {
132			info, err = fs.Stat(match)
133		}
134		if err != nil {
135			return GlobResult{}, err
136		}
137
138		if info.IsDir() {
139			matches[i] = match + "/"
140		}
141	}
142
143	return GlobResult{
144		Pattern:  pattern,
145		Excludes: excludes,
146		Matches:  matches,
147		Deps:     deps,
148	}, nil
149}
150
151// glob is a recursive helper function to handle globbing each level of the pattern individually,
152// allowing searched directories to be tracked.  Also handles the recursive glob pattern, **.
153func glob(fs FileSystem, pattern string, hasRecursive bool,
154	follow ShouldFollowSymlinks) (matches, dirs []string, err error) {
155
156	if !isWild(pattern) {
157		// If there are no wilds in the pattern, check whether the file exists or not.
158		// Uses filepath.Glob instead of manually statting to get consistent results.
159		pattern = filepath.Clean(pattern)
160		matches, err = fs.glob(pattern)
161		if err != nil {
162			return matches, dirs, err
163		}
164
165		if len(matches) == 0 {
166			// Some part of the non-wild pattern didn't exist.  Add the last existing directory
167			// as a dependency.
168			var matchDirs []string
169			for len(matchDirs) == 0 {
170				pattern = filepath.Dir(pattern)
171				matchDirs, err = fs.glob(pattern)
172				if err != nil {
173					return matches, dirs, err
174				}
175			}
176			dirs = append(dirs, matchDirs...)
177		}
178		return matches, dirs, err
179	}
180
181	dir, file := saneSplit(pattern)
182
183	if file == "**" {
184		if hasRecursive {
185			return matches, dirs, GlobMultipleRecursiveErr
186		}
187		hasRecursive = true
188	} else if strings.Contains(file, "**") {
189		return matches, dirs, GlobInvalidRecursiveErr
190	}
191
192	dirMatches, dirs, err := glob(fs, dir, hasRecursive, follow)
193	if err != nil {
194		return nil, nil, err
195	}
196
197	for _, m := range dirMatches {
198		isDir, err := fs.IsDir(m)
199		if os.IsNotExist(err) {
200			if isSymlink, _ := fs.IsSymlink(m); isSymlink {
201				return nil, nil, fmt.Errorf("dangling symlink: %s", m)
202			}
203		}
204		if err != nil {
205			return nil, nil, fmt.Errorf("unexpected error after glob: %s", err)
206		}
207
208		if isDir {
209			if file == "**" {
210				recurseDirs, err := fs.ListDirsRecursive(m, follow)
211				if err != nil {
212					return nil, nil, err
213				}
214				matches = append(matches, recurseDirs...)
215			} else {
216				dirs = append(dirs, m)
217				newMatches, err := fs.glob(filepath.Join(MatchEscape(m), file))
218				if err != nil {
219					return nil, nil, err
220				}
221				if file[0] != '.' {
222					newMatches = filterDotFiles(newMatches)
223				}
224				matches = append(matches, newMatches...)
225			}
226		}
227	}
228
229	return matches, dirs, nil
230}
231
232// Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations
233// Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is
234// not "/".  Returns ".", "" if path is "."
235func saneSplit(path string) (dir, file string) {
236	if path == "." {
237		return ".", ""
238	}
239	dir, file = filepath.Split(path)
240	switch dir {
241	case "":
242		dir = "."
243	case "/":
244		// Nothing
245	default:
246		dir = dir[:len(dir)-1]
247	}
248	return dir, file
249}
250
251func isWild(pattern string) bool {
252	return strings.ContainsAny(pattern, "*?[")
253}
254
255// Filters the strings in matches based on the glob patterns in excludes.  Hierarchical (a/*) and
256// recursive (**) glob patterns are supported.
257func filterExcludes(matches []string, excludes []string) ([]string, error) {
258	if len(excludes) == 0 {
259		return matches, nil
260	}
261
262	var ret []string
263matchLoop:
264	for _, m := range matches {
265		for _, e := range excludes {
266			exclude, err := Match(e, m)
267			if err != nil {
268				return nil, err
269			}
270			if exclude {
271				continue matchLoop
272			}
273		}
274		ret = append(ret, m)
275	}
276
277	return ret, nil
278}
279
280// filterDotFiles filters out files that start with '.'
281func filterDotFiles(matches []string) []string {
282	ret := make([]string, 0, len(matches))
283
284	for _, match := range matches {
285		_, name := filepath.Split(match)
286		if name[0] == '.' {
287			continue
288		}
289		ret = append(ret, match)
290	}
291
292	return ret
293}
294
295// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting
296// recursive globs (**).
297func Match(pattern, name string) (bool, error) {
298	if filepath.Base(pattern) == "**" {
299		return false, GlobLastRecursiveErr
300	}
301
302	patternDir := pattern[len(pattern)-1] == '/'
303	nameDir := name[len(name)-1] == '/'
304
305	if patternDir != nameDir {
306		return false, nil
307	}
308
309	if nameDir {
310		name = name[:len(name)-1]
311		pattern = pattern[:len(pattern)-1]
312	}
313
314	for {
315		var patternFile, nameFile string
316		pattern, patternFile = filepath.Dir(pattern), filepath.Base(pattern)
317
318		if patternFile == "**" {
319			if strings.Contains(pattern, "**") {
320				return false, GlobMultipleRecursiveErr
321			}
322			// Test if the any prefix of name matches the part of the pattern before **
323			for {
324				if name == "." || name == "/" {
325					return name == pattern, nil
326				}
327				if match, err := filepath.Match(pattern, name); err != nil {
328					return false, err
329				} else if match {
330					return true, nil
331				}
332				name = filepath.Dir(name)
333			}
334		} else if strings.Contains(patternFile, "**") {
335			return false, GlobInvalidRecursiveErr
336		}
337
338		name, nameFile = filepath.Dir(name), filepath.Base(name)
339
340		if nameFile == "." && patternFile == "." {
341			return true, nil
342		} else if nameFile == "/" && patternFile == "/" {
343			return true, nil
344		} else if nameFile == "." || patternFile == "." || nameFile == "/" || patternFile == "/" {
345			return false, nil
346		}
347
348		match, err := filepath.Match(patternFile, nameFile)
349		if err != nil || !match {
350			return match, err
351		}
352	}
353}
354
355// IsGlob returns true if the pattern contains any glob characters (*, ?, or [).
356func IsGlob(pattern string) bool {
357	return strings.IndexAny(pattern, "*?[") >= 0
358}
359
360// HasGlob returns true if any string in the list contains any glob characters (*, ?, or [).
361func HasGlob(in []string) bool {
362	for _, s := range in {
363		if IsGlob(s) {
364			return true
365		}
366	}
367
368	return false
369}
370
371// WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if
372// the files does not already exist with identical contents.  This can be used
373// along with ninja restat rules to skip rebuilding downstream rules if no
374// changes were made by a rule.
375func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error {
376	var isChanged bool
377
378	dir := filepath.Dir(filename)
379	err := os.MkdirAll(dir, 0777)
380	if err != nil {
381		return err
382	}
383
384	info, err := os.Stat(filename)
385	if err != nil {
386		if os.IsNotExist(err) {
387			// The file does not exist yet.
388			isChanged = true
389		} else {
390			return err
391		}
392	} else {
393		if info.Size() != int64(len(data)) {
394			isChanged = true
395		} else {
396			oldData, err := ioutil.ReadFile(filename)
397			if err != nil {
398				return err
399			}
400
401			if len(oldData) != len(data) {
402				isChanged = true
403			} else {
404				for i := range data {
405					if oldData[i] != data[i] {
406						isChanged = true
407						break
408					}
409				}
410			}
411		}
412	}
413
414	if isChanged {
415		err = ioutil.WriteFile(filename, data, perm)
416		if err != nil {
417			return err
418		}
419	}
420
421	return nil
422}
423
424var matchEscaper = strings.NewReplacer(
425	`*`, `\*`,
426	`?`, `\?`,
427	`[`, `\[`,
428	`]`, `\]`,
429)
430
431// MatchEscape returns its inputs with characters that would be interpreted by
432func MatchEscape(s string) string {
433	return matchEscaper.Replace(s)
434}
435