• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package pathtools
16
17import (
18	"bytes"
19	"fmt"
20	"io"
21	"io/ioutil"
22	"os"
23	"path/filepath"
24	"sort"
25	"strings"
26	"syscall"
27	"time"
28)
29
30// Based on Andrew Gerrand's "10 things you (probably) dont' know about Go"
31
32type ShouldFollowSymlinks bool
33
34const (
35	FollowSymlinks     = ShouldFollowSymlinks(true)
36	DontFollowSymlinks = ShouldFollowSymlinks(false)
37)
38
39var OsFs FileSystem = &osFs{}
40
41func MockFs(files map[string][]byte) FileSystem {
42	fs := &mockFs{
43		files:    make(map[string][]byte, len(files)),
44		dirs:     make(map[string]bool),
45		symlinks: make(map[string]string),
46		all:      []string(nil),
47	}
48
49	for f, b := range files {
50		if tokens := strings.SplitN(f, "->", 2); len(tokens) == 2 {
51			fs.symlinks[strings.TrimSpace(tokens[0])] = strings.TrimSpace(tokens[1])
52			continue
53		}
54
55		fs.files[filepath.Clean(f)] = b
56		dir := filepath.Dir(f)
57		for dir != "." && dir != "/" {
58			fs.dirs[dir] = true
59			dir = filepath.Dir(dir)
60		}
61		fs.dirs[dir] = true
62	}
63
64	fs.dirs["."] = true
65	fs.dirs["/"] = true
66
67	for f := range fs.files {
68		fs.all = append(fs.all, f)
69	}
70
71	for d := range fs.dirs {
72		fs.all = append(fs.all, d)
73	}
74
75	for s := range fs.symlinks {
76		fs.all = append(fs.all, s)
77	}
78
79	sort.Strings(fs.all)
80
81	return fs
82}
83
84type ReaderAtSeekerCloser interface {
85	io.Reader
86	io.ReaderAt
87	io.Seeker
88	io.Closer
89}
90
91type FileSystem interface {
92	// Open opens a file for reading. Follows symlinks.
93	Open(name string) (ReaderAtSeekerCloser, error)
94
95	// Exists returns whether the file exists and whether it is a directory.  Follows symlinks.
96	Exists(name string) (bool, bool, error)
97
98	Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (GlobResult, error)
99	glob(pattern string) (matches []string, err error)
100
101	// IsDir returns true if the path points to a directory, false it it points to a file.  Follows symlinks.
102	// Returns os.ErrNotExist if the path does not exist or is a symlink to a path that does not exist.
103	IsDir(name string) (bool, error)
104
105	// IsSymlink returns true if the path points to a symlink, even if that symlink points to a path that does
106	// not exist.  Returns os.ErrNotExist if the path does not exist.
107	IsSymlink(name string) (bool, error)
108
109	// Lstat returns info on a file without following symlinks.
110	Lstat(name string) (os.FileInfo, error)
111
112	// Lstat returns info on a file.
113	Stat(name string) (os.FileInfo, error)
114
115	// ListDirsRecursive returns a list of all the directories in a path, following symlinks if requested.
116	ListDirsRecursive(name string, follow ShouldFollowSymlinks) (dirs []string, err error)
117
118	// ReadDirNames returns a list of everything in a directory.
119	ReadDirNames(name string) ([]string, error)
120
121	// Readlink returns the destination of the named symbolic link.
122	Readlink(name string) (string, error)
123}
124
125// osFs implements FileSystem using the local disk.
126type osFs struct {
127	srcDir        string
128	openFilesChan chan bool
129}
130
131func NewOsFs(path string) FileSystem {
132	// Darwin has a default limit of 256 open files, rate limit open files to 200
133	limit := 200
134	return &osFs{
135		srcDir:        path,
136		openFilesChan: make(chan bool, limit),
137	}
138}
139
140func (fs *osFs) acquire() {
141	if fs.openFilesChan != nil {
142		fs.openFilesChan <- true
143	}
144}
145
146func (fs *osFs) release() {
147	if fs.openFilesChan != nil {
148		<-fs.openFilesChan
149	}
150}
151
152func (fs *osFs) toAbs(path string) string {
153	if filepath.IsAbs(path) {
154		return path
155	}
156	return filepath.Join(fs.srcDir, path)
157}
158
159func (fs *osFs) removeSrcDirPrefix(path string) string {
160	if fs.srcDir == "" {
161		return path
162	}
163	rel, err := filepath.Rel(fs.srcDir, path)
164	if err != nil {
165		panic(fmt.Errorf("unexpected failure in removeSrcDirPrefix filepath.Rel(%s, %s): %s",
166			fs.srcDir, path, err))
167	}
168	if strings.HasPrefix(rel, "../") {
169		panic(fmt.Errorf("unexpected relative path outside directory in removeSrcDirPrefix filepath.Rel(%s, %s): %s",
170			fs.srcDir, path, rel))
171	}
172	return rel
173}
174
175func (fs *osFs) removeSrcDirPrefixes(paths []string) []string {
176	if fs.srcDir != "" {
177		for i, path := range paths {
178			paths[i] = fs.removeSrcDirPrefix(path)
179		}
180	}
181	return paths
182}
183
184// OsFile wraps an os.File to also release open file descriptors semaphore on close
185type OsFile struct {
186	*os.File
187	fs *osFs
188}
189
190// Close closes file and releases the open file descriptor semaphore
191func (f *OsFile) Close() error {
192	err := f.File.Close()
193	f.fs.release()
194	return err
195}
196
197func (fs *osFs) Open(name string) (ReaderAtSeekerCloser, error) {
198	fs.acquire()
199	f, err := os.Open(fs.toAbs(name))
200	if err != nil {
201		return nil, err
202	}
203	return &OsFile{f, fs}, nil
204}
205
206func (fs *osFs) Exists(name string) (bool, bool, error) {
207	fs.acquire()
208	defer fs.release()
209	stat, err := os.Stat(fs.toAbs(name))
210	if err == nil {
211		return true, stat.IsDir(), nil
212	} else if os.IsNotExist(err) {
213		return false, false, nil
214	} else {
215		return false, false, err
216	}
217}
218
219func (fs *osFs) IsDir(name string) (bool, error) {
220	fs.acquire()
221	defer fs.release()
222	info, err := os.Stat(fs.toAbs(name))
223	if err != nil {
224		return false, err
225	}
226	return info.IsDir(), nil
227}
228
229func (fs *osFs) IsSymlink(name string) (bool, error) {
230	fs.acquire()
231	defer fs.release()
232	if info, err := os.Lstat(fs.toAbs(name)); err != nil {
233		return false, err
234	} else {
235		return info.Mode()&os.ModeSymlink != 0, nil
236	}
237}
238
239func (fs *osFs) Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (GlobResult, error) {
240	return startGlob(fs, pattern, excludes, follow)
241}
242
243func (fs *osFs) glob(pattern string) ([]string, error) {
244	fs.acquire()
245	defer fs.release()
246	paths, err := filepath.Glob(fs.toAbs(pattern))
247	fs.removeSrcDirPrefixes(paths)
248	return paths, err
249}
250
251func (fs *osFs) Lstat(path string) (stats os.FileInfo, err error) {
252	fs.acquire()
253	defer fs.release()
254	return os.Lstat(fs.toAbs(path))
255}
256
257func (fs *osFs) Stat(path string) (stats os.FileInfo, err error) {
258	fs.acquire()
259	defer fs.release()
260	return os.Stat(fs.toAbs(path))
261}
262
263// Returns a list of all directories under dir
264func (fs *osFs) ListDirsRecursive(name string, follow ShouldFollowSymlinks) (dirs []string, err error) {
265	return listDirsRecursive(fs, name, follow)
266}
267
268func (fs *osFs) ReadDirNames(name string) ([]string, error) {
269	fs.acquire()
270	defer fs.release()
271	dir, err := os.Open(fs.toAbs(name))
272	if err != nil {
273		return nil, err
274	}
275	defer dir.Close()
276
277	contents, err := dir.Readdirnames(-1)
278	if err != nil {
279		return nil, err
280	}
281
282	sort.Strings(contents)
283	return contents, nil
284}
285
286func (fs *osFs) Readlink(name string) (string, error) {
287	fs.acquire()
288	defer fs.release()
289	return os.Readlink(fs.toAbs(name))
290}
291
292type mockFs struct {
293	files    map[string][]byte
294	dirs     map[string]bool
295	symlinks map[string]string
296	all      []string
297}
298
299func (m *mockFs) followSymlinks(name string) string {
300	dir, file := quickSplit(name)
301	if dir != "." && dir != "/" {
302		dir = m.followSymlinks(dir)
303	}
304	name = filepath.Join(dir, file)
305
306	for i := 0; i < 255; i++ {
307		i++
308		if i > 255 {
309			panic("symlink loop")
310		}
311		to, exists := m.symlinks[name]
312		if !exists {
313			break
314		}
315		if filepath.IsAbs(to) {
316			name = to
317		} else {
318			name = filepath.Join(dir, to)
319		}
320	}
321	return name
322}
323
324func (m *mockFs) Open(name string) (ReaderAtSeekerCloser, error) {
325	name = filepath.Clean(name)
326	name = m.followSymlinks(name)
327	if f, ok := m.files[name]; ok {
328		return struct {
329			io.Closer
330			*bytes.Reader
331		}{
332			ioutil.NopCloser(nil),
333			bytes.NewReader(f),
334		}, nil
335	}
336
337	return nil, &os.PathError{
338		Op:   "open",
339		Path: name,
340		Err:  os.ErrNotExist,
341	}
342}
343
344func (m *mockFs) Exists(name string) (bool, bool, error) {
345	name = filepath.Clean(name)
346	name = m.followSymlinks(name)
347	if _, ok := m.files[name]; ok {
348		return ok, false, nil
349	}
350	if _, ok := m.dirs[name]; ok {
351		return ok, true, nil
352	}
353	return false, false, nil
354}
355
356func (m *mockFs) IsDir(name string) (bool, error) {
357	dir := filepath.Dir(name)
358	if dir != "." && dir != "/" {
359		isDir, err := m.IsDir(dir)
360
361		if serr, ok := err.(*os.SyscallError); ok && serr.Err == syscall.ENOTDIR {
362			isDir = false
363		} else if err != nil {
364			return false, err
365		}
366
367		if !isDir {
368			return false, os.NewSyscallError("stat "+name, syscall.ENOTDIR)
369		}
370	}
371
372	name = filepath.Clean(name)
373	name = m.followSymlinks(name)
374
375	if _, ok := m.dirs[name]; ok {
376		return true, nil
377	}
378	if _, ok := m.files[name]; ok {
379		return false, nil
380	}
381	return false, os.ErrNotExist
382}
383
384func (m *mockFs) IsSymlink(name string) (bool, error) {
385	dir, file := quickSplit(name)
386	dir = m.followSymlinks(dir)
387	name = filepath.Join(dir, file)
388
389	if _, isSymlink := m.symlinks[name]; isSymlink {
390		return true, nil
391	}
392	if _, isDir := m.dirs[name]; isDir {
393		return false, nil
394	}
395	if _, isFile := m.files[name]; isFile {
396		return false, nil
397	}
398	return false, os.ErrNotExist
399}
400
401func (m *mockFs) Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (GlobResult, error) {
402	return startGlob(m, pattern, excludes, follow)
403}
404
405func unescapeGlob(s string) string {
406	i := 0
407	for i < len(s) {
408		if s[i] == '\\' {
409			s = s[:i] + s[i+1:]
410		} else {
411			i++
412		}
413	}
414	return s
415}
416
417func (m *mockFs) glob(pattern string) ([]string, error) {
418	dir, file := quickSplit(pattern)
419
420	dir = unescapeGlob(dir)
421	toDir := m.followSymlinks(dir)
422
423	var matches []string
424	for _, f := range m.all {
425		fDir, fFile := quickSplit(f)
426		if toDir == fDir {
427			match, err := filepath.Match(file, fFile)
428			if err != nil {
429				return nil, err
430			}
431			if (f == "." || f == "/") && f != pattern {
432				// filepath.Glob won't return "." or "/" unless the pattern was "." or "/"
433				match = false
434			}
435			if match {
436				matches = append(matches, filepath.Join(dir, fFile))
437			}
438		}
439	}
440	return matches, nil
441}
442
443type mockStat struct {
444	name string
445	size int64
446	mode os.FileMode
447}
448
449func (ms *mockStat) Name() string       { return ms.name }
450func (ms *mockStat) IsDir() bool        { return ms.Mode().IsDir() }
451func (ms *mockStat) Size() int64        { return ms.size }
452func (ms *mockStat) Mode() os.FileMode  { return ms.mode }
453func (ms *mockStat) ModTime() time.Time { return time.Time{} }
454func (ms *mockStat) Sys() interface{}   { return nil }
455
456func (m *mockFs) Lstat(name string) (os.FileInfo, error) {
457	dir, file := quickSplit(name)
458	dir = m.followSymlinks(dir)
459	name = filepath.Join(dir, file)
460
461	ms := mockStat{
462		name: file,
463	}
464
465	if symlink, isSymlink := m.symlinks[name]; isSymlink {
466		ms.mode = os.ModeSymlink
467		ms.size = int64(len(symlink))
468	} else if _, isDir := m.dirs[name]; isDir {
469		ms.mode = os.ModeDir
470	} else if _, isFile := m.files[name]; isFile {
471		ms.mode = 0
472		ms.size = int64(len(m.files[name]))
473	} else {
474		return nil, os.ErrNotExist
475	}
476
477	return &ms, nil
478}
479
480func (m *mockFs) Stat(name string) (os.FileInfo, error) {
481	name = filepath.Clean(name)
482	origName := name
483	name = m.followSymlinks(name)
484
485	ms := mockStat{
486		name: filepath.Base(origName),
487		size: int64(len(m.files[name])),
488	}
489
490	if _, isDir := m.dirs[name]; isDir {
491		ms.mode = os.ModeDir
492	} else if _, isFile := m.files[name]; isFile {
493		ms.mode = 0
494		ms.size = int64(len(m.files[name]))
495	} else {
496		return nil, os.ErrNotExist
497	}
498
499	return &ms, nil
500}
501
502func (m *mockFs) ReadDirNames(name string) ([]string, error) {
503	name = filepath.Clean(name)
504	name = m.followSymlinks(name)
505
506	exists, isDir, err := m.Exists(name)
507	if err != nil {
508		return nil, err
509	}
510	if !exists {
511		return nil, os.ErrNotExist
512	}
513	if !isDir {
514		return nil, os.NewSyscallError("readdir", syscall.ENOTDIR)
515	}
516
517	var ret []string
518	for _, f := range m.all {
519		dir, file := quickSplit(f)
520		if dir == name && len(file) > 0 && file[0] != '.' {
521			ret = append(ret, file)
522		}
523	}
524	return ret, nil
525}
526
527func (m *mockFs) ListDirsRecursive(name string, follow ShouldFollowSymlinks) ([]string, error) {
528	return listDirsRecursive(m, name, follow)
529}
530
531func (m *mockFs) Readlink(name string) (string, error) {
532	dir, file := quickSplit(name)
533	dir = m.followSymlinks(dir)
534
535	origName := name
536	name = filepath.Join(dir, file)
537
538	if dest, isSymlink := m.symlinks[name]; isSymlink {
539		return dest, nil
540	}
541
542	if exists, _, err := m.Exists(name); err != nil {
543		return "", err
544	} else if !exists {
545		return "", os.ErrNotExist
546	} else {
547		return "", os.NewSyscallError("readlink: "+origName, syscall.EINVAL)
548	}
549}
550
551func listDirsRecursive(fs FileSystem, name string, follow ShouldFollowSymlinks) ([]string, error) {
552	name = filepath.Clean(name)
553
554	isDir, err := fs.IsDir(name)
555	if err != nil {
556		return nil, err
557	}
558
559	if !isDir {
560		return nil, nil
561	}
562
563	dirs := []string{name}
564
565	subDirs, err := listDirsRecursiveRelative(fs, name, follow, 0)
566	if err != nil {
567		return nil, err
568	}
569
570	for _, d := range subDirs {
571		dirs = append(dirs, filepath.Join(name, d))
572	}
573
574	return dirs, nil
575}
576
577func listDirsRecursiveRelative(fs FileSystem, name string, follow ShouldFollowSymlinks, depth int) ([]string, error) {
578	depth++
579	if depth > 255 {
580		return nil, fmt.Errorf("too many symlinks")
581	}
582	contents, err := fs.ReadDirNames(name)
583	if err != nil {
584		return nil, err
585	}
586
587	var dirs []string
588	for _, f := range contents {
589		if f[0] == '.' {
590			continue
591		}
592		f = filepath.Join(name, f)
593		var info os.FileInfo
594		if follow == DontFollowSymlinks {
595			info, err = fs.Lstat(f)
596			if err != nil {
597				continue
598			}
599			if info.Mode()&os.ModeSymlink != 0 {
600				continue
601			}
602		} else {
603			info, err = fs.Stat(f)
604			if err != nil {
605				continue
606			}
607		}
608		if info.IsDir() {
609			dirs = append(dirs, f)
610			subDirs, err := listDirsRecursiveRelative(fs, f, follow, depth)
611			if err != nil {
612				return nil, err
613			}
614			for _, s := range subDirs {
615				dirs = append(dirs, filepath.Join(f, s))
616			}
617		}
618	}
619
620	for i, d := range dirs {
621		rel, err := filepath.Rel(name, d)
622		if err != nil {
623			return nil, err
624		}
625		dirs[i] = rel
626	}
627
628	return dirs, nil
629}
630