1// Copyright 2016 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package pathtools 16 17import ( 18 "bytes" 19 "fmt" 20 "io" 21 "io/ioutil" 22 "os" 23 "path/filepath" 24 "sort" 25 "strings" 26 "syscall" 27 "time" 28) 29 30// Based on Andrew Gerrand's "10 things you (probably) dont' know about Go" 31 32type ShouldFollowSymlinks bool 33 34const ( 35 FollowSymlinks = ShouldFollowSymlinks(true) 36 DontFollowSymlinks = ShouldFollowSymlinks(false) 37) 38 39var OsFs FileSystem = &osFs{} 40 41func MockFs(files map[string][]byte) FileSystem { 42 fs := &mockFs{ 43 files: make(map[string][]byte, len(files)), 44 dirs: make(map[string]bool), 45 symlinks: make(map[string]string), 46 all: []string(nil), 47 } 48 49 for f, b := range files { 50 if tokens := strings.SplitN(f, "->", 2); len(tokens) == 2 { 51 fs.symlinks[strings.TrimSpace(tokens[0])] = strings.TrimSpace(tokens[1]) 52 continue 53 } 54 55 fs.files[filepath.Clean(f)] = b 56 dir := filepath.Dir(f) 57 for dir != "." && dir != "/" { 58 fs.dirs[dir] = true 59 dir = filepath.Dir(dir) 60 } 61 fs.dirs[dir] = true 62 } 63 64 fs.dirs["."] = true 65 fs.dirs["/"] = true 66 67 for f := range fs.files { 68 fs.all = append(fs.all, f) 69 } 70 71 for d := range fs.dirs { 72 fs.all = append(fs.all, d) 73 } 74 75 for s := range fs.symlinks { 76 fs.all = append(fs.all, s) 77 } 78 79 sort.Strings(fs.all) 80 81 return fs 82} 83 84type ReaderAtSeekerCloser interface { 85 io.Reader 86 io.ReaderAt 87 io.Seeker 88 io.Closer 89} 90 91type FileSystem interface { 92 // Open opens a file for reading. Follows symlinks. 93 Open(name string) (ReaderAtSeekerCloser, error) 94 95 // Exists returns whether the file exists and whether it is a directory. Follows symlinks. 96 Exists(name string) (bool, bool, error) 97 98 Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (GlobResult, error) 99 glob(pattern string) (matches []string, err error) 100 101 // IsDir returns true if the path points to a directory, false it it points to a file. Follows symlinks. 102 // Returns os.ErrNotExist if the path does not exist or is a symlink to a path that does not exist. 103 IsDir(name string) (bool, error) 104 105 // IsSymlink returns true if the path points to a symlink, even if that symlink points to a path that does 106 // not exist. Returns os.ErrNotExist if the path does not exist. 107 IsSymlink(name string) (bool, error) 108 109 // Lstat returns info on a file without following symlinks. 110 Lstat(name string) (os.FileInfo, error) 111 112 // Lstat returns info on a file. 113 Stat(name string) (os.FileInfo, error) 114 115 // ListDirsRecursive returns a list of all the directories in a path, following symlinks if requested. 116 ListDirsRecursive(name string, follow ShouldFollowSymlinks) (dirs []string, err error) 117 118 // ReadDirNames returns a list of everything in a directory. 119 ReadDirNames(name string) ([]string, error) 120 121 // Readlink returns the destination of the named symbolic link. 122 Readlink(name string) (string, error) 123} 124 125// osFs implements FileSystem using the local disk. 126type osFs struct { 127 srcDir string 128 openFilesChan chan bool 129} 130 131func NewOsFs(path string) FileSystem { 132 // Darwin has a default limit of 256 open files, rate limit open files to 200 133 limit := 200 134 return &osFs{ 135 srcDir: path, 136 openFilesChan: make(chan bool, limit), 137 } 138} 139 140func (fs *osFs) acquire() { 141 if fs.openFilesChan != nil { 142 fs.openFilesChan <- true 143 } 144} 145 146func (fs *osFs) release() { 147 if fs.openFilesChan != nil { 148 <-fs.openFilesChan 149 } 150} 151 152func (fs *osFs) toAbs(path string) string { 153 if filepath.IsAbs(path) { 154 return path 155 } 156 return filepath.Join(fs.srcDir, path) 157} 158 159func (fs *osFs) removeSrcDirPrefix(path string) string { 160 if fs.srcDir == "" { 161 return path 162 } 163 rel, err := filepath.Rel(fs.srcDir, path) 164 if err != nil { 165 panic(fmt.Errorf("unexpected failure in removeSrcDirPrefix filepath.Rel(%s, %s): %s", 166 fs.srcDir, path, err)) 167 } 168 if strings.HasPrefix(rel, "../") { 169 panic(fmt.Errorf("unexpected relative path outside directory in removeSrcDirPrefix filepath.Rel(%s, %s): %s", 170 fs.srcDir, path, rel)) 171 } 172 return rel 173} 174 175func (fs *osFs) removeSrcDirPrefixes(paths []string) []string { 176 if fs.srcDir != "" { 177 for i, path := range paths { 178 paths[i] = fs.removeSrcDirPrefix(path) 179 } 180 } 181 return paths 182} 183 184// OsFile wraps an os.File to also release open file descriptors semaphore on close 185type OsFile struct { 186 *os.File 187 fs *osFs 188} 189 190// Close closes file and releases the open file descriptor semaphore 191func (f *OsFile) Close() error { 192 err := f.File.Close() 193 f.fs.release() 194 return err 195} 196 197func (fs *osFs) Open(name string) (ReaderAtSeekerCloser, error) { 198 fs.acquire() 199 f, err := os.Open(fs.toAbs(name)) 200 if err != nil { 201 return nil, err 202 } 203 return &OsFile{f, fs}, nil 204} 205 206func (fs *osFs) Exists(name string) (bool, bool, error) { 207 fs.acquire() 208 defer fs.release() 209 stat, err := os.Stat(fs.toAbs(name)) 210 if err == nil { 211 return true, stat.IsDir(), nil 212 } else if os.IsNotExist(err) { 213 return false, false, nil 214 } else { 215 return false, false, err 216 } 217} 218 219func (fs *osFs) IsDir(name string) (bool, error) { 220 fs.acquire() 221 defer fs.release() 222 info, err := os.Stat(fs.toAbs(name)) 223 if err != nil { 224 return false, err 225 } 226 return info.IsDir(), nil 227} 228 229func (fs *osFs) IsSymlink(name string) (bool, error) { 230 fs.acquire() 231 defer fs.release() 232 if info, err := os.Lstat(fs.toAbs(name)); err != nil { 233 return false, err 234 } else { 235 return info.Mode()&os.ModeSymlink != 0, nil 236 } 237} 238 239func (fs *osFs) Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (GlobResult, error) { 240 return startGlob(fs, pattern, excludes, follow) 241} 242 243func (fs *osFs) glob(pattern string) ([]string, error) { 244 fs.acquire() 245 defer fs.release() 246 paths, err := filepath.Glob(fs.toAbs(pattern)) 247 fs.removeSrcDirPrefixes(paths) 248 return paths, err 249} 250 251func (fs *osFs) Lstat(path string) (stats os.FileInfo, err error) { 252 fs.acquire() 253 defer fs.release() 254 return os.Lstat(fs.toAbs(path)) 255} 256 257func (fs *osFs) Stat(path string) (stats os.FileInfo, err error) { 258 fs.acquire() 259 defer fs.release() 260 return os.Stat(fs.toAbs(path)) 261} 262 263// Returns a list of all directories under dir 264func (fs *osFs) ListDirsRecursive(name string, follow ShouldFollowSymlinks) (dirs []string, err error) { 265 return listDirsRecursive(fs, name, follow) 266} 267 268func (fs *osFs) ReadDirNames(name string) ([]string, error) { 269 fs.acquire() 270 defer fs.release() 271 dir, err := os.Open(fs.toAbs(name)) 272 if err != nil { 273 return nil, err 274 } 275 defer dir.Close() 276 277 contents, err := dir.Readdirnames(-1) 278 if err != nil { 279 return nil, err 280 } 281 282 sort.Strings(contents) 283 return contents, nil 284} 285 286func (fs *osFs) Readlink(name string) (string, error) { 287 fs.acquire() 288 defer fs.release() 289 return os.Readlink(fs.toAbs(name)) 290} 291 292type mockFs struct { 293 files map[string][]byte 294 dirs map[string]bool 295 symlinks map[string]string 296 all []string 297} 298 299func (m *mockFs) followSymlinks(name string) string { 300 dir, file := quickSplit(name) 301 if dir != "." && dir != "/" { 302 dir = m.followSymlinks(dir) 303 } 304 name = filepath.Join(dir, file) 305 306 for i := 0; i < 255; i++ { 307 i++ 308 if i > 255 { 309 panic("symlink loop") 310 } 311 to, exists := m.symlinks[name] 312 if !exists { 313 break 314 } 315 if filepath.IsAbs(to) { 316 name = to 317 } else { 318 name = filepath.Join(dir, to) 319 } 320 } 321 return name 322} 323 324func (m *mockFs) Open(name string) (ReaderAtSeekerCloser, error) { 325 name = filepath.Clean(name) 326 name = m.followSymlinks(name) 327 if f, ok := m.files[name]; ok { 328 return struct { 329 io.Closer 330 *bytes.Reader 331 }{ 332 ioutil.NopCloser(nil), 333 bytes.NewReader(f), 334 }, nil 335 } 336 337 return nil, &os.PathError{ 338 Op: "open", 339 Path: name, 340 Err: os.ErrNotExist, 341 } 342} 343 344func (m *mockFs) Exists(name string) (bool, bool, error) { 345 name = filepath.Clean(name) 346 name = m.followSymlinks(name) 347 if _, ok := m.files[name]; ok { 348 return ok, false, nil 349 } 350 if _, ok := m.dirs[name]; ok { 351 return ok, true, nil 352 } 353 return false, false, nil 354} 355 356func (m *mockFs) IsDir(name string) (bool, error) { 357 dir := filepath.Dir(name) 358 if dir != "." && dir != "/" { 359 isDir, err := m.IsDir(dir) 360 361 if serr, ok := err.(*os.SyscallError); ok && serr.Err == syscall.ENOTDIR { 362 isDir = false 363 } else if err != nil { 364 return false, err 365 } 366 367 if !isDir { 368 return false, os.NewSyscallError("stat "+name, syscall.ENOTDIR) 369 } 370 } 371 372 name = filepath.Clean(name) 373 name = m.followSymlinks(name) 374 375 if _, ok := m.dirs[name]; ok { 376 return true, nil 377 } 378 if _, ok := m.files[name]; ok { 379 return false, nil 380 } 381 return false, os.ErrNotExist 382} 383 384func (m *mockFs) IsSymlink(name string) (bool, error) { 385 dir, file := quickSplit(name) 386 dir = m.followSymlinks(dir) 387 name = filepath.Join(dir, file) 388 389 if _, isSymlink := m.symlinks[name]; isSymlink { 390 return true, nil 391 } 392 if _, isDir := m.dirs[name]; isDir { 393 return false, nil 394 } 395 if _, isFile := m.files[name]; isFile { 396 return false, nil 397 } 398 return false, os.ErrNotExist 399} 400 401func (m *mockFs) Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (GlobResult, error) { 402 return startGlob(m, pattern, excludes, follow) 403} 404 405func unescapeGlob(s string) string { 406 i := 0 407 for i < len(s) { 408 if s[i] == '\\' { 409 s = s[:i] + s[i+1:] 410 } else { 411 i++ 412 } 413 } 414 return s 415} 416 417func (m *mockFs) glob(pattern string) ([]string, error) { 418 dir, file := quickSplit(pattern) 419 420 dir = unescapeGlob(dir) 421 toDir := m.followSymlinks(dir) 422 423 var matches []string 424 for _, f := range m.all { 425 fDir, fFile := quickSplit(f) 426 if toDir == fDir { 427 match, err := filepath.Match(file, fFile) 428 if err != nil { 429 return nil, err 430 } 431 if (f == "." || f == "/") && f != pattern { 432 // filepath.Glob won't return "." or "/" unless the pattern was "." or "/" 433 match = false 434 } 435 if match { 436 matches = append(matches, filepath.Join(dir, fFile)) 437 } 438 } 439 } 440 return matches, nil 441} 442 443type mockStat struct { 444 name string 445 size int64 446 mode os.FileMode 447} 448 449func (ms *mockStat) Name() string { return ms.name } 450func (ms *mockStat) IsDir() bool { return ms.Mode().IsDir() } 451func (ms *mockStat) Size() int64 { return ms.size } 452func (ms *mockStat) Mode() os.FileMode { return ms.mode } 453func (ms *mockStat) ModTime() time.Time { return time.Time{} } 454func (ms *mockStat) Sys() interface{} { return nil } 455 456func (m *mockFs) Lstat(name string) (os.FileInfo, error) { 457 dir, file := quickSplit(name) 458 dir = m.followSymlinks(dir) 459 name = filepath.Join(dir, file) 460 461 ms := mockStat{ 462 name: file, 463 } 464 465 if symlink, isSymlink := m.symlinks[name]; isSymlink { 466 ms.mode = os.ModeSymlink 467 ms.size = int64(len(symlink)) 468 } else if _, isDir := m.dirs[name]; isDir { 469 ms.mode = os.ModeDir 470 } else if _, isFile := m.files[name]; isFile { 471 ms.mode = 0 472 ms.size = int64(len(m.files[name])) 473 } else { 474 return nil, os.ErrNotExist 475 } 476 477 return &ms, nil 478} 479 480func (m *mockFs) Stat(name string) (os.FileInfo, error) { 481 name = filepath.Clean(name) 482 origName := name 483 name = m.followSymlinks(name) 484 485 ms := mockStat{ 486 name: filepath.Base(origName), 487 size: int64(len(m.files[name])), 488 } 489 490 if _, isDir := m.dirs[name]; isDir { 491 ms.mode = os.ModeDir 492 } else if _, isFile := m.files[name]; isFile { 493 ms.mode = 0 494 ms.size = int64(len(m.files[name])) 495 } else { 496 return nil, os.ErrNotExist 497 } 498 499 return &ms, nil 500} 501 502func (m *mockFs) ReadDirNames(name string) ([]string, error) { 503 name = filepath.Clean(name) 504 name = m.followSymlinks(name) 505 506 exists, isDir, err := m.Exists(name) 507 if err != nil { 508 return nil, err 509 } 510 if !exists { 511 return nil, os.ErrNotExist 512 } 513 if !isDir { 514 return nil, os.NewSyscallError("readdir", syscall.ENOTDIR) 515 } 516 517 var ret []string 518 for _, f := range m.all { 519 dir, file := quickSplit(f) 520 if dir == name && len(file) > 0 && file[0] != '.' { 521 ret = append(ret, file) 522 } 523 } 524 return ret, nil 525} 526 527func (m *mockFs) ListDirsRecursive(name string, follow ShouldFollowSymlinks) ([]string, error) { 528 return listDirsRecursive(m, name, follow) 529} 530 531func (m *mockFs) Readlink(name string) (string, error) { 532 dir, file := quickSplit(name) 533 dir = m.followSymlinks(dir) 534 535 origName := name 536 name = filepath.Join(dir, file) 537 538 if dest, isSymlink := m.symlinks[name]; isSymlink { 539 return dest, nil 540 } 541 542 if exists, _, err := m.Exists(name); err != nil { 543 return "", err 544 } else if !exists { 545 return "", os.ErrNotExist 546 } else { 547 return "", os.NewSyscallError("readlink: "+origName, syscall.EINVAL) 548 } 549} 550 551func listDirsRecursive(fs FileSystem, name string, follow ShouldFollowSymlinks) ([]string, error) { 552 name = filepath.Clean(name) 553 554 isDir, err := fs.IsDir(name) 555 if err != nil { 556 return nil, err 557 } 558 559 if !isDir { 560 return nil, nil 561 } 562 563 dirs := []string{name} 564 565 subDirs, err := listDirsRecursiveRelative(fs, name, follow, 0) 566 if err != nil { 567 return nil, err 568 } 569 570 for _, d := range subDirs { 571 dirs = append(dirs, filepath.Join(name, d)) 572 } 573 574 return dirs, nil 575} 576 577func listDirsRecursiveRelative(fs FileSystem, name string, follow ShouldFollowSymlinks, depth int) ([]string, error) { 578 depth++ 579 if depth > 255 { 580 return nil, fmt.Errorf("too many symlinks") 581 } 582 contents, err := fs.ReadDirNames(name) 583 if err != nil { 584 return nil, err 585 } 586 587 var dirs []string 588 for _, f := range contents { 589 if f[0] == '.' { 590 continue 591 } 592 f = filepath.Join(name, f) 593 var info os.FileInfo 594 if follow == DontFollowSymlinks { 595 info, err = fs.Lstat(f) 596 if err != nil { 597 continue 598 } 599 if info.Mode()&os.ModeSymlink != 0 { 600 continue 601 } 602 } else { 603 info, err = fs.Stat(f) 604 if err != nil { 605 continue 606 } 607 } 608 if info.IsDir() { 609 dirs = append(dirs, f) 610 subDirs, err := listDirsRecursiveRelative(fs, f, follow, depth) 611 if err != nil { 612 return nil, err 613 } 614 for _, s := range subDirs { 615 dirs = append(dirs, filepath.Join(f, s)) 616 } 617 } 618 } 619 620 for i, d := range dirs { 621 rel, err := filepath.Rel(name, d) 622 if err != nil { 623 return nil, err 624 } 625 dirs[i] = rel 626 } 627 628 return dirs, nil 629} 630