1// Copyright 2015 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package zip 16 17import ( 18 "bytes" 19 "compress/flate" 20 "errors" 21 "fmt" 22 "hash/crc32" 23 "io" 24 "io/ioutil" 25 "os" 26 "path/filepath" 27 "sort" 28 "strings" 29 "sync" 30 "syscall" 31 "time" 32 "unicode" 33 34 "github.com/google/blueprint/pathtools" 35 36 "android/soong/jar" 37 "android/soong/third_party/zip" 38) 39 40// Block size used during parallel compression of a single file. 41const parallelBlockSize = 1 * 1024 * 1024 // 1MB 42 43// Minimum file size to use parallel compression. It requires more 44// flate.Writer allocations, since we can't change the dictionary 45// during Reset 46const minParallelFileSize = parallelBlockSize * 6 47 48// Size of the ZIP compression window (32KB) 49const windowSize = 32 * 1024 50 51type nopCloser struct { 52 io.Writer 53} 54 55func (nopCloser) Close() error { 56 return nil 57} 58 59type byteReaderCloser struct { 60 *bytes.Reader 61 io.Closer 62} 63 64type pathMapping struct { 65 dest, src string 66 zipMethod uint16 67} 68 69type FileArg struct { 70 PathPrefixInZip, SourcePrefixToStrip string 71 SourceFiles []string 72 JunkPaths bool 73 GlobDir string 74} 75 76type FileArgsBuilder struct { 77 state FileArg 78 err error 79 fs pathtools.FileSystem 80 81 fileArgs []FileArg 82} 83 84func NewFileArgsBuilder() *FileArgsBuilder { 85 return &FileArgsBuilder{ 86 fs: pathtools.OsFs, 87 } 88} 89 90func (b *FileArgsBuilder) JunkPaths(v bool) *FileArgsBuilder { 91 b.state.JunkPaths = v 92 b.state.SourcePrefixToStrip = "" 93 return b 94} 95 96func (b *FileArgsBuilder) SourcePrefixToStrip(prefixToStrip string) *FileArgsBuilder { 97 b.state.JunkPaths = false 98 b.state.SourcePrefixToStrip = prefixToStrip 99 return b 100} 101 102func (b *FileArgsBuilder) PathPrefixInZip(rootPrefix string) *FileArgsBuilder { 103 b.state.PathPrefixInZip = rootPrefix 104 return b 105} 106 107func (b *FileArgsBuilder) File(name string) *FileArgsBuilder { 108 if b.err != nil { 109 return b 110 } 111 112 arg := b.state 113 arg.SourceFiles = []string{name} 114 b.fileArgs = append(b.fileArgs, arg) 115 return b 116} 117 118func (b *FileArgsBuilder) Dir(name string) *FileArgsBuilder { 119 if b.err != nil { 120 return b 121 } 122 123 arg := b.state 124 arg.GlobDir = name 125 b.fileArgs = append(b.fileArgs, arg) 126 return b 127} 128 129func (b *FileArgsBuilder) List(name string) *FileArgsBuilder { 130 if b.err != nil { 131 return b 132 } 133 134 f, err := b.fs.Open(name) 135 if err != nil { 136 b.err = err 137 return b 138 } 139 defer f.Close() 140 141 list, err := ioutil.ReadAll(f) 142 if err != nil { 143 b.err = err 144 return b 145 } 146 147 arg := b.state 148 arg.SourceFiles = strings.Split(string(list), "\n") 149 b.fileArgs = append(b.fileArgs, arg) 150 return b 151} 152 153func (b *FileArgsBuilder) Error() error { 154 if b == nil { 155 return nil 156 } 157 return b.err 158} 159 160func (b *FileArgsBuilder) FileArgs() []FileArg { 161 if b == nil { 162 return nil 163 } 164 return b.fileArgs 165} 166 167type IncorrectRelativeRootError struct { 168 RelativeRoot string 169 Path string 170} 171 172func (x IncorrectRelativeRootError) Error() string { 173 return fmt.Sprintf("path %q is outside relative root %q", x.Path, x.RelativeRoot) 174} 175 176type ZipWriter struct { 177 time time.Time 178 createdFiles map[string]string 179 createdDirs map[string]string 180 directories bool 181 182 errors chan error 183 writeOps chan chan *zipEntry 184 185 cpuRateLimiter *CPURateLimiter 186 memoryRateLimiter *MemoryRateLimiter 187 188 compressorPool sync.Pool 189 compLevel int 190 191 followSymlinks pathtools.ShouldFollowSymlinks 192 ignoreMissingFiles bool 193 194 stderr io.Writer 195 fs pathtools.FileSystem 196} 197 198type zipEntry struct { 199 fh *zip.FileHeader 200 201 // List of delayed io.Reader 202 futureReaders chan chan io.Reader 203 204 // Only used for passing into the MemoryRateLimiter to ensure we 205 // release as much memory as much as we request 206 allocatedSize int64 207} 208 209type ZipArgs struct { 210 FileArgs []FileArg 211 OutputFilePath string 212 EmulateJar bool 213 AddDirectoryEntriesToZip bool 214 CompressionLevel int 215 ManifestSourcePath string 216 NumParallelJobs int 217 NonDeflatedFiles map[string]bool 218 WriteIfChanged bool 219 StoreSymlinks bool 220 IgnoreMissingFiles bool 221 222 Stderr io.Writer 223 Filesystem pathtools.FileSystem 224} 225 226const NOQUOTE = '\x00' 227 228func ReadRespFile(bytes []byte) []string { 229 var args []string 230 var arg []rune 231 232 isEscaping := false 233 quotingStart := NOQUOTE 234 for _, c := range string(bytes) { 235 switch { 236 case isEscaping: 237 if quotingStart == '"' { 238 if !(c == '"' || c == '\\') { 239 // '\"' or '\\' will be escaped under double quoting. 240 arg = append(arg, '\\') 241 } 242 } 243 arg = append(arg, c) 244 isEscaping = false 245 case c == '\\' && quotingStart != '\'': 246 isEscaping = true 247 case quotingStart == NOQUOTE && (c == '\'' || c == '"'): 248 quotingStart = c 249 case quotingStart != NOQUOTE && c == quotingStart: 250 quotingStart = NOQUOTE 251 case quotingStart == NOQUOTE && unicode.IsSpace(c): 252 // Current character is a space outside quotes 253 if len(arg) != 0 { 254 args = append(args, string(arg)) 255 } 256 arg = arg[:0] 257 default: 258 arg = append(arg, c) 259 } 260 } 261 262 if len(arg) != 0 { 263 args = append(args, string(arg)) 264 } 265 266 return args 267} 268 269func ZipTo(args ZipArgs, w io.Writer) error { 270 if args.EmulateJar { 271 args.AddDirectoryEntriesToZip = true 272 } 273 274 // Have Glob follow symlinks if they are not being stored as symlinks in the zip file. 275 followSymlinks := pathtools.ShouldFollowSymlinks(!args.StoreSymlinks) 276 277 z := &ZipWriter{ 278 time: jar.DefaultTime, 279 createdDirs: make(map[string]string), 280 createdFiles: make(map[string]string), 281 directories: args.AddDirectoryEntriesToZip, 282 compLevel: args.CompressionLevel, 283 followSymlinks: followSymlinks, 284 ignoreMissingFiles: args.IgnoreMissingFiles, 285 stderr: args.Stderr, 286 fs: args.Filesystem, 287 } 288 289 if z.fs == nil { 290 z.fs = pathtools.OsFs 291 } 292 293 if z.stderr == nil { 294 z.stderr = os.Stderr 295 } 296 297 pathMappings := []pathMapping{} 298 299 noCompression := args.CompressionLevel == 0 300 301 for _, fa := range args.FileArgs { 302 var srcs []string 303 for _, s := range fa.SourceFiles { 304 s = strings.TrimSpace(s) 305 if s == "" { 306 continue 307 } 308 309 globbed, _, err := z.fs.Glob(s, nil, followSymlinks) 310 if err != nil { 311 return err 312 } 313 if len(globbed) == 0 { 314 err := &os.PathError{ 315 Op: "lstat", 316 Path: s, 317 Err: os.ErrNotExist, 318 } 319 if args.IgnoreMissingFiles { 320 fmt.Fprintln(z.stderr, "warning:", err) 321 } else { 322 return err 323 } 324 } 325 srcs = append(srcs, globbed...) 326 } 327 if fa.GlobDir != "" { 328 if exists, isDir, err := z.fs.Exists(fa.GlobDir); err != nil { 329 return err 330 } else if !exists && !args.IgnoreMissingFiles { 331 err := &os.PathError{ 332 Op: "lstat", 333 Path: fa.GlobDir, 334 Err: os.ErrNotExist, 335 } 336 if args.IgnoreMissingFiles { 337 fmt.Fprintln(z.stderr, "warning:", err) 338 } else { 339 return err 340 } 341 } else if !isDir && !args.IgnoreMissingFiles { 342 err := &os.PathError{ 343 Op: "lstat", 344 Path: fa.GlobDir, 345 Err: syscall.ENOTDIR, 346 } 347 if args.IgnoreMissingFiles { 348 fmt.Fprintln(z.stderr, "warning:", err) 349 } else { 350 return err 351 } 352 } 353 globbed, _, err := z.fs.Glob(filepath.Join(fa.GlobDir, "**/*"), nil, followSymlinks) 354 if err != nil { 355 return err 356 } 357 srcs = append(srcs, globbed...) 358 } 359 for _, src := range srcs { 360 err := fillPathPairs(fa, src, &pathMappings, args.NonDeflatedFiles, noCompression) 361 if err != nil { 362 return err 363 } 364 } 365 } 366 367 return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.NumParallelJobs) 368} 369 370func Zip(args ZipArgs) error { 371 if args.OutputFilePath == "" { 372 return fmt.Errorf("output file path must be nonempty") 373 } 374 375 buf := &bytes.Buffer{} 376 var out io.Writer = buf 377 378 if !args.WriteIfChanged { 379 f, err := os.Create(args.OutputFilePath) 380 if err != nil { 381 return err 382 } 383 384 defer f.Close() 385 defer func() { 386 if err != nil { 387 os.Remove(args.OutputFilePath) 388 } 389 }() 390 391 out = f 392 } 393 394 err := ZipTo(args, out) 395 if err != nil { 396 return err 397 } 398 399 if args.WriteIfChanged { 400 err := pathtools.WriteFileIfChanged(args.OutputFilePath, buf.Bytes(), 0666) 401 if err != nil { 402 return err 403 } 404 } 405 406 return nil 407} 408 409func fillPathPairs(fa FileArg, src string, pathMappings *[]pathMapping, 410 nonDeflatedFiles map[string]bool, noCompression bool) error { 411 412 var dest string 413 414 if fa.JunkPaths { 415 dest = filepath.Base(src) 416 } else { 417 var err error 418 dest, err = filepath.Rel(fa.SourcePrefixToStrip, src) 419 if err != nil { 420 return err 421 } 422 if strings.HasPrefix(dest, "../") { 423 return IncorrectRelativeRootError{ 424 Path: src, 425 RelativeRoot: fa.SourcePrefixToStrip, 426 } 427 } 428 429 } 430 dest = filepath.Join(fa.PathPrefixInZip, dest) 431 432 zipMethod := zip.Deflate 433 if _, found := nonDeflatedFiles[dest]; found || noCompression { 434 zipMethod = zip.Store 435 } 436 *pathMappings = append(*pathMappings, 437 pathMapping{dest: dest, src: src, zipMethod: zipMethod}) 438 439 return nil 440} 441 442func jarSort(mappings []pathMapping) { 443 less := func(i int, j int) (smaller bool) { 444 return jar.EntryNamesLess(mappings[i].dest, mappings[j].dest) 445 } 446 sort.SliceStable(mappings, less) 447} 448 449func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar bool, parallelJobs int) error { 450 z.errors = make(chan error) 451 defer close(z.errors) 452 453 // This channel size can be essentially unlimited -- it's used as a fifo 454 // queue decouple the CPU and IO loads. Directories don't require any 455 // compression time, but still cost some IO. Similar with small files that 456 // can be very fast to compress. Some files that are more difficult to 457 // compress won't take a corresponding longer time writing out. 458 // 459 // The optimum size here depends on your CPU and IO characteristics, and 460 // the the layout of your zip file. 1000 was chosen mostly at random as 461 // something that worked reasonably well for a test file. 462 // 463 // The RateLimit object will put the upper bounds on the number of 464 // parallel compressions and outstanding buffers. 465 z.writeOps = make(chan chan *zipEntry, 1000) 466 z.cpuRateLimiter = NewCPURateLimiter(int64(parallelJobs)) 467 z.memoryRateLimiter = NewMemoryRateLimiter(0) 468 defer func() { 469 z.cpuRateLimiter.Stop() 470 z.memoryRateLimiter.Stop() 471 }() 472 473 if manifest != "" && !emulateJar { 474 return errors.New("must specify --jar when specifying a manifest via -m") 475 } 476 477 if emulateJar { 478 // manifest may be empty, in which case addManifest will fill in a default 479 pathMappings = append(pathMappings, pathMapping{jar.ManifestFile, manifest, zip.Deflate}) 480 481 jarSort(pathMappings) 482 } 483 484 go func() { 485 var err error 486 defer close(z.writeOps) 487 488 for _, ele := range pathMappings { 489 if emulateJar && ele.dest == jar.ManifestFile { 490 err = z.addManifest(ele.dest, ele.src, ele.zipMethod) 491 } else { 492 err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar) 493 } 494 if err != nil { 495 z.errors <- err 496 return 497 } 498 } 499 }() 500 501 zipw := zip.NewWriter(f) 502 503 var currentWriteOpChan chan *zipEntry 504 var currentWriter io.WriteCloser 505 var currentReaders chan chan io.Reader 506 var currentReader chan io.Reader 507 var done bool 508 509 for !done { 510 var writeOpsChan chan chan *zipEntry 511 var writeOpChan chan *zipEntry 512 var readersChan chan chan io.Reader 513 514 if currentReader != nil { 515 // Only read and process errors 516 } else if currentReaders != nil { 517 readersChan = currentReaders 518 } else if currentWriteOpChan != nil { 519 writeOpChan = currentWriteOpChan 520 } else { 521 writeOpsChan = z.writeOps 522 } 523 524 select { 525 case writeOp, ok := <-writeOpsChan: 526 if !ok { 527 done = true 528 } 529 530 currentWriteOpChan = writeOp 531 532 case op := <-writeOpChan: 533 currentWriteOpChan = nil 534 535 var err error 536 if op.fh.Method == zip.Deflate { 537 currentWriter, err = zipw.CreateCompressedHeader(op.fh) 538 } else { 539 var zw io.Writer 540 541 op.fh.CompressedSize64 = op.fh.UncompressedSize64 542 543 zw, err = zipw.CreateHeaderAndroid(op.fh) 544 currentWriter = nopCloser{zw} 545 } 546 if err != nil { 547 return err 548 } 549 550 currentReaders = op.futureReaders 551 if op.futureReaders == nil { 552 currentWriter.Close() 553 currentWriter = nil 554 } 555 z.memoryRateLimiter.Finish(op.allocatedSize) 556 557 case futureReader, ok := <-readersChan: 558 if !ok { 559 // Done with reading 560 currentWriter.Close() 561 currentWriter = nil 562 currentReaders = nil 563 } 564 565 currentReader = futureReader 566 567 case reader := <-currentReader: 568 _, err := io.Copy(currentWriter, reader) 569 if err != nil { 570 return err 571 } 572 573 currentReader = nil 574 575 case err := <-z.errors: 576 return err 577 } 578 } 579 580 // One last chance to catch an error 581 select { 582 case err := <-z.errors: 583 return err 584 default: 585 zipw.Close() 586 return nil 587 } 588} 589 590// imports (possibly with compression) <src> into the zip at sub-path <dest> 591func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar bool) error { 592 var fileSize int64 593 var executable bool 594 595 var s os.FileInfo 596 var err error 597 if z.followSymlinks { 598 s, err = z.fs.Stat(src) 599 } else { 600 s, err = z.fs.Lstat(src) 601 } 602 603 if err != nil { 604 if os.IsNotExist(err) && z.ignoreMissingFiles { 605 fmt.Fprintln(z.stderr, "warning:", err) 606 return nil 607 } 608 return err 609 } else if s.IsDir() { 610 if z.directories { 611 return z.writeDirectory(dest, src, emulateJar) 612 } 613 return nil 614 } else { 615 if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil { 616 return err 617 } 618 619 if prev, exists := z.createdDirs[dest]; exists { 620 return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src) 621 } 622 if prev, exists := z.createdFiles[dest]; exists { 623 return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src) 624 } 625 626 z.createdFiles[dest] = src 627 628 if s.Mode()&os.ModeSymlink != 0 { 629 return z.writeSymlink(dest, src) 630 } else if !s.Mode().IsRegular() { 631 return fmt.Errorf("%s is not a file, directory, or symlink", src) 632 } 633 634 fileSize = s.Size() 635 executable = s.Mode()&0100 != 0 636 } 637 638 r, err := z.fs.Open(src) 639 if err != nil { 640 return err 641 } 642 643 header := &zip.FileHeader{ 644 Name: dest, 645 Method: method, 646 UncompressedSize64: uint64(fileSize), 647 } 648 649 if executable { 650 header.SetMode(0700) 651 } 652 653 return z.writeFileContents(header, r) 654} 655 656func (z *ZipWriter) addManifest(dest string, src string, method uint16) error { 657 if prev, exists := z.createdDirs[dest]; exists { 658 return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src) 659 } 660 if prev, exists := z.createdFiles[dest]; exists { 661 return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src) 662 } 663 664 if err := z.writeDirectory(filepath.Dir(dest), src, true); err != nil { 665 return err 666 } 667 668 var contents []byte 669 if src != "" { 670 f, err := z.fs.Open(src) 671 if err != nil { 672 return err 673 } 674 675 contents, err = ioutil.ReadAll(f) 676 f.Close() 677 if err != nil { 678 return err 679 } 680 } 681 682 fh, buf, err := jar.ManifestFileContents(contents) 683 if err != nil { 684 return err 685 } 686 687 reader := &byteReaderCloser{bytes.NewReader(buf), ioutil.NopCloser(nil)} 688 689 return z.writeFileContents(fh, reader) 690} 691 692func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r pathtools.ReaderAtSeekerCloser) (err error) { 693 694 header.SetModTime(z.time) 695 696 compressChan := make(chan *zipEntry, 1) 697 z.writeOps <- compressChan 698 699 // Pre-fill a zipEntry, it will be sent in the compressChan once 700 // we're sure about the Method and CRC. 701 ze := &zipEntry{ 702 fh: header, 703 } 704 705 ze.allocatedSize = int64(header.UncompressedSize64) 706 z.cpuRateLimiter.Request() 707 z.memoryRateLimiter.Request(ze.allocatedSize) 708 709 fileSize := int64(header.UncompressedSize64) 710 if fileSize == 0 { 711 fileSize = int64(header.UncompressedSize) 712 } 713 714 if header.Method == zip.Deflate && fileSize >= minParallelFileSize { 715 wg := new(sync.WaitGroup) 716 717 // Allocate enough buffer to hold all readers. We'll limit 718 // this based on actual buffer sizes in RateLimit. 719 ze.futureReaders = make(chan chan io.Reader, (fileSize/parallelBlockSize)+1) 720 721 // Calculate the CRC in the background, since reading the entire 722 // file could take a while. 723 // 724 // We could split this up into chunks as well, but it's faster 725 // than the compression. Due to the Go Zip API, we also need to 726 // know the result before we can begin writing the compressed 727 // data out to the zipfile. 728 wg.Add(1) 729 go z.crcFile(r, ze, compressChan, wg) 730 731 for start := int64(0); start < fileSize; start += parallelBlockSize { 732 sr := io.NewSectionReader(r, start, parallelBlockSize) 733 resultChan := make(chan io.Reader, 1) 734 ze.futureReaders <- resultChan 735 736 z.cpuRateLimiter.Request() 737 738 last := !(start+parallelBlockSize < fileSize) 739 var dict []byte 740 if start >= windowSize { 741 dict, err = ioutil.ReadAll(io.NewSectionReader(r, start-windowSize, windowSize)) 742 if err != nil { 743 return err 744 } 745 } 746 747 wg.Add(1) 748 go z.compressPartialFile(sr, dict, last, resultChan, wg) 749 } 750 751 close(ze.futureReaders) 752 753 // Close the file handle after all readers are done 754 go func(wg *sync.WaitGroup, closer io.Closer) { 755 wg.Wait() 756 closer.Close() 757 }(wg, r) 758 } else { 759 go func() { 760 z.compressWholeFile(ze, r, compressChan) 761 r.Close() 762 }() 763 } 764 765 return nil 766} 767 768func (z *ZipWriter) crcFile(r io.Reader, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) { 769 defer wg.Done() 770 defer z.cpuRateLimiter.Finish() 771 772 crc := crc32.NewIEEE() 773 _, err := io.Copy(crc, r) 774 if err != nil { 775 z.errors <- err 776 return 777 } 778 779 ze.fh.CRC32 = crc.Sum32() 780 resultChan <- ze 781 close(resultChan) 782} 783 784func (z *ZipWriter) compressPartialFile(r io.Reader, dict []byte, last bool, resultChan chan io.Reader, wg *sync.WaitGroup) { 785 defer wg.Done() 786 787 result, err := z.compressBlock(r, dict, last) 788 if err != nil { 789 z.errors <- err 790 return 791 } 792 793 z.cpuRateLimiter.Finish() 794 795 resultChan <- result 796} 797 798func (z *ZipWriter) compressBlock(r io.Reader, dict []byte, last bool) (*bytes.Buffer, error) { 799 buf := new(bytes.Buffer) 800 var fw *flate.Writer 801 var err error 802 if len(dict) > 0 { 803 // There's no way to Reset a Writer with a new dictionary, so 804 // don't use the Pool 805 fw, err = flate.NewWriterDict(buf, z.compLevel, dict) 806 } else { 807 var ok bool 808 if fw, ok = z.compressorPool.Get().(*flate.Writer); ok { 809 fw.Reset(buf) 810 } else { 811 fw, err = flate.NewWriter(buf, z.compLevel) 812 } 813 defer z.compressorPool.Put(fw) 814 } 815 if err != nil { 816 return nil, err 817 } 818 819 _, err = io.Copy(fw, r) 820 if err != nil { 821 return nil, err 822 } 823 if last { 824 fw.Close() 825 } else { 826 fw.Flush() 827 } 828 829 return buf, nil 830} 831 832func (z *ZipWriter) compressWholeFile(ze *zipEntry, r io.ReadSeeker, compressChan chan *zipEntry) { 833 834 crc := crc32.NewIEEE() 835 _, err := io.Copy(crc, r) 836 if err != nil { 837 z.errors <- err 838 return 839 } 840 841 ze.fh.CRC32 = crc.Sum32() 842 843 _, err = r.Seek(0, 0) 844 if err != nil { 845 z.errors <- err 846 return 847 } 848 849 readFile := func(reader io.ReadSeeker) ([]byte, error) { 850 _, err := reader.Seek(0, 0) 851 if err != nil { 852 return nil, err 853 } 854 855 buf, err := ioutil.ReadAll(reader) 856 if err != nil { 857 return nil, err 858 } 859 860 return buf, nil 861 } 862 863 ze.futureReaders = make(chan chan io.Reader, 1) 864 futureReader := make(chan io.Reader, 1) 865 ze.futureReaders <- futureReader 866 close(ze.futureReaders) 867 868 if ze.fh.Method == zip.Deflate { 869 compressed, err := z.compressBlock(r, nil, true) 870 if err != nil { 871 z.errors <- err 872 return 873 } 874 if uint64(compressed.Len()) < ze.fh.UncompressedSize64 { 875 futureReader <- compressed 876 } else { 877 buf, err := readFile(r) 878 if err != nil { 879 z.errors <- err 880 return 881 } 882 ze.fh.Method = zip.Store 883 futureReader <- bytes.NewReader(buf) 884 } 885 } else { 886 buf, err := readFile(r) 887 if err != nil { 888 z.errors <- err 889 return 890 } 891 ze.fh.Method = zip.Store 892 futureReader <- bytes.NewReader(buf) 893 } 894 895 z.cpuRateLimiter.Finish() 896 897 close(futureReader) 898 899 compressChan <- ze 900 close(compressChan) 901} 902 903// writeDirectory annotates that dir is a directory created for the src file or directory, and adds 904// the directory entry to the zip file if directories are enabled. 905func (z *ZipWriter) writeDirectory(dir string, src string, emulateJar bool) error { 906 // clean the input 907 dir = filepath.Clean(dir) 908 909 // discover any uncreated directories in the path 910 zipDirs := []string{} 911 for dir != "" && dir != "." { 912 if _, exists := z.createdDirs[dir]; exists { 913 break 914 } 915 916 if prev, exists := z.createdFiles[dir]; exists { 917 return fmt.Errorf("destination %q is both a directory %q and a file %q", dir, src, prev) 918 } 919 920 z.createdDirs[dir] = src 921 // parent directories precede their children 922 zipDirs = append([]string{dir}, zipDirs...) 923 924 dir = filepath.Dir(dir) 925 } 926 927 if z.directories { 928 // make a directory entry for each uncreated directory 929 for _, cleanDir := range zipDirs { 930 var dirHeader *zip.FileHeader 931 932 if emulateJar && cleanDir+"/" == jar.MetaDir { 933 dirHeader = jar.MetaDirFileHeader() 934 } else { 935 dirHeader = &zip.FileHeader{ 936 Name: cleanDir + "/", 937 } 938 dirHeader.SetMode(0700 | os.ModeDir) 939 } 940 941 dirHeader.SetModTime(z.time) 942 943 ze := make(chan *zipEntry, 1) 944 ze <- &zipEntry{ 945 fh: dirHeader, 946 } 947 close(ze) 948 z.writeOps <- ze 949 } 950 } 951 952 return nil 953} 954 955func (z *ZipWriter) writeSymlink(rel, file string) error { 956 fileHeader := &zip.FileHeader{ 957 Name: rel, 958 } 959 fileHeader.SetModTime(z.time) 960 fileHeader.SetMode(0777 | os.ModeSymlink) 961 962 dest, err := z.fs.Readlink(file) 963 if err != nil { 964 return err 965 } 966 967 fileHeader.UncompressedSize64 = uint64(len(dest)) 968 fileHeader.CRC32 = crc32.ChecksumIEEE([]byte(dest)) 969 970 ze := make(chan *zipEntry, 1) 971 futureReaders := make(chan chan io.Reader, 1) 972 futureReader := make(chan io.Reader, 1) 973 futureReaders <- futureReader 974 close(futureReaders) 975 futureReader <- bytes.NewBufferString(dest) 976 close(futureReader) 977 978 ze <- &zipEntry{ 979 fh: fileHeader, 980 futureReaders: futureReaders, 981 } 982 close(ze) 983 z.writeOps <- ze 984 985 return nil 986} 987