1// Copyright 2018 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package zip 16 17import ( 18 "bytes" 19 "encoding/hex" 20 "hash/crc32" 21 "io" 22 "os" 23 "reflect" 24 "syscall" 25 "testing" 26 27 "android/soong/third_party/zip" 28 29 "github.com/google/blueprint/pathtools" 30) 31 32var ( 33 fileA = []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") 34 fileB = []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB") 35 fileC = []byte("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC") 36 fileEmpty = []byte("") 37 fileManifest = []byte("Manifest-Version: 1.0\nCreated-By: soong_zip\n\n") 38 39 sha256FileA = "d53eda7a637c99cc7fb566d96e9fa109bf15c478410a3f5eb4d4c4e26cd081f6" 40 sha256FileB = "430c56c5818e62bcb6d478901ef86284e97714c138f3c86aa14fd6a84b7ce5d3" 41 sha256FileC = "31c5ab6111f1d6aa13c2c4e92bb3c0f7c76b61b42d141af1e846eb7f6586a51c" 42 43 fileCustomManifest = []byte("Custom manifest: true\n") 44 customManifestAfter = []byte("Manifest-Version: 1.0\nCreated-By: soong_zip\nCustom manifest: true\n\n") 45) 46 47var mockFs = pathtools.MockFs(map[string][]byte{ 48 "a/a/a": fileA, 49 "a/a/b": fileB, 50 "a/a/c -> ../../c": nil, 51 "dangling -> missing": nil, 52 "a/a/d -> b": nil, 53 "c": fileC, 54 "d/a/a": nil, 55 "l_nl": []byte("a/a/a\na/a/b\nc\n\\[\n"), 56 "l_sp": []byte("a/a/a a/a/b c \\["), 57 "l2": []byte("missing\n"), 58 "rsp": []byte("'a/a/a'\na/a/b\n'@'\n'foo'\\''bar'\n'['"), 59 "@ -> c": nil, 60 "foo'bar -> c": nil, 61 "manifest.txt": fileCustomManifest, 62 "[": fileEmpty, 63}) 64 65func fh(name string, contents []byte, method uint16) zip.FileHeader { 66 return zip.FileHeader{ 67 Name: name, 68 Method: method, 69 CRC32: crc32.ChecksumIEEE(contents), 70 UncompressedSize64: uint64(len(contents)), 71 ExternalAttrs: (syscall.S_IFREG | 0644) << 16, 72 } 73} 74 75func fhWithSHA256(name string, contents []byte, method uint16, sha256 string) zip.FileHeader { 76 h := fh(name, contents, method) 77 // The extra field contains 38 bytes, including 2 bytes of header ID, 2 bytes 78 // of size, 2 bytes of signature, and 32 bytes of checksum data block. 79 var extra [38]byte 80 // The first 6 bytes contains Sha256HeaderID (0x4967), size (unit(34)) and 81 // Sha256HeaderSignature (0x9514) 82 copy(extra[0:], []byte{103, 73, 34, 0, 20, 149}) 83 sha256Bytes, _ := hex.DecodeString(sha256) 84 copy(extra[6:], sha256Bytes) 85 h.Extra = append(h.Extra, extra[:]...) 86 return h 87} 88 89func fhManifest(contents []byte) zip.FileHeader { 90 return zip.FileHeader{ 91 Name: "META-INF/MANIFEST.MF", 92 Method: zip.Store, 93 CRC32: crc32.ChecksumIEEE(contents), 94 UncompressedSize64: uint64(len(contents)), 95 ExternalAttrs: (syscall.S_IFREG | 0644) << 16, 96 } 97} 98 99func fhLink(name string, to string) zip.FileHeader { 100 return zip.FileHeader{ 101 Name: name, 102 Method: zip.Store, 103 CRC32: crc32.ChecksumIEEE([]byte(to)), 104 UncompressedSize64: uint64(len(to)), 105 ExternalAttrs: (syscall.S_IFLNK | 0777) << 16, 106 } 107} 108 109type fhDirOptions struct { 110 extra []byte 111} 112 113func fhDir(name string, opts fhDirOptions) zip.FileHeader { 114 return zip.FileHeader{ 115 Name: name, 116 Method: zip.Store, 117 CRC32: crc32.ChecksumIEEE(nil), 118 UncompressedSize64: 0, 119 ExternalAttrs: (syscall.S_IFDIR|0755)<<16 | 0x10, 120 Extra: opts.extra, 121 } 122} 123 124func fileArgsBuilder() *FileArgsBuilder { 125 return &FileArgsBuilder{ 126 fs: mockFs, 127 } 128} 129 130func TestZip(t *testing.T) { 131 testCases := []struct { 132 name string 133 args *FileArgsBuilder 134 compressionLevel int 135 emulateJar bool 136 nonDeflatedFiles map[string]bool 137 dirEntries bool 138 manifest string 139 storeSymlinks bool 140 ignoreMissingFiles bool 141 sha256Checksum bool 142 143 files []zip.FileHeader 144 err error 145 }{ 146 { 147 name: "empty args", 148 args: fileArgsBuilder(), 149 150 files: []zip.FileHeader{}, 151 }, 152 { 153 name: "files", 154 args: fileArgsBuilder(). 155 File("a/a/a"). 156 File("a/a/b"). 157 File("c"). 158 File(`\[`), 159 compressionLevel: 9, 160 161 files: []zip.FileHeader{ 162 fh("a/a/a", fileA, zip.Deflate), 163 fh("a/a/b", fileB, zip.Deflate), 164 fh("c", fileC, zip.Deflate), 165 fh("[", fileEmpty, zip.Store), 166 }, 167 }, 168 { 169 name: "files glob", 170 args: fileArgsBuilder(). 171 SourcePrefixToStrip("a"). 172 File("a/**/*"), 173 compressionLevel: 9, 174 storeSymlinks: true, 175 176 files: []zip.FileHeader{ 177 fh("a/a", fileA, zip.Deflate), 178 fh("a/b", fileB, zip.Deflate), 179 fhLink("a/c", "../../c"), 180 fhLink("a/d", "b"), 181 }, 182 }, 183 { 184 name: "dir", 185 args: fileArgsBuilder(). 186 SourcePrefixToStrip("a"). 187 Dir("a"), 188 compressionLevel: 9, 189 storeSymlinks: true, 190 191 files: []zip.FileHeader{ 192 fh("a/a", fileA, zip.Deflate), 193 fh("a/b", fileB, zip.Deflate), 194 fhLink("a/c", "../../c"), 195 fhLink("a/d", "b"), 196 }, 197 }, 198 { 199 name: "stored files", 200 args: fileArgsBuilder(). 201 File("a/a/a"). 202 File("a/a/b"). 203 File("c"), 204 compressionLevel: 0, 205 206 files: []zip.FileHeader{ 207 fh("a/a/a", fileA, zip.Store), 208 fh("a/a/b", fileB, zip.Store), 209 fh("c", fileC, zip.Store), 210 }, 211 }, 212 { 213 name: "symlinks in zip", 214 args: fileArgsBuilder(). 215 File("a/a/a"). 216 File("a/a/b"). 217 File("a/a/c"). 218 File("a/a/d"), 219 compressionLevel: 9, 220 storeSymlinks: true, 221 222 files: []zip.FileHeader{ 223 fh("a/a/a", fileA, zip.Deflate), 224 fh("a/a/b", fileB, zip.Deflate), 225 fhLink("a/a/c", "../../c"), 226 fhLink("a/a/d", "b"), 227 }, 228 }, 229 { 230 name: "follow symlinks", 231 args: fileArgsBuilder(). 232 File("a/a/a"). 233 File("a/a/b"). 234 File("a/a/c"). 235 File("a/a/d"), 236 compressionLevel: 9, 237 storeSymlinks: false, 238 239 files: []zip.FileHeader{ 240 fh("a/a/a", fileA, zip.Deflate), 241 fh("a/a/b", fileB, zip.Deflate), 242 fh("a/a/c", fileC, zip.Deflate), 243 fh("a/a/d", fileB, zip.Deflate), 244 }, 245 }, 246 { 247 name: "dangling symlinks", 248 args: fileArgsBuilder(). 249 File("dangling"), 250 compressionLevel: 9, 251 storeSymlinks: true, 252 253 files: []zip.FileHeader{ 254 fhLink("dangling", "missing"), 255 }, 256 }, 257 { 258 name: "list", 259 args: fileArgsBuilder(). 260 List("l_nl"), 261 compressionLevel: 9, 262 263 files: []zip.FileHeader{ 264 fh("a/a/a", fileA, zip.Deflate), 265 fh("a/a/b", fileB, zip.Deflate), 266 fh("c", fileC, zip.Deflate), 267 fh("[", fileEmpty, zip.Store), 268 }, 269 }, 270 { 271 name: "list", 272 args: fileArgsBuilder(). 273 List("l_sp"), 274 compressionLevel: 9, 275 276 files: []zip.FileHeader{ 277 fh("a/a/a", fileA, zip.Deflate), 278 fh("a/a/b", fileB, zip.Deflate), 279 fh("c", fileC, zip.Deflate), 280 fh("[", fileEmpty, zip.Store), 281 }, 282 }, 283 { 284 name: "rsp", 285 args: fileArgsBuilder(). 286 RspFile("rsp"), 287 compressionLevel: 9, 288 289 files: []zip.FileHeader{ 290 fh("a/a/a", fileA, zip.Deflate), 291 fh("a/a/b", fileB, zip.Deflate), 292 fh("@", fileC, zip.Deflate), 293 fh("foo'bar", fileC, zip.Deflate), 294 fh("[", fileEmpty, zip.Store), 295 }, 296 }, 297 { 298 name: "prefix in zip", 299 args: fileArgsBuilder(). 300 PathPrefixInZip("foo"). 301 File("a/a/a"). 302 File("a/a/b"). 303 File("c"), 304 compressionLevel: 9, 305 306 files: []zip.FileHeader{ 307 fh("foo/a/a/a", fileA, zip.Deflate), 308 fh("foo/a/a/b", fileB, zip.Deflate), 309 fh("foo/c", fileC, zip.Deflate), 310 }, 311 }, 312 { 313 name: "relative root", 314 args: fileArgsBuilder(). 315 SourcePrefixToStrip("a"). 316 File("a/a/a"). 317 File("a/a/b"), 318 compressionLevel: 9, 319 320 files: []zip.FileHeader{ 321 fh("a/a", fileA, zip.Deflate), 322 fh("a/b", fileB, zip.Deflate), 323 }, 324 }, 325 { 326 name: "multiple relative root", 327 args: fileArgsBuilder(). 328 SourcePrefixToStrip("a"). 329 File("a/a/a"). 330 SourcePrefixToStrip("a/a"). 331 File("a/a/b"), 332 compressionLevel: 9, 333 334 files: []zip.FileHeader{ 335 fh("a/a", fileA, zip.Deflate), 336 fh("b", fileB, zip.Deflate), 337 }, 338 }, 339 { 340 name: "emulate jar", 341 args: fileArgsBuilder(). 342 File("a/a/a"). 343 File("a/a/b"), 344 compressionLevel: 9, 345 emulateJar: true, 346 347 files: []zip.FileHeader{ 348 fhDir("META-INF/", fhDirOptions{extra: []byte{254, 202, 0, 0}}), 349 fhManifest(fileManifest), 350 fhDir("a/", fhDirOptions{}), 351 fhDir("a/a/", fhDirOptions{}), 352 fh("a/a/a", fileA, zip.Deflate), 353 fh("a/a/b", fileB, zip.Deflate), 354 }, 355 }, 356 { 357 name: "emulate jar with manifest", 358 args: fileArgsBuilder(). 359 File("a/a/a"). 360 File("a/a/b"), 361 compressionLevel: 9, 362 emulateJar: true, 363 manifest: "manifest.txt", 364 365 files: []zip.FileHeader{ 366 fhDir("META-INF/", fhDirOptions{extra: []byte{254, 202, 0, 0}}), 367 fhManifest(customManifestAfter), 368 fhDir("a/", fhDirOptions{}), 369 fhDir("a/a/", fhDirOptions{}), 370 fh("a/a/a", fileA, zip.Deflate), 371 fh("a/a/b", fileB, zip.Deflate), 372 }, 373 }, 374 { 375 name: "dir entries", 376 args: fileArgsBuilder(). 377 File("a/a/a"). 378 File("a/a/b"), 379 compressionLevel: 9, 380 dirEntries: true, 381 382 files: []zip.FileHeader{ 383 fhDir("a/", fhDirOptions{}), 384 fhDir("a/a/", fhDirOptions{}), 385 fh("a/a/a", fileA, zip.Deflate), 386 fh("a/a/b", fileB, zip.Deflate), 387 }, 388 }, 389 { 390 name: "junk paths", 391 args: fileArgsBuilder(). 392 JunkPaths(true). 393 File("a/a/a"). 394 File("a/a/b"), 395 compressionLevel: 9, 396 397 files: []zip.FileHeader{ 398 fh("a", fileA, zip.Deflate), 399 fh("b", fileB, zip.Deflate), 400 }, 401 }, 402 { 403 name: "non deflated files", 404 args: fileArgsBuilder(). 405 File("a/a/a"). 406 File("a/a/b"), 407 compressionLevel: 9, 408 nonDeflatedFiles: map[string]bool{"a/a/a": true}, 409 410 files: []zip.FileHeader{ 411 fh("a/a/a", fileA, zip.Store), 412 fh("a/a/b", fileB, zip.Deflate), 413 }, 414 }, 415 { 416 name: "ignore missing files", 417 args: fileArgsBuilder(). 418 File("a/a/a"). 419 File("a/a/b"). 420 File("missing"), 421 compressionLevel: 9, 422 ignoreMissingFiles: true, 423 424 files: []zip.FileHeader{ 425 fh("a/a/a", fileA, zip.Deflate), 426 fh("a/a/b", fileB, zip.Deflate), 427 }, 428 }, 429 { 430 name: "duplicate sources", 431 args: fileArgsBuilder(). 432 File("a/a/a"). 433 File("a/a/a"), 434 compressionLevel: 9, 435 436 files: []zip.FileHeader{ 437 fh("a/a/a", fileA, zip.Deflate), 438 }, 439 }, 440 { 441 name: "generate SHA256 checksum", 442 args: fileArgsBuilder(). 443 File("a/a/a"). 444 File("a/a/b"). 445 File("a/a/c"). 446 File("c"), 447 compressionLevel: 9, 448 sha256Checksum: true, 449 450 files: []zip.FileHeader{ 451 fhWithSHA256("a/a/a", fileA, zip.Deflate, sha256FileA), 452 fhWithSHA256("a/a/b", fileB, zip.Deflate, sha256FileB), 453 fhWithSHA256("a/a/c", fileC, zip.Deflate, sha256FileC), 454 fhWithSHA256("c", fileC, zip.Deflate, sha256FileC), 455 }, 456 }, 457 458 // errors 459 { 460 name: "error missing file", 461 args: fileArgsBuilder(). 462 File("missing"), 463 err: os.ErrNotExist, 464 }, 465 { 466 name: "error missing dir", 467 args: fileArgsBuilder(). 468 Dir("missing"), 469 err: os.ErrNotExist, 470 }, 471 { 472 name: "error missing file in list", 473 args: fileArgsBuilder(). 474 List("l2"), 475 err: os.ErrNotExist, 476 }, 477 { 478 name: "error incorrect relative root", 479 args: fileArgsBuilder(). 480 SourcePrefixToStrip("b"). 481 File("a/a/a"), 482 err: IncorrectRelativeRootError{}, 483 }, 484 { 485 name: "error conflicting file", 486 args: fileArgsBuilder(). 487 SourcePrefixToStrip("a"). 488 File("a/a/a"). 489 SourcePrefixToStrip("d"). 490 File("d/a/a"), 491 err: ConflictingFileError{}, 492 }, 493 } 494 495 for _, test := range testCases { 496 t.Run(test.name, func(t *testing.T) { 497 if test.args.Error() != nil { 498 t.Fatal(test.args.Error()) 499 } 500 501 args := ZipArgs{} 502 args.FileArgs = test.args.FileArgs() 503 args.CompressionLevel = test.compressionLevel 504 args.EmulateJar = test.emulateJar 505 args.AddDirectoryEntriesToZip = test.dirEntries 506 args.NonDeflatedFiles = test.nonDeflatedFiles 507 args.ManifestSourcePath = test.manifest 508 args.StoreSymlinks = test.storeSymlinks 509 args.IgnoreMissingFiles = test.ignoreMissingFiles 510 args.Sha256Checksum = test.sha256Checksum 511 args.Filesystem = mockFs 512 args.Stderr = &bytes.Buffer{} 513 514 buf := &bytes.Buffer{} 515 err := zipTo(args, buf) 516 517 if (err != nil) != (test.err != nil) { 518 t.Fatalf("want error %v, got %v", test.err, err) 519 } else if test.err != nil { 520 if os.IsNotExist(test.err) { 521 if !os.IsNotExist(err) { 522 t.Fatalf("want error %v, got %v", test.err, err) 523 } 524 } else if _, wantRelativeRootErr := test.err.(IncorrectRelativeRootError); wantRelativeRootErr { 525 if _, gotRelativeRootErr := err.(IncorrectRelativeRootError); !gotRelativeRootErr { 526 t.Fatalf("want error %v, got %v", test.err, err) 527 } 528 } else if _, wantConflictingFileError := test.err.(ConflictingFileError); wantConflictingFileError { 529 if _, gotConflictingFileError := err.(ConflictingFileError); !gotConflictingFileError { 530 t.Fatalf("want error %v, got %v", test.err, err) 531 } 532 } else { 533 t.Fatalf("want error %v, got %v", test.err, err) 534 } 535 return 536 } 537 538 br := bytes.NewReader(buf.Bytes()) 539 zr, err := zip.NewReader(br, int64(br.Len())) 540 if err != nil { 541 t.Fatal(err) 542 } 543 544 var files []zip.FileHeader 545 for _, f := range zr.File { 546 r, err := f.Open() 547 if err != nil { 548 t.Fatalf("error when opening %s: %s", f.Name, err) 549 } 550 551 crc := crc32.NewIEEE() 552 len, err := io.Copy(crc, r) 553 r.Close() 554 if err != nil { 555 t.Fatalf("error when reading %s: %s", f.Name, err) 556 } 557 558 if uint64(len) != f.UncompressedSize64 { 559 t.Errorf("incorrect length for %s, want %d got %d", f.Name, f.UncompressedSize64, len) 560 } 561 562 if crc.Sum32() != f.CRC32 { 563 t.Errorf("incorrect crc for %s, want %x got %x", f.Name, f.CRC32, crc) 564 } 565 566 files = append(files, f.FileHeader) 567 } 568 569 if len(files) != len(test.files) { 570 t.Fatalf("want %d files, got %d", len(test.files), len(files)) 571 } 572 573 for i := range files { 574 want := test.files[i] 575 got := files[i] 576 577 if want.Name != got.Name { 578 t.Errorf("incorrect file %d want %q got %q", i, want.Name, got.Name) 579 continue 580 } 581 582 if want.UncompressedSize64 != got.UncompressedSize64 { 583 t.Errorf("incorrect file %s length want %v got %v", want.Name, 584 want.UncompressedSize64, got.UncompressedSize64) 585 } 586 587 if want.ExternalAttrs != got.ExternalAttrs { 588 t.Errorf("incorrect file %s attrs want %x got %x", want.Name, 589 want.ExternalAttrs, got.ExternalAttrs) 590 } 591 592 if want.CRC32 != got.CRC32 { 593 t.Errorf("incorrect file %s crc want %v got %v", want.Name, 594 want.CRC32, got.CRC32) 595 } 596 597 if want.Method != got.Method { 598 t.Errorf("incorrect file %s method want %v got %v", want.Name, 599 want.Method, got.Method) 600 } 601 602 if !bytes.Equal(want.Extra, got.Extra) { 603 t.Errorf("incorrect file %s extra want %v got %v", want.Name, 604 want.Extra, got.Extra) 605 } 606 } 607 }) 608 } 609} 610 611func TestSrcJar(t *testing.T) { 612 mockFs := pathtools.MockFs(map[string][]byte{ 613 "wrong_package.java": []byte("package foo;"), 614 "foo/correct_package.java": []byte("package foo;"), 615 "src/no_package.java": nil, 616 "src2/parse_error.java": []byte("error"), 617 }) 618 619 want := []string{ 620 "foo/", 621 "foo/wrong_package.java", 622 "foo/correct_package.java", 623 "no_package.java", 624 "src2/", 625 "src2/parse_error.java", 626 } 627 628 args := ZipArgs{} 629 args.FileArgs = NewFileArgsBuilder().File("**/*.java").FileArgs() 630 631 args.SrcJar = true 632 args.AddDirectoryEntriesToZip = true 633 args.Filesystem = mockFs 634 args.Stderr = &bytes.Buffer{} 635 636 buf := &bytes.Buffer{} 637 err := zipTo(args, buf) 638 if err != nil { 639 t.Fatalf("got error %v", err) 640 } 641 642 br := bytes.NewReader(buf.Bytes()) 643 zr, err := zip.NewReader(br, int64(br.Len())) 644 if err != nil { 645 t.Fatal(err) 646 } 647 648 var got []string 649 for _, f := range zr.File { 650 r, err := f.Open() 651 if err != nil { 652 t.Fatalf("error when opening %s: %s", f.Name, err) 653 } 654 655 crc := crc32.NewIEEE() 656 len, err := io.Copy(crc, r) 657 r.Close() 658 if err != nil { 659 t.Fatalf("error when reading %s: %s", f.Name, err) 660 } 661 662 if uint64(len) != f.UncompressedSize64 { 663 t.Errorf("incorrect length for %s, want %d got %d", f.Name, f.UncompressedSize64, len) 664 } 665 666 if crc.Sum32() != f.CRC32 { 667 t.Errorf("incorrect crc for %s, want %x got %x", f.Name, f.CRC32, crc) 668 } 669 670 got = append(got, f.Name) 671 } 672 673 if !reflect.DeepEqual(want, got) { 674 t.Errorf("want files %q, got %q", want, got) 675 } 676} 677