1 // 2 // Copyright (c) 2016-2019 Vinnie Falco (vinnie dot falco at gmail dot com) 3 // 4 // Distributed under the Boost Software License, Version 1.0. (See accompanying 5 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 6 // 7 // Official repository: https://github.com/boostorg/beast 8 // 9 10 // Test that header file is self-contained. 11 #include <boost/beast/zlib/deflate_stream.hpp> 12 13 #include <boost/beast/core/string.hpp> 14 #include <boost/beast/_experimental/unit_test/suite.hpp> 15 #include <array> 16 #include <cstdint> 17 #include <numeric> 18 #include <random> 19 20 #include "zlib-1.2.11/zlib.h" 21 22 namespace boost { 23 namespace beast { 24 namespace zlib { 25 26 class deflate_stream_test : public beast::unit_test::suite 27 { 28 struct ICompressor { 29 virtual void init() = 0; 30 virtual void init( 31 int level, 32 int windowBits, 33 int memLevel, 34 int strategy) = 0; 35 36 virtual std::size_t avail_in() const noexcept = 0; 37 virtual void avail_in(std::size_t) noexcept = 0; 38 virtual void const* next_in() const noexcept = 0; 39 virtual void next_in(const void*) noexcept = 0; 40 virtual std::size_t avail_out() const noexcept = 0; 41 virtual void avail_out(std::size_t) noexcept = 0; 42 virtual void* next_out() const noexcept = 0; 43 virtual void next_out(void*) noexcept = 0; 44 virtual std::size_t total_out() const noexcept = 0; 45 46 virtual std::size_t bound(std::size_t) = 0; 47 virtual error_code write(Flush) = 0; 48 virtual ~ICompressor() = default; 49 }; 50 class ZlibCompressor : public ICompressor { 51 z_stream zs{}; 52 53 public: 54 ZlibCompressor() = default; init()55 void init() override { 56 deflateEnd(&zs); 57 zs = {}; 58 const auto res = deflateInit2(&zs, -1, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); 59 if(res != Z_OK) 60 throw std::invalid_argument{"zlib compressor: failure"}; 61 } init(int level,int windowBits,int memLevel,int strategy)62 void init( 63 int level, 64 int windowBits, 65 int memLevel, 66 int strategy) override 67 { 68 deflateEnd(&zs); 69 zs = {}; 70 const auto res = deflateInit2(&zs, level, Z_DEFLATED, -windowBits, memLevel, strategy); 71 if(res != Z_OK) 72 BOOST_THROW_EXCEPTION(std::invalid_argument{"zlib compressor: bad arg"}); 73 } 74 avail_in() const75 virtual std::size_t avail_in() const noexcept override { return zs.avail_in; } avail_in(std::size_t n)76 virtual void avail_in(std::size_t n) noexcept override { zs.avail_in = n; } next_in() const77 virtual void const* next_in() const noexcept override { return zs.next_in; } next_in(const void * ptr)78 virtual void next_in(const void* ptr) noexcept override { zs.next_in = const_cast<Bytef*>(static_cast<const Bytef*>(ptr)); } avail_out() const79 virtual std::size_t avail_out() const noexcept override { return zs.avail_out; } avail_out(std::size_t n_out)80 virtual void avail_out(std::size_t n_out) noexcept override { zs.avail_out = n_out; } next_out() const81 virtual void* next_out() const noexcept override { return zs.next_out; } next_out(void * ptr)82 virtual void next_out(void* ptr) noexcept override { zs.next_out = (Bytef*)ptr; } total_out() const83 virtual std::size_t total_out() const noexcept override { return zs.total_out; } 84 bound(std::size_t src_size)85 std::size_t bound(std::size_t src_size) override { 86 return deflateBound(&zs, static_cast<uLong>(src_size)); 87 } write(Flush flush)88 error_code write(Flush flush) override { 89 constexpr static int zlib_flushes[] = {0, Z_BLOCK, Z_PARTIAL_FLUSH, Z_SYNC_FLUSH, Z_FULL_FLUSH, Z_FINISH, Z_TREES}; 90 const auto zlib_flush = zlib_flushes[static_cast<int>(flush)]; 91 if(zs.next_in == nullptr && zs.avail_in != 0) 92 BOOST_THROW_EXCEPTION(std::invalid_argument{"zlib compressor: invalid input"}); 93 const auto res = deflate(&zs, zlib_flush); 94 switch(res){ 95 case Z_OK: 96 return {}; 97 case Z_STREAM_END: 98 return error::end_of_stream; 99 case Z_STREAM_ERROR: 100 return error::stream_error; 101 case Z_BUF_ERROR: 102 return error::need_buffers; 103 default: 104 throw; 105 } 106 } 107 ~ZlibCompressor()108 ~ZlibCompressor() override { 109 deflateEnd(&zs); 110 } 111 } zlib_compressor; 112 class BeastCompressor : public ICompressor { 113 z_params zp; 114 deflate_stream ds; 115 116 public: 117 BeastCompressor() = default; init()118 void init() override { 119 zp = {}; 120 ds.clear(); 121 ds.reset(); 122 } init(int level,int windowBits,int memLevel,int strategy)123 void init( 124 int level, 125 int windowBits, 126 int memLevel, 127 int strategy) override 128 { 129 zp = {}; 130 ds.clear(); 131 ds.reset( 132 level, 133 windowBits, 134 memLevel, 135 toStrategy(strategy)); 136 } 137 avail_in() const138 virtual std::size_t avail_in() const noexcept override { return zp.avail_in; } avail_in(std::size_t n)139 virtual void avail_in(std::size_t n) noexcept override { zp.avail_in = n; } next_in() const140 virtual void const* next_in() const noexcept override { return zp.next_in; } next_in(const void * ptr)141 virtual void next_in(const void* ptr) noexcept override { zp.next_in = ptr; } avail_out() const142 virtual std::size_t avail_out() const noexcept override { return zp.avail_out; } avail_out(std::size_t n_out)143 virtual void avail_out(std::size_t n_out) noexcept override { zp.avail_out = n_out; } next_out() const144 virtual void* next_out() const noexcept override { return zp.next_out; } next_out(void * ptr)145 virtual void next_out(void* ptr) noexcept override { zp.next_out = (Bytef*)ptr; } total_out() const146 virtual std::size_t total_out() const noexcept override { return zp.total_out; } 147 bound(std::size_t src_size)148 std::size_t bound(std::size_t src_size) override { 149 return ds.upper_bound(src_size); 150 } write(Flush flush)151 error_code write(Flush flush) override { 152 error_code ec{}; 153 ds.write(zp, flush, ec); 154 return ec; 155 } 156 157 ~BeastCompressor() override = default; 158 } beast_compressor; 159 160 public: 161 // Lots of repeats, limited char range 162 static 163 std::string corpus1(std::size_t n)164 corpus1(std::size_t n) 165 { 166 static std::string const alphabet{ 167 "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" 168 }; 169 std::string s; 170 s.reserve(n + 5); 171 std::mt19937 g; 172 std::uniform_int_distribution<std::size_t> d0{ 173 0, alphabet.size() - 1}; 174 std::uniform_int_distribution<std::size_t> d1{ 175 1, 5}; 176 while(s.size() < n) 177 { 178 auto const rep = d1(g); 179 auto const ch = alphabet[d0(g)]; 180 s.insert(s.end(), rep, ch); 181 } 182 s.resize(n); 183 return s; 184 } 185 186 // Random data 187 static 188 std::string corpus2(std::size_t n)189 corpus2(std::size_t n) 190 { 191 std::string s; 192 s.reserve(n); 193 std::mt19937 g; 194 std::uniform_int_distribution<std::uint32_t> d0{0, 255}; 195 while(n--) 196 s.push_back(static_cast<char>(d0(g))); 197 return s; 198 } 199 200 static 201 std::string compress(string_view const & in,int level,int windowBits,int memLevel)202 compress( 203 string_view const& in, 204 int level, // 0=none, 1..9, -1=default 205 int windowBits, // 9..15 206 int memLevel) // 1..9 (8=default) 207 { 208 int const strategy = Z_DEFAULT_STRATEGY; 209 int result; 210 z_stream zs; 211 memset(&zs, 0, sizeof(zs)); 212 result = deflateInit2( 213 &zs, 214 level, 215 Z_DEFLATED, 216 -windowBits, 217 memLevel, 218 strategy); 219 if(result != Z_OK) 220 throw std::logic_error{"deflateInit2 failed"}; 221 zs.next_in = (Bytef*)in.data(); 222 zs.avail_in = static_cast<uInt>(in.size()); 223 std::string out; 224 out.resize(deflateBound(&zs, 225 static_cast<uLong>(in.size()))); 226 zs.next_in = (Bytef*)in.data(); 227 zs.avail_in = static_cast<uInt>(in.size()); 228 zs.next_out = (Bytef*)&out[0]; 229 zs.avail_out = static_cast<uInt>(out.size()); 230 result = deflate(&zs, Z_FULL_FLUSH); 231 if(result != Z_OK) 232 throw std::logic_error("deflate failed"); 233 out.resize(zs.total_out); 234 deflateEnd(&zs); 235 return out; 236 } 237 238 static 239 std::string decompress(string_view const & in)240 decompress(string_view const& in) 241 { 242 int result; 243 std::string out; 244 z_stream zs; 245 memset(&zs, 0, sizeof(zs)); 246 result = inflateInit2(&zs, -15); 247 if(result != Z_OK) 248 throw std::logic_error{"inflateInit2 failed"}; 249 try 250 { 251 zs.next_in = (Bytef*)in.data(); 252 zs.avail_in = static_cast<uInt>(in.size()); 253 for(;;) 254 { 255 out.resize(zs.total_out + 1024); 256 zs.next_out = (Bytef*)&out[zs.total_out]; 257 zs.avail_out = static_cast<uInt>( 258 out.size() - zs.total_out); 259 result = inflate(&zs, Z_SYNC_FLUSH); 260 if( result == Z_NEED_DICT || 261 result == Z_DATA_ERROR || 262 result == Z_MEM_ERROR) 263 { 264 throw std::logic_error("inflate failed"); 265 } 266 if(zs.avail_out > 0) 267 break; 268 if(result == Z_STREAM_END) 269 break; 270 } 271 out.resize(zs.total_out); 272 inflateEnd(&zs); 273 } 274 catch(...) 275 { 276 inflateEnd(&zs); 277 throw; 278 } 279 return out; 280 } 281 282 //-------------------------------------------------------------------------- 283 284 using self = deflate_stream_test; 285 typedef void(self::*pmf_t)( 286 ICompressor& c, 287 int level, int windowBits, int memLevel, 288 int strategy, std::string const&); 289 290 static 291 Strategy toStrategy(int strategy)292 toStrategy(int strategy) 293 { 294 switch(strategy) 295 { 296 default: 297 case 0: return Strategy::normal; 298 case 1: return Strategy::filtered; 299 case 2: return Strategy::huffman; 300 case 3: return Strategy::rle; 301 case 4: return Strategy::fixed; 302 } 303 } 304 305 void doDeflate1_beast(ICompressor & c,int level,int windowBits,int memLevel,int strategy,std::string const & check)306 doDeflate1_beast( 307 ICompressor& c, 308 int level, int windowBits, int memLevel, 309 int strategy, std::string const& check) 310 { 311 std::string out; 312 c.init( 313 level, 314 windowBits, 315 memLevel, 316 strategy); 317 out.resize(c.bound(check.size())); 318 c.next_in(check.data()); 319 c.avail_in(check.size()); 320 c.next_out((void*)out.data()); 321 c.avail_out(out.size()); 322 { 323 bool progress = true; 324 for(;;) 325 { 326 error_code ec = c.write(Flush::full); 327 if( ec == error::need_buffers || 328 ec == error::end_of_stream) // per zlib FAQ 329 goto fin; 330 if(! BEAST_EXPECTS(! ec, ec.message())) 331 goto err; 332 if(! BEAST_EXPECT(progress)) 333 goto err; 334 progress = false; 335 } 336 } 337 338 fin: 339 out.resize(c.total_out()); 340 BEAST_EXPECT(decompress(out) == check); 341 342 err: 343 ; 344 } 345 346 //-------------------------------------------------------------------------- 347 348 void doDeflate2_beast(ICompressor & c,int level,int windowBits,int memLevel,int strategy,std::string const & check)349 doDeflate2_beast( 350 ICompressor& c, 351 int level, int windowBits, int memLevel, 352 int strategy, std::string const& check) 353 { 354 for(std::size_t i = 1; i < check.size(); ++i) 355 { 356 for(std::size_t j = 1;; ++j) 357 { 358 c.init( 359 level, 360 windowBits, 361 memLevel, 362 strategy); 363 std::string out; 364 out.resize(c.bound(check.size())); 365 if(j >= out.size()) 366 break; 367 c.next_in((void*)check.data()); 368 c.avail_in(i); 369 c.next_out((void*)out.data()); 370 c.avail_out(j); 371 bool bi = false; 372 bool bo = false; 373 for(;;) 374 { 375 error_code ec = c.write( 376 bi ? Flush::full : Flush::none); 377 if( ec == error::need_buffers || 378 ec == error::end_of_stream) // per zlib FAQ 379 goto fin; 380 if(! BEAST_EXPECTS(! ec, ec.message())) 381 goto err; 382 if(c.avail_in() == 0 && ! bi) 383 { 384 bi = true; 385 c.avail_in(check.size() - i); 386 } 387 if(c.avail_out() == 0 && ! bo) 388 { 389 bo = true; 390 c.avail_out(out.size() - j); 391 } 392 } 393 394 fin: 395 out.resize(c.total_out()); 396 BEAST_EXPECT(decompress(out) == check); 397 398 err: 399 ; 400 } 401 } 402 } 403 404 //-------------------------------------------------------------------------- 405 406 void doMatrix(ICompressor & c,std::string const & check,pmf_t pmf)407 doMatrix(ICompressor& c, std::string const& check, pmf_t pmf) 408 { 409 for(int level = 0; level <= 9; ++level) 410 { 411 for(int windowBits = 8; windowBits <= 9; ++windowBits) 412 { 413 // zlib has a bug with windowBits==8 414 if(windowBits == 8) 415 continue; 416 for(int strategy = 0; strategy <= 4; ++strategy) 417 { 418 for (int memLevel = 8; memLevel <= 9; ++memLevel) 419 { 420 (this->*pmf)( 421 c, level, windowBits, memLevel, strategy, check); 422 } 423 } 424 } 425 } 426 427 // Check default settings 428 (this->*pmf)(c, compression::default_size, 15, 8, 0, check); 429 } 430 431 void testDeflate(ICompressor & c)432 testDeflate(ICompressor& c) 433 { 434 doMatrix(c, "Hello, world!", &self::doDeflate1_beast); 435 doMatrix(c, "Hello, world!", &self::doDeflate2_beast); 436 log << "no-silence keepalive" << std::endl; 437 doMatrix(c, corpus1(56), &self::doDeflate2_beast); 438 doMatrix(c, corpus1(1024), &self::doDeflate1_beast); 439 } 440 testInvalidSettings(ICompressor & c)441 void testInvalidSettings(ICompressor& c) 442 { 443 except<std::invalid_argument>( 444 [&]() 445 { 446 c.init(-42, 15, 8, static_cast<int>(Strategy::normal)); 447 }); 448 except<std::invalid_argument>( 449 [&]() 450 { 451 c.init(compression::default_size, -1, 8, static_cast<int>(Strategy::normal)); 452 }); 453 except<std::invalid_argument>( 454 [&]() 455 { 456 c.init(compression::default_size, 15, -1, static_cast<int>(Strategy::normal)); 457 }); 458 except<std::invalid_argument>( 459 [&]() 460 { 461 c.init(); 462 c.avail_in(1); 463 c.next_in(nullptr); 464 c.write(Flush::full); 465 }); 466 } 467 468 void testWriteAfterFinish(ICompressor & c)469 testWriteAfterFinish(ICompressor& c) 470 { 471 c.init(); 472 std::string out; 473 out.resize(1024); 474 string_view s = "Hello"; 475 c.next_in(s.data()); 476 c.avail_in(s.size()); 477 c.next_out(&out.front()); 478 c.avail_out(out.size()); 479 error_code ec = c.write(Flush::sync); 480 BEAST_EXPECT(!ec); 481 c.next_in(nullptr); 482 c.avail_in(0); 483 ec = c.write(Flush::finish); 484 BEAST_EXPECT(ec == error::end_of_stream); 485 c.next_in(s.data()); 486 c.avail_in(s.size()); 487 c.next_out(&out.front()); 488 c.avail_out(out.size()); 489 ec = c.write(Flush::sync); 490 BEAST_EXPECT(ec == error::stream_error); 491 ec = c.write(Flush::finish); 492 BEAST_EXPECT(ec == error::need_buffers); 493 } 494 495 void testFlushPartial(ICompressor & c)496 testFlushPartial(ICompressor& c) 497 { 498 c.init(); 499 std::string out; 500 out.resize(1024); 501 string_view s = "Hello"; 502 c.next_in(s.data()); 503 c.avail_in(s.size()); 504 c.next_out(&out.front()); 505 c.avail_out(out.size()); 506 error_code ec; 507 ec = c.write(Flush::none); 508 BEAST_EXPECT(!ec); 509 ec = c.write(Flush::partial); 510 BEAST_EXPECT(!ec); 511 } 512 513 void testFlushAtLiteralBufferFull(ICompressor & c)514 testFlushAtLiteralBufferFull(ICompressor& c) 515 { 516 struct fixture 517 { 518 ICompressor& c; 519 explicit fixture(ICompressor&c, std::size_t n, Strategy s) : c(c) 520 { 521 c.init(8, 15, 1, static_cast<int>(s)); 522 std::iota(in.begin(), in.end(), std::uint8_t{0}); 523 out.resize(n); 524 c.next_in(in.data()); 525 c.avail_in(in.size()); 526 c.next_out(&out.front()); 527 c.avail_out(out.size()); 528 } 529 530 std::array<std::uint8_t, 255> in; 531 std::string out; 532 }; 533 534 for (auto s : {Strategy::huffman, Strategy::rle, Strategy::normal}) 535 { 536 { 537 fixture f{c, 264, s}; 538 error_code ec = c.write(Flush::finish); 539 BEAST_EXPECT(ec == error::end_of_stream); 540 BEAST_EXPECT(c.avail_out() == 1); 541 } 542 543 { 544 fixture f{c,263, s}; 545 error_code ec = c.write(Flush::finish); 546 BEAST_EXPECT(!ec); 547 BEAST_EXPECT(c.avail_out() == 0); 548 } 549 550 { 551 fixture f{c, 20, s}; 552 error_code ec = c.write(Flush::sync); 553 BEAST_EXPECT(!ec); 554 } 555 556 } 557 } 558 559 void testRLEMatchLengthExceedLookahead(ICompressor & c)560 testRLEMatchLengthExceedLookahead(ICompressor& c) 561 { 562 std::vector<std::uint8_t> in; 563 in.resize(300); 564 565 c.init(8, 15, 1, static_cast<int>(Strategy::rle)); 566 std::fill_n(in.begin(), 4, 'a'); 567 std::string out; 568 out.resize(in.size() * 2); 569 c.next_in(in.data()); 570 c.avail_in(in.size()); 571 c.next_out(&out.front()); 572 c.avail_out(out.size()); 573 574 error_code ec; 575 ec = c.write(Flush::sync); 576 BEAST_EXPECT(!ec); 577 } 578 579 void testFlushAfterDistMatch(ICompressor & c)580 testFlushAfterDistMatch(ICompressor& c) 581 { 582 for (auto out_size : {144, 129}) 583 { 584 std::array<std::uint8_t, 256> in{}; 585 // 125 will mostly fill the lit buffer, so emitting a distance code 586 // results in a flush. 587 auto constexpr n = 125; 588 std::iota(in.begin(), in.begin() + n, 589 static_cast<std::uint8_t>(0)); 590 std::iota(in.begin() + n, in.end(), 591 static_cast<std::uint8_t>(0)); 592 593 c.init(8, 15, 1, static_cast<int>(Strategy::normal)); 594 std::string out; 595 out.resize(out_size); 596 c.next_in(in.data()); 597 c.avail_in(in.size()); 598 c.next_out(&out.front()); 599 c.avail_out(out.size()); 600 601 error_code ec; 602 ec = c.write(Flush::sync); 603 BEAST_EXPECT(!ec); 604 } 605 } 606 607 void run()608 run() override 609 { 610 log << 611 "sizeof(deflate_stream) == " << 612 sizeof(deflate_stream) << std::endl; 613 614 testDeflate(zlib_compressor); 615 testDeflate(beast_compressor); 616 testInvalidSettings(zlib_compressor); 617 testInvalidSettings(beast_compressor); 618 testWriteAfterFinish(zlib_compressor); 619 testWriteAfterFinish(beast_compressor); 620 testFlushPartial(zlib_compressor); 621 testFlushPartial(beast_compressor); 622 testFlushAtLiteralBufferFull(zlib_compressor); 623 testFlushAtLiteralBufferFull(beast_compressor); 624 testRLEMatchLengthExceedLookahead(zlib_compressor); 625 testRLEMatchLengthExceedLookahead(beast_compressor); 626 testFlushAfterDistMatch(zlib_compressor); 627 testFlushAfterDistMatch(beast_compressor); 628 } 629 }; 630 631 BEAST_DEFINE_TESTSUITE(beast,zlib,deflate_stream); 632 633 } // zlib 634 } // beast 635 } // boost 636