libs/beast/test/beast/zlib/deflate_stream.cpp

   1 //
   2 // Copyright (c) 2016-2019 Vinnie Falco (vinnie dot falco at gmail dot com)
   3 //
   4 // Distributed under the Boost Software License, Version 1.0. (See accompanying
   5 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
   6 //
   7 // Official repository: https://github.com/boostorg/beast
   8 //
   9
  10 // Test that header file is self-contained.
  11 #include <boost/beast/zlib/deflate_stream.hpp>
  12
  13 #include <boost/beast/core/string.hpp>
  14 #include <boost/beast/_experimental/unit_test/suite.hpp>
  15 #include <array>
  16 #include <cstdint>
  17 #include <numeric>
  18 #include <random>
  19
  20 #include "zlib-1.2.11/zlib.h"
  21
  22 namespace boost {
  23 namespace beast {
  24 namespace zlib {
  25
  26 class deflate_stream_test : public beast::unit_test::suite
  27 {
  28     struct ICompressor {
  29         virtual void init() = 0;
  30         virtual void init(
  31             int level,
  32             int windowBits,
  33             int memLevel,
  34             int strategy) = 0;
  35
  36         virtual std::size_t avail_in() const noexcept = 0;
  37         virtual void avail_in(std::size_t) noexcept = 0;
  38         virtual void const* next_in() const noexcept = 0;
  39         virtual void next_in(const void*) noexcept = 0;
  40         virtual std::size_t avail_out() const noexcept = 0;
  41         virtual void avail_out(std::size_t) noexcept = 0;
  42         virtual void* next_out() const noexcept = 0;
  43         virtual void next_out(void*) noexcept = 0;
  44         virtual std::size_t total_out() const noexcept = 0;
  45
  46         virtual std::size_t bound(std::size_t) = 0;
  47         virtual error_code write(Flush) = 0;
  48         virtual ~ICompressor() = default;
  49     };
  50     class ZlibCompressor : public ICompressor {
  51         z_stream zs{};
  52
  53     public:
  54         ZlibCompressor() = default;
  55         void init() override {
  56             deflateEnd(&zs);
  57             zs = {};
  58             const auto res = deflateInit2(&zs, -1, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
  59             if(res != Z_OK)
  60                throw std::invalid_argument{"zlib compressor: failure"};
  61         }
  62         void init(
  63             int level,
  64             int windowBits,
  65             int memLevel,
  66             int strategy) override
  67         {
  68             deflateEnd(&zs);
  69             zs = {};
  70             const auto res = deflateInit2(&zs, level, Z_DEFLATED, -windowBits, memLevel, strategy);
  71             if(res != Z_OK)
  72                BOOST_THROW_EXCEPTION(std::invalid_argument{"zlib compressor: bad arg"});
  73         }
  74
  75         virtual std::size_t avail_in() const noexcept override  { return zs.avail_in; }
  76         virtual void avail_in(std::size_t n) noexcept override { zs.avail_in = n; }
  77         virtual void const* next_in() const noexcept override { return zs.next_in; }
  78         virtual void next_in(const void* ptr) noexcept override { zs.next_in = const_cast<Bytef*>(static_cast<const Bytef*>(ptr)); }
  79         virtual std::size_t avail_out() const noexcept override { return zs.avail_out; }
  80         virtual void avail_out(std::size_t n_out) noexcept override { zs.avail_out = n_out; }
  81         virtual void* next_out() const noexcept override { return zs.next_out; }
  82         virtual void next_out(void* ptr) noexcept override { zs.next_out = (Bytef*)ptr; }
  83         virtual std::size_t total_out() const noexcept override { return zs.total_out; }
  84
  85         std::size_t bound(std::size_t src_size) override {
  86            return deflateBound(&zs, static_cast<uLong>(src_size));
  87         }
  88         error_code write(Flush flush) override {
  89             constexpr static int zlib_flushes[] = {0, Z_BLOCK, Z_PARTIAL_FLUSH, Z_SYNC_FLUSH, Z_FULL_FLUSH, Z_FINISH, Z_TREES};
  90             const auto zlib_flush = zlib_flushes[static_cast<int>(flush)];
  91             if(zs.next_in == nullptr && zs.avail_in != 0)
  92               BOOST_THROW_EXCEPTION(std::invalid_argument{"zlib compressor: invalid input"});
  93             const auto res = deflate(&zs, zlib_flush);
  94             switch(res){
  95             case Z_OK:
  96                 return {};
  97             case Z_STREAM_END:
  98                 return error::end_of_stream;
  99             case Z_STREAM_ERROR:
 100                 return error::stream_error;
 101             case Z_BUF_ERROR:
 102                 return error::need_buffers;
 103             default:
 104                 throw;
 105             }
 106         }
 107
 108         ~ZlibCompressor() override {
 109             deflateEnd(&zs);
 110         }
 111     } zlib_compressor;
 112     class BeastCompressor : public ICompressor {
 113         z_params zp;
 114         deflate_stream ds;
 115
 116     public:
 117         BeastCompressor() = default;
 118         void init() override {
 119           zp = {};
 120           ds.clear();
 121           ds.reset();
 122         }
 123         void init(
 124             int level,
 125             int windowBits,
 126             int memLevel,
 127             int strategy) override
 128         {
 129             zp = {};
 130             ds.clear();
 131             ds.reset(
 132                 level,
 133                 windowBits,
 134                 memLevel,
 135                 toStrategy(strategy));
 136         }
 137
 138         virtual std::size_t avail_in() const noexcept override  { return zp.avail_in; }
 139         virtual void avail_in(std::size_t n) noexcept override { zp.avail_in = n; }
 140         virtual void const* next_in() const noexcept override { return zp.next_in; }
 141         virtual void next_in(const void* ptr) noexcept override { zp.next_in = ptr; }
 142         virtual std::size_t avail_out() const noexcept override { return zp.avail_out; }
 143         virtual void avail_out(std::size_t n_out) noexcept override { zp.avail_out = n_out; }
 144         virtual void* next_out() const noexcept override { return zp.next_out; }
 145         virtual void next_out(void* ptr) noexcept override { zp.next_out = (Bytef*)ptr; }
 146         virtual std::size_t total_out() const noexcept override { return zp.total_out; }
 147
 148         std::size_t bound(std::size_t src_size) override {
 149             return ds.upper_bound(src_size);
 150         }
 151         error_code write(Flush flush) override {
 152           error_code ec{};
 153           ds.write(zp, flush, ec);
 154           return ec;
 155         }
 156
 157         ~BeastCompressor() override = default;
 158     } beast_compressor;
 159
 160 public:
 161     // Lots of repeats, limited char range
 162     static
 163     std::string
 164     corpus1(std::size_t n)
 165     {
 166         static std::string const alphabet{
 167             "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
 168         };
 169         std::string s;
 170         s.reserve(n + 5);
 171         std::mt19937 g;
 172         std::uniform_int_distribution<std::size_t> d0{
 173             0, alphabet.size() - 1};
 174         std::uniform_int_distribution<std::size_t> d1{
 175             1, 5};
 176         while(s.size() < n)
 177         {
 178             auto const rep = d1(g);
 179             auto const ch = alphabet[d0(g)];
 180             s.insert(s.end(), rep, ch);
 181         }
 182         s.resize(n);
 183         return s;
 184     }
 185
 186     // Random data
 187     static
 188     std::string
 189     corpus2(std::size_t n)
 190     {
 191         std::string s;
 192         s.reserve(n);
 193         std::mt19937 g;
 194         std::uniform_int_distribution<std::uint32_t> d0{0, 255};
 195         while(n--)
 196             s.push_back(static_cast<char>(d0(g)));
 197         return s;
 198     }
 199
 200     static
 201     std::string
 202     compress(
 203         string_view const& in,
 204         int level,                  // 0=none, 1..9, -1=default
 205         int windowBits,             // 9..15
 206         int memLevel)               // 1..9 (8=default)
 207     {
 208         int const strategy = Z_DEFAULT_STRATEGY;
 209         int result;
 210         z_stream zs;
 211         memset(&zs, 0, sizeof(zs));
 212         result = deflateInit2(
 213             &zs,
 214             level,
 215             Z_DEFLATED,
 216             -windowBits,
 217             memLevel,
 218             strategy);
 219         if(result != Z_OK)
 220             throw std::logic_error{"deflateInit2 failed"};
 221         zs.next_in = (Bytef*)in.data();
 222         zs.avail_in = static_cast<uInt>(in.size());
 223         std::string out;
 224         out.resize(deflateBound(&zs,
 225             static_cast<uLong>(in.size())));
 226         zs.next_in = (Bytef*)in.data();
 227         zs.avail_in = static_cast<uInt>(in.size());
 228         zs.next_out = (Bytef*)&out[0];
 229         zs.avail_out = static_cast<uInt>(out.size());
 230         result = deflate(&zs, Z_FULL_FLUSH);
 231         if(result != Z_OK)
 232             throw std::logic_error("deflate failed");
 233         out.resize(zs.total_out);
 234         deflateEnd(&zs);
 235         return out;
 236     }
 237
 238     static
 239     std::string
 240     decompress(string_view const& in)
 241     {
 242         int result;
 243         std::string out;
 244         z_stream zs;
 245         memset(&zs, 0, sizeof(zs));
 246         result = inflateInit2(&zs, -15);
 247         if(result != Z_OK)
 248             throw std::logic_error{"inflateInit2 failed"};
 249         try
 250         {
 251             zs.next_in = (Bytef*)in.data();
 252             zs.avail_in = static_cast<uInt>(in.size());
 253             for(;;)
 254             {
 255                 out.resize(zs.total_out + 1024);
 256                 zs.next_out = (Bytef*)&out[zs.total_out];
 257                 zs.avail_out = static_cast<uInt>(
 258                     out.size() - zs.total_out);
 259                 result = inflate(&zs, Z_SYNC_FLUSH);
 260                 if( result == Z_NEED_DICT ||
 261                     result == Z_DATA_ERROR ||
 262                     result == Z_MEM_ERROR)
 263                 {
 264                     throw std::logic_error("inflate failed");
 265                 }
 266                 if(zs.avail_out > 0)
 267                     break;
 268                 if(result == Z_STREAM_END)
 269                     break;
 270             }
 271             out.resize(zs.total_out);
 272             inflateEnd(&zs);
 273         }
 274         catch(...)
 275         {
 276             inflateEnd(&zs);
 277             throw;
 278         }
 279         return out;
 280     }
 281
 282     //--------------------------------------------------------------------------
 283
 284     using self = deflate_stream_test;
 285     typedef void(self::*pmf_t)(
 286         ICompressor& c,
 287         int level, int windowBits, int memLevel,
 288         int strategy, std::string const&);
 289
 290     static
 291     Strategy
 292     toStrategy(int strategy)
 293     {
 294         switch(strategy)
 295         {
 296         default:
 297         case 0: return Strategy::normal;
 298         case 1: return Strategy::filtered;
 299         case 2: return Strategy::huffman;
 300         case 3: return Strategy::rle;
 301         case 4: return Strategy::fixed;
 302         }
 303     }
 304
 305     void
 306     doDeflate1_beast(
 307         ICompressor& c,
 308         int level, int windowBits, int memLevel,
 309         int strategy, std::string const& check)
 310     {
 311         std::string out;
 312         c.init(
 313             level,
 314             windowBits,
 315             memLevel,
 316             strategy);
 317         out.resize(c.bound(check.size()));
 318         c.next_in(check.data());
 319         c.avail_in(check.size());
 320         c.next_out((void*)out.data());
 321         c.avail_out(out.size());
 322         {
 323             bool progress = true;
 324             for(;;)
 325             {
 326                 error_code ec = c.write(Flush::full);
 327                 if( ec == error::need_buffers ||
 328                     ec == error::end_of_stream) // per zlib FAQ
 329                     goto fin;
 330                 if(! BEAST_EXPECTS(! ec, ec.message()))
 331                     goto err;
 332                 if(! BEAST_EXPECT(progress))
 333                     goto err;
 334                 progress = false;
 335             }
 336         }
 337
 338     fin:
 339         out.resize(c.total_out());
 340         BEAST_EXPECT(decompress(out) == check);
 341
 342     err:
 343         ;
 344     }
 345
 346     //--------------------------------------------------------------------------
 347
 348     void
 349     doDeflate2_beast(
 350         ICompressor& c,
 351         int level, int windowBits, int memLevel,
 352         int strategy, std::string const& check)
 353     {
 354         for(std::size_t i = 1; i < check.size(); ++i)
 355         {
 356             for(std::size_t j = 1;; ++j)
 357             {
 358                 c.init(
 359                     level,
 360                     windowBits,
 361                     memLevel,
 362                     strategy);
 363                 std::string out;
 364                 out.resize(c.bound(check.size()));
 365                 if(j >= out.size())
 366                     break;
 367                 c.next_in((void*)check.data());
 368                 c.avail_in(i);
 369                 c.next_out((void*)out.data());
 370                 c.avail_out(j);
 371                 bool bi = false;
 372                 bool bo = false;
 373                 for(;;)
 374                 {
 375                     error_code ec = c.write(
 376                         bi ? Flush::full : Flush::none);
 377                     if( ec == error::need_buffers ||
 378                         ec == error::end_of_stream) // per zlib FAQ
 379                         goto fin;
 380                     if(! BEAST_EXPECTS(! ec, ec.message()))
 381                         goto err;
 382                     if(c.avail_in() == 0 && ! bi)
 383                     {
 384                         bi = true;
 385                         c.avail_in(check.size() - i);
 386                     }
 387                     if(c.avail_out() == 0 && ! bo)
 388                     {
 389                         bo = true;
 390                         c.avail_out(out.size() - j);
 391                     }
 392                 }
 393
 394             fin:
 395                 out.resize(c.total_out());
 396                 BEAST_EXPECT(decompress(out) == check);
 397
 398             err:
 399                 ;
 400             }
 401         }
 402     }
 403
 404     //--------------------------------------------------------------------------
 405
 406     void
 407     doMatrix(ICompressor& c, std::string const& check, pmf_t pmf)
 408     {
 409         for(int level = 0; level <= 9; ++level)
 410         {
 411             for(int windowBits = 8; windowBits <= 9; ++windowBits)
 412             {
 413                 // zlib has a bug with windowBits==8
 414                 if(windowBits == 8)
 415                     continue;
 416                 for(int strategy = 0; strategy <= 4; ++strategy)
 417                 {
 418                     for (int memLevel = 8; memLevel <= 9; ++memLevel)
 419                     {
 420                         (this->*pmf)(
 421                             c, level, windowBits, memLevel, strategy, check);
 422                     }
 423                 }
 424             }
 425         }
 426
 427         // Check default settings
 428         (this->*pmf)(c, compression::default_size, 15, 8, 0, check);
 429     }
 430
 431     void
 432     testDeflate(ICompressor& c)
 433     {
 434         doMatrix(c, "Hello, world!", &self::doDeflate1_beast);
 435         doMatrix(c, "Hello, world!", &self::doDeflate2_beast);
 436         log << "no-silence keepalive" << std::endl;
 437         doMatrix(c, corpus1(56), &self::doDeflate2_beast);
 438         doMatrix(c, corpus1(1024), &self::doDeflate1_beast);
 439     }
 440
 441     void testInvalidSettings(ICompressor& c)
 442     {
 443         except<std::invalid_argument>(
 444             [&]()
 445             {
 446                 c.init(-42, 15, 8, static_cast<int>(Strategy::normal));
 447             });
 448         except<std::invalid_argument>(
 449             [&]()
 450             {
 451                 c.init(compression::default_size, -1, 8, static_cast<int>(Strategy::normal));
 452             });
 453         except<std::invalid_argument>(
 454             [&]()
 455             {
 456                 c.init(compression::default_size, 15, -1, static_cast<int>(Strategy::normal));
 457             });
 458         except<std::invalid_argument>(
 459             [&]()
 460             {
 461                 c.init();
 462                 c.avail_in(1);
 463                 c.next_in(nullptr);
 464                 c.write(Flush::full);
 465             });
 466     }
 467
 468     void
 469     testWriteAfterFinish(ICompressor& c)
 470     {
 471         c.init();
 472         std::string out;
 473         out.resize(1024);
 474         string_view s = "Hello";
 475         c.next_in(s.data());
 476         c.avail_in(s.size());
 477         c.next_out(&out.front());
 478         c.avail_out(out.size());
 479         error_code ec = c.write(Flush::sync);
 480         BEAST_EXPECT(!ec);
 481         c.next_in(nullptr);
 482         c.avail_in(0);
 483         ec = c.write(Flush::finish);
 484         BEAST_EXPECT(ec == error::end_of_stream);
 485         c.next_in(s.data());
 486         c.avail_in(s.size());
 487         c.next_out(&out.front());
 488         c.avail_out(out.size());
 489         ec = c.write(Flush::sync);
 490         BEAST_EXPECT(ec == error::stream_error);
 491         ec = c.write(Flush::finish);
 492         BEAST_EXPECT(ec == error::need_buffers);
 493     }
 494
 495     void
 496     testFlushPartial(ICompressor& c)
 497     {
 498         c.init();
 499         std::string out;
 500         out.resize(1024);
 501         string_view s = "Hello";
 502         c.next_in(s.data());
 503         c.avail_in(s.size());
 504         c.next_out(&out.front());
 505         c.avail_out(out.size());
 506         error_code ec;
 507         ec = c.write(Flush::none);
 508         BEAST_EXPECT(!ec);
 509         ec = c.write(Flush::partial);
 510         BEAST_EXPECT(!ec);
 511     }
 512
 513     void
 514     testFlushAtLiteralBufferFull(ICompressor& c)
 515     {
 516         struct fixture
 517         {
 518             ICompressor& c;
 519             explicit fixture(ICompressor&c, std::size_t n, Strategy s) : c(c)
 520             {
 521                 c.init(8, 15, 1, static_cast<int>(s));
 522                 std::iota(in.begin(), in.end(), std::uint8_t{0});
 523                 out.resize(n);
 524                 c.next_in(in.data());
 525                 c.avail_in(in.size());
 526                 c.next_out(&out.front());
 527                 c.avail_out(out.size());
 528             }
 529
 530             std::array<std::uint8_t, 255> in;
 531             std::string out;
 532         };
 533
 534         for (auto s : {Strategy::huffman, Strategy::rle, Strategy::normal})
 535         {
 536             {
 537                 fixture f{c, 264, s};
 538                 error_code ec = c.write(Flush::finish);
 539                 BEAST_EXPECT(ec == error::end_of_stream);
 540                 BEAST_EXPECT(c.avail_out() == 1);
 541             }
 542
 543             {
 544                 fixture f{c,263, s};
 545                 error_code ec = c.write(Flush::finish);
 546                 BEAST_EXPECT(!ec);
 547                 BEAST_EXPECT(c.avail_out() == 0);
 548             }
 549
 550             {
 551                 fixture f{c, 20, s};
 552                 error_code ec = c.write(Flush::sync);
 553                 BEAST_EXPECT(!ec);
 554             }
 555
 556         }
 557     }
 558
 559     void
 560     testRLEMatchLengthExceedLookahead(ICompressor& c)
 561     {
 562         std::vector<std::uint8_t> in;
 563         in.resize(300);
 564
 565         c.init(8, 15, 1, static_cast<int>(Strategy::rle));
 566         std::fill_n(in.begin(), 4, 'a');
 567         std::string out;
 568         out.resize(in.size() * 2);
 569         c.next_in(in.data());
 570         c.avail_in(in.size());
 571         c.next_out(&out.front());
 572         c.avail_out(out.size());
 573
 574         error_code ec;
 575         ec = c.write(Flush::sync);
 576         BEAST_EXPECT(!ec);
 577     }
 578
 579     void
 580     testFlushAfterDistMatch(ICompressor& c)
 581     {
 582         for (auto out_size : {144, 129})
 583         {
 584             std::array<std::uint8_t, 256> in{};
 585             // 125 will mostly fill the lit buffer, so emitting a distance code
 586             // results in a flush.
 587             auto constexpr n = 125;
 588             std::iota(in.begin(), in.begin() + n,
 589                 static_cast<std::uint8_t>(0));
 590             std::iota(in.begin() + n, in.end(),
 591                 static_cast<std::uint8_t>(0));
 592
 593             c.init(8, 15, 1, static_cast<int>(Strategy::normal));
 594             std::string out;
 595             out.resize(out_size);
 596             c.next_in(in.data());
 597             c.avail_in(in.size());
 598             c.next_out(&out.front());
 599             c.avail_out(out.size());
 600
 601             error_code ec;
 602             ec = c.write(Flush::sync);
 603             BEAST_EXPECT(!ec);
 604         }
 605     }
 606
 607     void
 608     run() override
 609     {
 610         log <<
 611             "sizeof(deflate_stream) == " <<
 612             sizeof(deflate_stream) << std::endl;
 613
 614         testDeflate(zlib_compressor);
 615         testDeflate(beast_compressor);
 616         testInvalidSettings(zlib_compressor);
 617         testInvalidSettings(beast_compressor);
 618         testWriteAfterFinish(zlib_compressor);
 619         testWriteAfterFinish(beast_compressor);
 620         testFlushPartial(zlib_compressor);
 621         testFlushPartial(beast_compressor);
 622         testFlushAtLiteralBufferFull(zlib_compressor);
 623         testFlushAtLiteralBufferFull(beast_compressor);
 624         testRLEMatchLengthExceedLookahead(zlib_compressor);
 625         testRLEMatchLengthExceedLookahead(beast_compressor);
 626         testFlushAfterDistMatch(zlib_compressor);
 627         testFlushAfterDistMatch(beast_compressor);
 628     }
 629 };
 630
 631 BEAST_DEFINE_TESTSUITE(beast,zlib,deflate_stream);
 632
 633 } // zlib
 634 } // beast
 635 } // boost