modules/core/src/zip_input.cpp

   1 /*
   2  * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        zip_input.cpp
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @version     1.0
  20  * @brief       This file is the implementation file of zip input
  21  */
  22 #include <stddef.h>
  23 #include <iomanip>
  24 #include <sys/stat.h>
  25 #include <sys/mman.h>
  26 #include <fcntl.h>
  27 #include <errno.h>
  28 #include <dpl/zip_input.h>
  29 #include <dpl/scoped_close.h>
  30 #include <dpl/binary_queue.h>
  31 #include <dpl/scoped_free.h>
  32 #include <memory>
  33 #include <dpl/scoped_array.h>
  34 #include <dpl/foreach.h>
  35 #include <dpl/log/log.h>
  36 #include <new>
  37 #include <minizip/unzip.h>
  38
  39 namespace DPL {
  40 namespace // anonymous
  41 {
  42 const size_t EXTRACT_BUFFER_SIZE = 4096;
  43
  44 class ScopedUnzClose
  45 {
  46   private:
  47     unzFile m_file;
  48
  49   public:
  50     ScopedUnzClose(unzFile file) :
  51         m_file(file)
  52     {}
  53
  54     ~ScopedUnzClose()
  55     {
  56         if (!m_file) {
  57             return;
  58         }
  59
  60         if (unzClose(m_file) != UNZ_OK) {
  61             LogPedantic("Failed to close zip input file");
  62         }
  63     }
  64
  65     unzFile Release()
  66     {
  67         unzFile file = m_file;
  68
  69         m_file = NULL;
  70
  71         return file;
  72     }
  73 };
  74 } // namespace anonymous
  75
  76 /*
  77  * Seekable multiplexing device
  78  *
  79  * Explanation:
  80  * Minizip library lacks serious support for multithreaded
  81  * access to zip files. Thus, they cannot be easily extracted
  82  * simulateously. Here is introduced seekable device which does
  83  * have a context with seek index for each file. File is mapped to
  84  * memory and because of that no real synchronization is needed.
  85  * Memory addresses can be indexed.
  86  *
  87  * About generalization:
  88  * To achieve the same results on abstract input device, there must be
  89  * provided a mechanism to read data from random address without
  90  * synchronization.
  91  * In other words: stateless. As described above, stateless property can be
  92  * achieved via memory mapping.
  93  */
  94 class Device
  95 {
  96   private:
  97     int m_handle;
  98     off64_t m_size; // file mapping size
  99     unsigned char *m_address; // mapping base address
 100
 101     struct File
 102     {
 103         off64_t offset;
 104         Device *device;
 105
 106         File(Device *d) :
 107             offset(0),
 108             device(d)
 109         {}
 110     };
 111
 112   public:
 113     Device(const std::string &fileName)
 114     {
 115         LogPedantic("Creating file mapping");
 116         // Open device and map it to user space
 117         int file = TEMP_FAILURE_RETRY(open(fileName.c_str(), O_RDONLY));
 118
 119         if (file == -1) {
 120             int error = errno;
 121             ThrowMsg(ZipInput::Exception::OpenFailed,
 122                      "Failed to open file. errno = " << error);
 123         }
 124
 125         // Scoped close on file
 126         ScopedClose scopedClose(file);
 127
 128         // Calculate file size
 129         off64_t size = lseek64(file, 0, SEEK_END);
 130
 131         if (size == static_cast<off64_t>(-1)) {
 132             int error = errno;
 133             ThrowMsg(ZipInput::Exception::OpenFailed,
 134                      "Failed to seek file. errno = " << error);
 135         }
 136
 137         // Map file to usespace
 138         void *address = mmap(0, static_cast<size_t>(size),
 139                              PROT_READ, MAP_SHARED, file, 0);
 140
 141         if (address == MAP_FAILED) {
 142             int error = errno;
 143             ThrowMsg(ZipInput::Exception::OpenFailed,
 144                      "Failed to map file. errno = " << error);
 145         }
 146
 147         // Release scoped close
 148         m_handle = scopedClose.Release();
 149
 150         // Save mapped up address
 151         m_size = size;
 152         m_address = static_cast<unsigned char *>(address);
 153
 154         LogPedantic("Created file mapping: " << fileName <<
 155                     " of size: " << m_size <<
 156                     " at address: " << std::hex <<
 157                     static_cast<void *>(m_address));
 158     }
 159
 160     ~Device()
 161     {
 162         // Close mapping
 163         if (munmap(m_address, static_cast<size_t>(m_size)) == -1) {
 164             int error = errno;
 165             LogPedantic("Failed to munmap file. errno = " << error);
 166         }
 167
 168         // Close file descriptor
 169         if (close(m_handle) == -1) {
 170             int error = errno;
 171             LogPedantic("Failed to close file. errno = " << error);
 172         }
 173     }
 174
 175     // zlib_filefunc64_def interface: files
 176     static voidpf ZCALLBACK open64_file(voidpf opaque,
 177                                         const void* /*filename*/,
 178                                         int /*mode*/)
 179     {
 180         Device *device = static_cast<Device *>(opaque);
 181
 182         // Open file for master device
 183         return new File(device);
 184     }
 185
 186     static uLong ZCALLBACK read_file(voidpf opaque,
 187                                      voidpf pstream,
 188                                      void* buf,
 189                                      uLong size)
 190     {
 191         Device *device = static_cast<Device *>(opaque);
 192         File *deviceFile = static_cast<File *>(pstream);
 193
 194         // Check if offset is out of bounds
 195         if (deviceFile->offset >= device->m_size) {
 196             LogPedantic("Device: read offset out of bounds");
 197             return -1;
 198         }
 199
 200         off64_t bytesLeft = device->m_size -
 201             deviceFile->offset;
 202
 203         off64_t bytesToRead;
 204
 205         // Calculate bytes to read
 206         if (static_cast<off64_t>(size) > bytesLeft) {
 207             bytesToRead = bytesLeft;
 208         } else {
 209             bytesToRead = static_cast<off64_t>(size);
 210         }
 211
 212         // Do copy
 213         memcpy(buf,
 214                device->m_address + deviceFile->offset,
 215                static_cast<size_t>(bytesToRead));
 216
 217         // Increment file offset
 218         deviceFile->offset += bytesToRead;
 219
 220         // Return bytes that were actually read
 221         return static_cast<uLong>(bytesToRead);
 222     }
 223
 224     static uLong ZCALLBACK write_file(voidpf /*opaque*/,
 225                                       voidpf /*stream*/,
 226                                       const void* /*buf*/,
 227                                       uLong /*size*/)
 228     {
 229         // Not supported by device
 230         LogPedantic("Unsupported function called!");
 231         return -1;
 232     }
 233
 234     static int ZCALLBACK close_file(voidpf /*opaque*/, voidpf stream)
 235     {
 236         File *deviceFile = static_cast<File *>(stream);
 237
 238         // Delete file
 239         delete deviceFile;
 240
 241         // Always OK
 242         return 0;
 243     }
 244
 245     static int ZCALLBACK testerror_file(voidpf /*opaque*/, voidpf /*stream*/)
 246     {
 247         // No errors
 248         return 0;
 249     }
 250
 251     static ZPOS64_T ZCALLBACK tell64_file(voidpf /*opaque*/, voidpf stream)
 252     {
 253         File *deviceFile = static_cast<File *>(stream);
 254
 255         return static_cast<ZPOS64_T>(deviceFile->offset);
 256     }
 257
 258     static long ZCALLBACK seek64_file(voidpf opaque,
 259                                       voidpf stream,
 260                                       ZPOS64_T offset,
 261                                       int origin)
 262     {
 263         Device *device = static_cast<Device *>(opaque);
 264         File *deviceFile = static_cast<File *>(stream);
 265
 266         switch (origin) {
 267         case ZLIB_FILEFUNC_SEEK_SET:
 268             deviceFile->offset = static_cast<off64_t>(offset);
 269
 270             break;
 271
 272         case ZLIB_FILEFUNC_SEEK_CUR:
 273             deviceFile->offset += static_cast<off64_t>(offset);
 274
 275             break;
 276
 277         case ZLIB_FILEFUNC_SEEK_END:
 278             deviceFile->offset =
 279                 device->m_size -
 280                 static_cast<off64_t>(offset);
 281
 282             break;
 283
 284         default:
 285             return -1;
 286         }
 287
 288         return 0;
 289     }
 290 };
 291
 292 ZipInput::ZipInput(const std::string &fileName) :
 293     m_device(NULL),
 294     m_numberOfFiles(0),
 295     m_globalComment(),
 296     m_totalUncompressedSize(0),
 297     m_fileInfos()
 298 {
 299     LogPedantic("Zip input file: " << fileName);
 300
 301     // Create master device
 302     LogPedantic("Creating master device");
 303     std::unique_ptr<Device> device(new Device(fileName));
 304
 305     // Open master file
 306     zlib_filefunc64_def interface;
 307     interface.zopen64_file = &Device::open64_file;
 308     interface.zread_file = &Device::read_file;
 309     interface.zwrite_file = &Device::write_file;
 310     interface.ztell64_file = &Device::tell64_file;
 311     interface.zseek64_file = &Device::seek64_file;
 312     interface.zclose_file = &Device::close_file;
 313     interface.zerror_file = &Device::testerror_file;
 314     interface.opaque = device.get();
 315
 316     LogPedantic("Opening zip file");
 317     unzFile file = unzOpen2_64(NULL, &interface);
 318
 319     if (file == NULL) {
 320         LogPedantic("Failed to open zip file");
 321
 322         // Some errror occured
 323         ThrowMsg(Exception::OpenFailed,
 324                  "Failed to open zip file: " << fileName);
 325     }
 326
 327     // Begin scope
 328     ScopedUnzClose scopedUnzClose(file);
 329
 330     // Read contents
 331     ReadGlobalInfo(file);
 332     ReadGlobalComment(file);
 333     ReadInfos(file);
 334
 335     // Release scoped unz close
 336     m_masterFile = scopedUnzClose.Release();
 337     m_device = device.release();
 338
 339     LogPedantic("Zip file opened");
 340 }
 341
 342 ZipInput::~ZipInput()
 343 {
 344     // Close zip
 345     if (unzClose(static_cast<unzFile>(m_masterFile)) != UNZ_OK) {
 346         LogPedantic("Failed to close zip input file");
 347     }
 348
 349     // Close device
 350     delete m_device;
 351 }
 352
 353 void ZipInput::ReadGlobalInfo(void *masterFile)
 354 {
 355     // Read number of entries and global comment
 356     unz_global_info globalInfo;
 357
 358     if (unzGetGlobalInfo(static_cast<unzFile>(masterFile),
 359                          &globalInfo) != UNZ_OK)
 360     {
 361         LogPedantic("Failed to read zip global info");
 362
 363         ThrowMsg(Exception::ReadGlobalInfoFailed,
 364                  "Failed to read global info");
 365     }
 366
 367     m_numberOfFiles = static_cast<size_t>(globalInfo.number_entry);
 368     m_globalCommentSize = static_cast<size_t>(globalInfo.size_comment);
 369
 370     LogPedantic("Number of files: " << m_numberOfFiles);
 371     LogPedantic("Global comment size: " << m_globalCommentSize);
 372 }
 373
 374 void ZipInput::ReadGlobalComment(void *masterFile)
 375 {
 376     ScopedArray<char> comment(new char[m_globalCommentSize + 1]);
 377
 378     if (unzGetGlobalComment(static_cast<unzFile>(masterFile),
 379                             comment.Get(),
 380                             m_globalCommentSize + 1) != UNZ_OK)
 381     {
 382         LogPedantic("Failed to read zip global comment");
 383
 384         ThrowMsg(Exception::ReadGlobalCommentFailed,
 385                  "Failed to read global comment");
 386     }
 387
 388     m_globalComment = comment.Get();
 389     LogPedantic("Global comment: " << m_globalComment);
 390 }
 391
 392 void ZipInput::ReadInfos(void *masterFile)
 393 {
 394     // Read infos
 395     m_fileInfos.reserve(m_numberOfFiles);
 396
 397     if (unzGoToFirstFile(static_cast<unzFile>(masterFile)) != UNZ_OK) {
 398         LogPedantic("Failed to go to first file");
 399         ThrowMsg(Exception::SeekFileFailed, "Failed to seek first file");
 400     }
 401
 402     for (size_t i = 0; i < m_numberOfFiles; ++i) {
 403         unz_file_pos_s filePos;
 404
 405         if (unzGetFilePos(static_cast<unzFile>(masterFile),
 406                           &filePos) != UNZ_OK)
 407         {
 408             LogPedantic("Failed to get file pos");
 409             ThrowMsg(Exception::FileInfoFailed, "Failed to get zip file info");
 410         }
 411
 412         unz_file_info64 fileInfo;
 413
 414         if (unzGetCurrentFileInfo64(static_cast<unzFile>(masterFile),
 415                                     &fileInfo,
 416                                     NULL,
 417                                     0,
 418                                     NULL,
 419                                     0,
 420                                     NULL,
 421                                     0) != UNZ_OK)
 422         {
 423             LogPedantic("Failed to get file pos");
 424             ThrowMsg(Exception::FileInfoFailed, "Failed to get zip file info");
 425         }
 426
 427         ScopedArray<char> fileName(new char[fileInfo.size_filename + 1]);
 428         ScopedArray<char> fileComment(new char[fileInfo.size_file_comment + 1]);
 429
 430         if (unzGetCurrentFileInfo64(static_cast<unzFile>(masterFile),
 431                                     &fileInfo,
 432                                     fileName.Get(),
 433                                     fileInfo.size_filename + 1,
 434                                     NULL,
 435                                     0,
 436                                     fileComment.Get(),
 437                                     fileInfo.size_file_comment + 1) != UNZ_OK)
 438         {
 439             LogPedantic("Failed to get file pos");
 440             ThrowMsg(Exception::FileInfoFailed, "Failed to get zip file info");
 441         }
 442
 443         m_fileInfos.push_back(
 444             FileInfo(
 445                 FileHandle(
 446                     static_cast<size_t>(filePos.pos_in_zip_directory),
 447                     static_cast<size_t>(filePos.num_of_file)
 448                     ),
 449                 std::string(fileName.Get()),
 450                 std::string(fileComment.Get()),
 451                 static_cast<off64_t>(fileInfo.compressed_size),
 452                 static_cast<off64_t>(fileInfo.uncompressed_size)
 453                 )
 454             );
 455         m_totalUncompressedSize += static_cast<size_t>(fileInfo.uncompressed_size);
 456
 457         // If this is not the last file, go to next one
 458         if (i != m_numberOfFiles - 1) {
 459             if (unzGoToNextFile(
 460                     static_cast<unzFile>(masterFile)) != UNZ_OK)
 461             {
 462                 LogPedantic("Failed to go to next file");
 463
 464                 ThrowMsg(Exception::FileInfoFailed,
 465                          "Failed to go to next file");
 466             }
 467         }
 468     }
 469 }
 470
 471 ZipInput::const_iterator ZipInput::begin() const
 472 {
 473     return m_fileInfos.begin();
 474 }
 475
 476 ZipInput::const_iterator ZipInput::end() const
 477 {
 478     return m_fileInfos.end();
 479 }
 480
 481 ZipInput::const_reverse_iterator ZipInput::rbegin() const
 482 {
 483     return m_fileInfos.rbegin();
 484 }
 485
 486 ZipInput::const_reverse_iterator ZipInput::rend() const
 487 {
 488     return m_fileInfos.rend();
 489 }
 490
 491 ZipInput::size_type ZipInput::size() const
 492 {
 493     return m_fileInfos.size();
 494 }
 495
 496 ZipInput::File *ZipInput::OpenFile(const std::string &fileName)
 497 {
 498     FOREACH(iterator, m_fileInfos)
 499     {
 500         if (iterator->name == fileName) {
 501             return new File(m_device, iterator->handle);
 502         }
 503     }
 504
 505     ThrowMsg(Exception::OpenFileFailed,
 506              "Failed to open zip file: " << fileName);
 507 }
 508
 509 ZipInput::File::File(class Device *device, FileHandle handle)
 510 {
 511     // Open file file
 512     zlib_filefunc64_def interface;
 513     interface.zopen64_file = &Device::open64_file;
 514     interface.zread_file = &Device::read_file;
 515     interface.zwrite_file = &Device::write_file;
 516     interface.ztell64_file = &Device::tell64_file;
 517     interface.zseek64_file = &Device::seek64_file;
 518     interface.zclose_file = &Device::close_file;
 519     interface.zerror_file = &Device::testerror_file;
 520     interface.opaque = device;
 521
 522     LogPedantic("Opening zip file");
 523     unzFile file = unzOpen2_64(NULL, &interface);
 524
 525     if (file == NULL) {
 526         LogPedantic("Failed to open zip file");
 527
 528         // Some errror occured
 529         ThrowMsg(ZipInput::Exception::OpenFileFailed,
 530                  "Failed to open zip file");
 531     }
 532
 533     // Begin scope
 534     ScopedUnzClose scopedUnzClose(file);
 535
 536     // Look up file handle
 537     unz64_file_pos filePos = {
 538         static_cast<ZPOS64_T>(handle.first),
 539         static_cast<ZPOS64_T>(handle.second)
 540     };
 541
 542     if (unzGoToFilePos64(file, &filePos) != UNZ_OK) {
 543         LogPedantic("Failed to seek to zip file");
 544
 545         // Some errror occured
 546         ThrowMsg(ZipInput::Exception::OpenFileFailed,
 547                  "Failed to seek into zip file");
 548     }
 549
 550     // Open current file for reading
 551     if (unzOpenCurrentFile(file) != UNZ_OK) {
 552         LogPedantic("Failed to open current zip file");
 553
 554         // Some errror occured
 555         ThrowMsg(ZipInput::Exception::OpenFileFailed,
 556                  "Failed to open current zip file");
 557     }
 558
 559     // Release scoped unz close
 560     m_file = scopedUnzClose.Release();
 561
 562     LogPedantic("Zip file opened");
 563 }
 564
 565 ZipInput::File::~File()
 566 {
 567     // Close current file for reading
 568     if (unzCloseCurrentFile(static_cast<unzFile>(m_file)) != UNZ_OK) {
 569         LogPedantic("Failed to close current zip input file");
 570     }
 571
 572     // Close zip file
 573     if (unzClose(static_cast<unzFile>(m_file)) != UNZ_OK) {
 574         LogPedantic("Failed to close zip input file");
 575     }
 576 }
 577
 578 DPL::BinaryQueueAutoPtr ZipInput::File::Read(size_t size)
 579 {
 580     // Do not even try to unzip if requested zero bytes
 581     if (size == 0) {
 582         return DPL::BinaryQueueAutoPtr(new DPL::BinaryQueue());
 583     }
 584
 585     // Calc data to read
 586     size_t sizeToRead = size > EXTRACT_BUFFER_SIZE ?
 587         EXTRACT_BUFFER_SIZE :
 588         size;
 589
 590     // Extract zip file data (one-copy)
 591     ScopedFree<void> rawBuffer(malloc(sizeToRead));
 592
 593     if (!rawBuffer) {
 594         throw std::bad_alloc();
 595     }
 596
 597     // Do unpack
 598     int bytes = unzReadCurrentFile(static_cast<unzFile>(m_file),
 599                                    rawBuffer.Get(),
 600                                    sizeToRead);
 601
 602     // Internal unzipper error
 603     if (bytes < 0) {
 604         LogPedantic("Extract failed. Error: " << bytes);
 605
 606         ThrowMsg(ZipInput::Exception::ReadFileFailed,
 607                  "Failed to extract file with error: " << bytes);
 608     }
 609
 610     // Data was read (may be zero bytes)
 611     DPL::BinaryQueueAutoPtr buffer(new DPL::BinaryQueue());
 612
 613     buffer->AppendUnmanaged(rawBuffer.Get(),
 614                             static_cast<size_t>(bytes),
 615                             &DPL::BinaryQueue::BufferDeleterFree,
 616                             NULL);
 617
 618     rawBuffer.Release();
 619
 620     return buffer;
 621 }
 622
 623 const std::string &ZipInput::GetGlobalComment() const
 624 {
 625     return m_globalComment;
 626 }
 627
 628 bool ZipInput::empty() const
 629 {
 630     return m_fileInfos.empty();
 631 }
 632
 633 size_t ZipInput::GetTotalUncompressedSize() const
 634 {
 635     return m_totalUncompressedSize;
 636 }
 637 } // namespace DPL