Source/cmArchiveWrite.cxx

   1 /* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
   2    file Copyright.txt or https://cmake.org/licensing for details.  */
   3 #include "cmArchiveWrite.h"
   4
   5 #include <cstdlib>
   6 #include <cstring>
   7 #include <ctime>
   8 #include <iostream>
   9 #include <limits>
  10 #include <sstream>
  11 #include <string>
  12 #include <thread>
  13
  14 #include <cm/algorithm>
  15
  16 #include <cm3p/archive.h>
  17 #include <cm3p/archive_entry.h>
  18
  19 #include "cmsys/Directory.hxx"
  20 #include "cmsys/Encoding.hxx"
  21 #include "cmsys/FStream.hxx"
  22
  23 #include "cm_get_date.h"
  24
  25 #include "cmLocale.h"
  26 #include "cmStringAlgorithms.h"
  27 #include "cmSystemTools.h"
  28
  29 #ifndef __LA_SSIZE_T
  30 #  define __LA_SSIZE_T la_ssize_t
  31 #endif
  32
  33 static std::string cm_archive_error_string(struct archive* a)
  34 {
  35   const char* e = archive_error_string(a);
  36   return e ? e : "unknown error";
  37 }
  38
  39 static void cm_archive_entry_copy_pathname(struct archive_entry* e,
  40                                            const std::string& dest)
  41 {
  42 #if cmsys_STL_HAS_WSTRING
  43   archive_entry_copy_pathname_w(e, cmsys::Encoding::ToWide(dest).c_str());
  44 #else
  45   archive_entry_copy_pathname(e, dest.c_str());
  46 #endif
  47 }
  48
  49 static void cm_archive_entry_copy_sourcepath(struct archive_entry* e,
  50                                              const std::string& file)
  51 {
  52 #if cmsys_STL_HAS_WSTRING
  53   archive_entry_copy_sourcepath_w(e, cmsys::Encoding::ToWide(file).c_str());
  54 #else
  55   archive_entry_copy_sourcepath(e, file.c_str());
  56 #endif
  57 }
  58
  59 class cmArchiveWrite::Entry
  60 {
  61   struct archive_entry* Object;
  62
  63 public:
  64   Entry()
  65     : Object(archive_entry_new())
  66   {
  67   }
  68   ~Entry() { archive_entry_free(this->Object); }
  69   Entry(const Entry&) = delete;
  70   Entry& operator=(const Entry&) = delete;
  71   operator struct archive_entry*() { return this->Object; }
  72 };
  73
  74 struct cmArchiveWrite::Callback
  75 {
  76   // archive_write_callback
  77   static __LA_SSIZE_T Write(struct archive* /*unused*/, void* cd,
  78                             const void* b, size_t n)
  79   {
  80     cmArchiveWrite* self = static_cast<cmArchiveWrite*>(cd);
  81     if (self->Stream.write(static_cast<const char*>(b),
  82                            static_cast<std::streamsize>(n))) {
  83       return static_cast<__LA_SSIZE_T>(n);
  84     }
  85     return static_cast<__LA_SSIZE_T>(-1);
  86   }
  87 };
  88
  89 cmArchiveWrite::cmArchiveWrite(std::ostream& os, Compress c,
  90                                std::string const& format, int compressionLevel,
  91                                int numThreads)
  92   : Stream(os)
  93   , Archive(archive_write_new())
  94   , Disk(archive_read_disk_new())
  95   , Format(format)
  96 {
  97   // Upstream fixed an issue with their integer parsing in 3.4.0
  98   // which would cause spurious errors to be raised from `strtoull`.
  99
 100   if (numThreads < 1) {
 101     int upperLimit = (numThreads == 0) ? std::numeric_limits<int>::max()
 102                                        : std::abs(numThreads);
 103
 104     numThreads =
 105       cm::clamp<int>(std::thread::hardware_concurrency(), 1, upperLimit);
 106   }
 107
 108   std::string sNumThreads = std::to_string(numThreads);
 109
 110   switch (c) {
 111     case CompressNone:
 112       if (archive_write_add_filter_none(this->Archive) != ARCHIVE_OK) {
 113         this->Error = cmStrCat("archive_write_add_filter_none: ",
 114                                cm_archive_error_string(this->Archive));
 115         return;
 116       }
 117       break;
 118     case CompressCompress:
 119       if (archive_write_add_filter_compress(this->Archive) != ARCHIVE_OK) {
 120         this->Error = cmStrCat("archive_write_add_filter_compress: ",
 121                                cm_archive_error_string(this->Archive));
 122         return;
 123       }
 124       break;
 125     case CompressGZip: {
 126       if (archive_write_add_filter_gzip(this->Archive) != ARCHIVE_OK) {
 127         this->Error = cmStrCat("archive_write_add_filter_gzip: ",
 128                                cm_archive_error_string(this->Archive));
 129         return;
 130       }
 131       std::string source_date_epoch;
 132       cmSystemTools::GetEnv("SOURCE_DATE_EPOCH", source_date_epoch);
 133       if (!source_date_epoch.empty()) {
 134         // We're not able to specify an arbitrary timestamp for gzip.
 135         // The next best thing is to omit the timestamp entirely.
 136         if (archive_write_set_filter_option(this->Archive, "gzip", "timestamp",
 137                                             nullptr) != ARCHIVE_OK) {
 138           this->Error = cmStrCat("archive_write_set_filter_option: ",
 139                                  cm_archive_error_string(this->Archive));
 140           return;
 141         }
 142       }
 143     } break;
 144     case CompressBZip2:
 145       if (archive_write_add_filter_bzip2(this->Archive) != ARCHIVE_OK) {
 146         this->Error = cmStrCat("archive_write_add_filter_bzip2: ",
 147                                cm_archive_error_string(this->Archive));
 148         return;
 149       }
 150       break;
 151     case CompressLZMA:
 152       if (archive_write_add_filter_lzma(this->Archive) != ARCHIVE_OK) {
 153         this->Error = cmStrCat("archive_write_add_filter_lzma: ",
 154                                cm_archive_error_string(this->Archive));
 155         return;
 156       }
 157       break;
 158     case CompressXZ:
 159       if (archive_write_add_filter_xz(this->Archive) != ARCHIVE_OK) {
 160         this->Error = cmStrCat("archive_write_add_filter_xz: ",
 161                                cm_archive_error_string(this->Archive));
 162         return;
 163       }
 164
 165 #if ARCHIVE_VERSION_NUMBER >= 3004000
 166
 167 #  ifdef _AIX
 168       // FIXME: Using more than 2 threads creates an empty archive.
 169       // Enforce this limit pending further investigation.
 170       if (numThreads > 2) {
 171         numThreads = 2;
 172         sNumThreads = std::to_string(numThreads);
 173       }
 174 #  endif
 175       if (archive_write_set_filter_option(this->Archive, "xz", "threads",
 176                                           sNumThreads.c_str()) != ARCHIVE_OK) {
 177         this->Error = cmStrCat("archive_compressor_xz_options: ",
 178                                cm_archive_error_string(this->Archive));
 179         return;
 180       }
 181 #endif
 182
 183       break;
 184     case CompressZstd:
 185       if (archive_write_add_filter_zstd(this->Archive) != ARCHIVE_OK) {
 186         this->Error = cmStrCat("archive_write_add_filter_zstd: ",
 187                                cm_archive_error_string(this->Archive));
 188         return;
 189       }
 190
 191 #if ARCHIVE_VERSION_NUMBER >= 3006000
 192       if (archive_write_set_filter_option(this->Archive, "zstd", "threads",
 193                                           sNumThreads.c_str()) != ARCHIVE_OK) {
 194         this->Error = cmStrCat("archive_compressor_zstd_options: ",
 195                                cm_archive_error_string(this->Archive));
 196         return;
 197       }
 198 #endif
 199       break;
 200   }
 201
 202   if (compressionLevel != 0) {
 203     std::string compressionLevelStr = std::to_string(compressionLevel);
 204     std::string archiveFilterName;
 205     switch (c) {
 206       case CompressNone:
 207       case CompressCompress:
 208         break;
 209       case CompressGZip:
 210         archiveFilterName = "gzip";
 211         break;
 212       case CompressBZip2:
 213         archiveFilterName = "bzip2";
 214         break;
 215       case CompressLZMA:
 216         archiveFilterName = "lzma";
 217         break;
 218       case CompressXZ:
 219         archiveFilterName = "xz";
 220         break;
 221       case CompressZstd:
 222         archiveFilterName = "zstd";
 223         break;
 224     }
 225     if (!archiveFilterName.empty()) {
 226       if (archive_write_set_filter_option(
 227             this->Archive, archiveFilterName.c_str(), "compression-level",
 228             compressionLevelStr.c_str()) != ARCHIVE_OK) {
 229         this->Error = cmStrCat("archive_write_set_filter_option: ",
 230                                cm_archive_error_string(this->Archive));
 231         return;
 232       }
 233     }
 234   }
 235
 236 #if !defined(_WIN32) || defined(__CYGWIN__)
 237   if (archive_read_disk_set_standard_lookup(this->Disk) != ARCHIVE_OK) {
 238     this->Error = cmStrCat("archive_read_disk_set_standard_lookup: ",
 239                            cm_archive_error_string(this->Archive));
 240     return;
 241   }
 242 #endif
 243
 244   if (archive_write_set_format_by_name(this->Archive, format.c_str()) !=
 245       ARCHIVE_OK) {
 246     this->Error = cmStrCat("archive_write_set_format_by_name: ",
 247                            cm_archive_error_string(this->Archive));
 248     return;
 249   }
 250
 251   // do not pad the last block!!
 252   if (archive_write_set_bytes_in_last_block(this->Archive, 1)) {
 253     this->Error = cmStrCat("archive_write_set_bytes_in_last_block: ",
 254                            cm_archive_error_string(this->Archive));
 255     return;
 256   }
 257 }
 258
 259 bool cmArchiveWrite::Open()
 260 {
 261   if (!this->Error.empty()) {
 262     return false;
 263   }
 264   if (archive_write_open(
 265         this->Archive, this, nullptr,
 266         reinterpret_cast<archive_write_callback*>(&Callback::Write),
 267         nullptr) != ARCHIVE_OK) {
 268     this->Error =
 269       cmStrCat("archive_write_open: ", cm_archive_error_string(this->Archive));
 270     return false;
 271   }
 272   return true;
 273 }
 274
 275 cmArchiveWrite::~cmArchiveWrite()
 276 {
 277   archive_read_free(this->Disk);
 278   archive_write_free(this->Archive);
 279 }
 280
 281 bool cmArchiveWrite::Add(std::string path, size_t skip, const char* prefix,
 282                          bool recursive)
 283 {
 284   if (!path.empty() && path.back() == '/') {
 285     path.erase(path.size() - 1);
 286   }
 287   this->AddPath(path.c_str(), skip, prefix, recursive);
 288   return this->Okay();
 289 }
 290
 291 bool cmArchiveWrite::AddPath(const char* path, size_t skip, const char* prefix,
 292                              bool recursive)
 293 {
 294   if (strcmp(path, ".") != 0 ||
 295       (this->Format != "zip" && this->Format != "7zip")) {
 296     if (!this->AddFile(path, skip, prefix)) {
 297       return false;
 298     }
 299   }
 300   if ((!cmSystemTools::FileIsDirectory(path) || !recursive) ||
 301       cmSystemTools::FileIsSymlink(path)) {
 302     return true;
 303   }
 304   cmsys::Directory d;
 305   if (d.Load(path)) {
 306     std::string next = cmStrCat(path, '/');
 307     if (next == "./" && (this->Format == "zip" || this->Format == "7zip")) {
 308       next.clear();
 309     }
 310     std::string::size_type end = next.size();
 311     unsigned long n = d.GetNumberOfFiles();
 312     for (unsigned long i = 0; i < n; ++i) {
 313       const char* file = d.GetFile(i);
 314       if (strcmp(file, ".") != 0 && strcmp(file, "..") != 0) {
 315         next.erase(end);
 316         next += file;
 317         if (!this->AddPath(next.c_str(), skip, prefix)) {
 318           return false;
 319         }
 320       }
 321     }
 322   }
 323   return true;
 324 }
 325
 326 bool cmArchiveWrite::AddFile(const char* file, size_t skip, const char* prefix)
 327 {
 328   this->Error = "";
 329   // Skip the file if we have no name for it.  This may happen on a
 330   // top-level directory, which does not need to be included anyway.
 331   if (skip >= strlen(file)) {
 332     return true;
 333   }
 334   const char* out = file + skip;
 335
 336   cmLocaleRAII localeRAII;
 337   static_cast<void>(localeRAII);
 338
 339   // Meta-data.
 340   std::string dest = cmStrCat(prefix ? prefix : "", out);
 341   if (this->Verbose) {
 342     std::cout << dest << "\n";
 343   }
 344   Entry e;
 345   cm_archive_entry_copy_sourcepath(e, file);
 346   cm_archive_entry_copy_pathname(e, dest);
 347   if (archive_read_disk_entry_from_file(this->Disk, e, -1, nullptr) !=
 348       ARCHIVE_OK) {
 349     this->Error = cmStrCat("Unable to read from file '", file,
 350                            "': ", cm_archive_error_string(this->Disk));
 351     return false;
 352   }
 353   if (!this->MTime.empty()) {
 354     time_t now;
 355     time(&now);
 356     time_t t = cm_get_date(now, this->MTime.c_str());
 357     if (t == -1) {
 358       this->Error = cmStrCat("unable to parse mtime '", this->MTime, '\'');
 359       return false;
 360     }
 361     archive_entry_set_mtime(e, t, 0);
 362   } else {
 363     std::string source_date_epoch;
 364     cmSystemTools::GetEnv("SOURCE_DATE_EPOCH", source_date_epoch);
 365     if (!source_date_epoch.empty()) {
 366       std::istringstream iss(source_date_epoch);
 367       time_t epochTime;
 368       iss >> epochTime;
 369       if (iss.eof() && !iss.fail()) {
 370         // Set all of the file times to the epoch time to handle archive
 371         // formats that include creation/access time.
 372         archive_entry_set_mtime(e, epochTime, 0);
 373         archive_entry_set_atime(e, epochTime, 0);
 374         archive_entry_set_ctime(e, epochTime, 0);
 375         archive_entry_set_birthtime(e, epochTime, 0);
 376       }
 377     }
 378   }
 379
 380   // manages the uid/guid of the entry (if any)
 381   if (this->Uid.IsSet() && this->Gid.IsSet()) {
 382     archive_entry_set_uid(e, this->Uid.Get());
 383     archive_entry_set_gid(e, this->Gid.Get());
 384   }
 385
 386   if (!this->Uname.empty() && !this->Gname.empty()) {
 387     archive_entry_set_uname(e, this->Uname.c_str());
 388     archive_entry_set_gname(e, this->Gname.c_str());
 389   }
 390
 391   // manages the permissions
 392   if (this->Permissions.IsSet()) {
 393     archive_entry_set_perm(e, this->Permissions.Get());
 394   }
 395
 396   if (this->PermissionsMask.IsSet()) {
 397     int perm = archive_entry_perm(e);
 398     archive_entry_set_perm(e, perm & this->PermissionsMask.Get());
 399   }
 400
 401   // Clear acl and xattr fields not useful for distribution.
 402   archive_entry_acl_clear(e);
 403   archive_entry_xattr_clear(e);
 404   archive_entry_set_fflags(e, 0, 0);
 405
 406   if (this->Format == "pax" || this->Format == "paxr") {
 407     // Sparse files are a GNU tar extension.
 408     // Do not use them in standard tar files.
 409     archive_entry_sparse_clear(e);
 410   }
 411
 412   if (archive_write_header(this->Archive, e) != ARCHIVE_OK) {
 413     this->Error = cmStrCat("archive_write_header: ",
 414                            cm_archive_error_string(this->Archive));
 415     return false;
 416   }
 417
 418   // do not copy content of symlink
 419   if (!archive_entry_symlink(e)) {
 420     // Content.
 421     if (size_t size = static_cast<size_t>(archive_entry_size(e))) {
 422       return this->AddData(file, size);
 423     }
 424   }
 425   return true;
 426 }
 427
 428 bool cmArchiveWrite::AddData(const char* file, size_t size)
 429 {
 430   cmsys::ifstream fin(file, std::ios::in | std::ios::binary);
 431   if (!fin) {
 432     this->Error = cmStrCat("Error opening \"", file,
 433                            "\": ", cmSystemTools::GetLastSystemError());
 434     return false;
 435   }
 436
 437   char buffer[16384];
 438   size_t nleft = size;
 439   while (nleft > 0) {
 440     using ssize_type = std::streamsize;
 441     size_t const nnext = nleft > sizeof(buffer) ? sizeof(buffer) : nleft;
 442     ssize_type const nnext_s = static_cast<ssize_type>(nnext);
 443     fin.read(buffer, nnext_s);
 444     // Some stream libraries (older HPUX) return failure at end of
 445     // file on the last read even if some data were read.  Check
 446     // gcount instead of trusting the stream error status.
 447     if (static_cast<size_t>(fin.gcount()) != nnext) {
 448       break;
 449     }
 450     if (archive_write_data(this->Archive, buffer, nnext) != nnext_s) {
 451       this->Error = cmStrCat("archive_write_data: ",
 452                              cm_archive_error_string(this->Archive));
 453       return false;
 454     }
 455     nleft -= nnext;
 456   }
 457   if (nleft > 0) {
 458     this->Error = cmStrCat("Error reading \"", file,
 459                            "\": ", cmSystemTools::GetLastSystemError());
 460     return false;
 461   }
 462   return true;
 463 }