src/manifest.c

   1 // Copyright (C) 2009-2018 Joel Rosdahl
   2 //
   3 // This program is free software; you can redistribute it and/or modify it
   4 // under the terms of the GNU General Public License as published by the Free
   5 // Software Foundation; either version 3 of the License, or (at your option)
   6 // any later version.
   7 //
   8 // This program is distributed in the hope that it will be useful, but WITHOUT
   9 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  11 // more details.
  12 //
  13 // You should have received a copy of the GNU General Public License along with
  14 // this program; if not, write to the Free Software Foundation, Inc., 51
  15 // Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  16
  17 #include "ccache.h"
  18 #include "hashtable_itr.h"
  19 #include "hashutil.h"
  20 #include "manifest.h"
  21 #include "murmurhashneutral2.h"
  22
  23 #include <zlib.h>
  24
  25 // Sketchy specification of the manifest disk format:
  26 //
  27 // <magic>         magic number                        (4 bytes)
  28 // <version>       file format version                 (1 byte unsigned int)
  29 // <hash_size>     size of the hash fields (in bytes)  (1 byte unsigned int)
  30 // <reserved>      reserved for future use             (2 bytes)
  31 // ----------------------------------------------------------------------------
  32 // <n>             number of include file paths        (4 bytes unsigned int)
  33 // <path_0>        path to include file                (NUL-terminated string,
  34 // ...                                                  at most 1024 bytes)
  35 // <path_n-1>
  36 // ----------------------------------------------------------------------------
  37 // <n>             number of include file hash entries (4 bytes unsigned int)
  38 // <index[0]>      index of include file path          (4 bytes unsigned int)
  39 // <hash[0]>       hash of include file                (<hash_size> bytes)
  40 // <size[0]>       size of include file                (4 bytes unsigned int)
  41 // <mtime[0]>      mtime of include file               (8 bytes signed int)
  42 // <ctime[0]>      ctime of include file               (8 bytes signed int)
  43 // ...
  44 // <index[n-1]>
  45 // <hash[n-1]>
  46 // <size[n-1]>
  47 // <mtime[n-1]>
  48 // <ctime[n-1]>
  49 // ----------------------------------------------------------------------------
  50 // <n>             number of object name entries       (4 bytes unsigned int)
  51 // <m[0]>          number of include file hash indexes (4 bytes unsigned int)
  52 // <index[0][0]>   include file hash index             (4 bytes unsigned int)
  53 // ...
  54 // <index[0][m[0]-1]>
  55 // <hash[0]>       hash part of object name            (<hash_size> bytes)
  56 // <size[0]>       size part of object name            (4 bytes unsigned int)
  57 // ...
  58 // <m[n-1]>        number of include file hash indexes
  59 // <index[n-1][0]> include file hash index
  60 // ...
  61 // <index[n-1][m[n-1]]>
  62 // <hash[n-1]>
  63 // <size[n-1]>
  64
  65 static const uint32_t MAGIC = 0x63436d46U;
  66 static const uint32_t MAX_MANIFEST_ENTRIES = 100;
  67 static const uint32_t MAX_MANIFEST_FILE_INFO_ENTRIES = 10000;
  68
  69 #define ccache_static_assert(e) \
  70         do { enum { ccache_static_assert__ = 1/(e) }; } while (false)
  71
  72 struct file_info {
  73         // Index to n_files.
  74         uint32_t index;
  75         // Hash of referenced file.
  76         uint8_t hash[16];
  77         // Size of referenced file.
  78         uint32_t size;
  79         // mtime of referenced file.
  80         int64_t mtime;
  81         // ctime of referenced file.
  82         int64_t ctime;
  83 };
  84
  85 struct object {
  86         // Number of entries in file_info_indexes.
  87         uint32_t n_file_info_indexes;
  88         // Indexes to file_infos.
  89         uint32_t *file_info_indexes;
  90         // Hash of the object itself.
  91         struct file_hash hash;
  92 };
  93
  94 struct manifest {
  95         // Version of decoded file.
  96         uint8_t version;
  97
  98         // Reserved for future use.
  99         uint16_t reserved;
 100
 101         // Size of hash fields (in bytes).
 102         uint8_t hash_size;
 103
 104         // Referenced include files.
 105         uint32_t n_files;
 106         char **files;
 107
 108         // Information about referenced include files.
 109         uint32_t n_file_infos;
 110         struct file_info *file_infos;
 111
 112         // Object names plus references to include file hashes.
 113         uint32_t n_objects;
 114         struct object *objects;
 115 };
 116
 117 struct file_stats {
 118         uint32_t size;
 119         int64_t mtime;
 120         int64_t ctime;
 121 };
 122
 123 static unsigned int
 124 hash_from_file_info(void *key)
 125 {
 126         ccache_static_assert(sizeof(struct file_info) == 40); // No padding.
 127         return murmurhashneutral2(key, sizeof(struct file_info), 0);
 128 }
 129
 130 static int
 131 file_infos_equal(void *key1, void *key2)
 132 {
 133         struct file_info *fi1 = (struct file_info *)key1;
 134         struct file_info *fi2 = (struct file_info *)key2;
 135         return fi1->index == fi2->index
 136                && memcmp(fi1->hash, fi2->hash, 16) == 0
 137                && fi1->size == fi2->size
 138                && fi1->mtime == fi2->mtime
 139                && fi1->ctime == fi2->ctime;
 140 }
 141
 142 static void
 143 free_manifest(struct manifest *mf)
 144 {
 145         for (uint32_t i = 0; i < mf->n_files; i++) {
 146                 free(mf->files[i]);
 147         }
 148         free(mf->files);
 149         free(mf->file_infos);
 150         for (uint32_t i = 0; i < mf->n_objects; i++) {
 151                 free(mf->objects[i].file_info_indexes);
 152         }
 153         free(mf->objects);
 154         free(mf);
 155 }
 156
 157 #define READ_BYTE(var) \
 158         do { \
 159                 int ch_ = gzgetc(f); \
 160                 if (ch_ == EOF) { \
 161                         goto error; \
 162                 } \
 163                 (var) = ch_ & 0xFF; \
 164         } while (false)
 165
 166 #define READ_INT(size, var) \
 167         do { \
 168                 uint64_t u_ = 0; \
 169                 for (size_t i_ = 0; i_ < (size); i_++) { \
 170                         int ch_ = gzgetc(f); \
 171                         if (ch_ == EOF) { \
 172                                 goto error; \
 173                         } \
 174                         u_ <<= 8; \
 175                         u_ |= ch_ & 0xFF; \
 176                 } \
 177                 (var) = u_; \
 178         } while (false)
 179
 180 #define READ_STR(var) \
 181         do { \
 182                 char buf_[1024]; \
 183                 size_t i_; \
 184                 for (i_ = 0; i_ < sizeof(buf_); i_++) { \
 185                         int ch_ = gzgetc(f); \
 186                         if (ch_ == EOF) { \
 187                                 goto error; \
 188                         } \
 189                         buf_[i_] = ch_; \
 190                         if (ch_ == '\0') { \
 191                                 break; \
 192                         } \
 193                 } \
 194                 if (i_ == sizeof(buf_)) { \
 195                         goto error; \
 196                 } \
 197                 (var) = x_strdup(buf_); \
 198         } while (false)
 199
 200 #define READ_BYTES(n, var) \
 201         do { \
 202                 for (size_t i_ = 0; i_ < (n); i_++) { \
 203                         int ch_ = gzgetc(f); \
 204                         if (ch_ == EOF) { \
 205                                 goto error; \
 206                         } \
 207                         (var)[i_] = ch_; \
 208                 } \
 209         } while (false)
 210
 211 static struct manifest *
 212 create_empty_manifest(void)
 213 {
 214         struct manifest *mf = x_malloc(sizeof(*mf));
 215         mf->hash_size = 16;
 216         mf->n_files = 0;
 217         mf->files = NULL;
 218         mf->n_file_infos = 0;
 219         mf->file_infos = NULL;
 220         mf->n_objects = 0;
 221         mf->objects = NULL;
 222
 223         return mf;
 224 }
 225
 226 static struct manifest *
 227 read_manifest(gzFile f)
 228 {
 229         struct manifest *mf = create_empty_manifest();
 230
 231         uint32_t magic;
 232         READ_INT(4, magic);
 233         if (magic != MAGIC) {
 234                 cc_log("Manifest file has bad magic number %u", magic);
 235                 goto error;
 236         }
 237
 238         READ_BYTE(mf->version);
 239         if (mf->version != MANIFEST_VERSION) {
 240                 cc_log("Manifest file has unknown version %u", mf->version);
 241                 goto error;
 242         }
 243
 244         READ_BYTE(mf->hash_size);
 245         if (mf->hash_size != 16) {
 246                 // Temporary measure until we support different hash algorithms.
 247                 cc_log("Manifest file has unsupported hash size %u", mf->hash_size);
 248                 goto error;
 249         }
 250
 251         READ_INT(2, mf->reserved);
 252
 253         READ_INT(4, mf->n_files);
 254         mf->files = x_calloc(mf->n_files, sizeof(*mf->files));
 255         for (uint32_t i = 0; i < mf->n_files; i++) {
 256                 READ_STR(mf->files[i]);
 257         }
 258
 259         READ_INT(4, mf->n_file_infos);
 260         mf->file_infos = x_calloc(mf->n_file_infos, sizeof(*mf->file_infos));
 261         for (uint32_t i = 0; i < mf->n_file_infos; i++) {
 262                 READ_INT(4, mf->file_infos[i].index);
 263                 READ_BYTES(mf->hash_size, mf->file_infos[i].hash);
 264                 READ_INT(4, mf->file_infos[i].size);
 265                 READ_INT(8, mf->file_infos[i].mtime);
 266                 READ_INT(8, mf->file_infos[i].ctime);
 267         }
 268
 269         READ_INT(4, mf->n_objects);
 270         mf->objects = x_calloc(mf->n_objects, sizeof(*mf->objects));
 271         for (uint32_t i = 0; i < mf->n_objects; i++) {
 272                 READ_INT(4, mf->objects[i].n_file_info_indexes);
 273                 mf->objects[i].file_info_indexes =
 274                         x_calloc(mf->objects[i].n_file_info_indexes,
 275                                  sizeof(*mf->objects[i].file_info_indexes));
 276                 for (uint32_t j = 0; j < mf->objects[i].n_file_info_indexes; j++) {
 277                         READ_INT(4, mf->objects[i].file_info_indexes[j]);
 278                 }
 279                 READ_BYTES(mf->hash_size, mf->objects[i].hash.hash);
 280                 READ_INT(4, mf->objects[i].hash.size);
 281         }
 282
 283         return mf;
 284
 285 error:
 286         cc_log("Corrupt manifest file");
 287         free_manifest(mf);
 288         return NULL;
 289 }
 290
 291 #define WRITE_INT(size, var) \
 292         do { \
 293                 uint64_t u_ = (var); \
 294                 uint8_t ch_; \
 295                 size_t i_; \
 296                 for (i_ = 0; i_ < (size); i_++) { \
 297                         ch_ = (u_ >> (8 * ((size) - i_ - 1))); \
 298                         if (gzputc(f, ch_) == EOF) { \
 299                                 goto error; \
 300                         } \
 301                 } \
 302         } while (false)
 303
 304 #define WRITE_STR(var) \
 305         do { \
 306                 if (gzputs(f, var) == EOF || gzputc(f, '\0') == EOF) { \
 307                         goto error; \
 308                 } \
 309         } while (false)
 310
 311 #define WRITE_BYTES(n, var) \
 312         do { \
 313                 size_t i_; \
 314                 for (i_ = 0; i_ < (n); i_++) { \
 315                         if (gzputc(f, (var)[i_]) == EOF) { \
 316                                 goto error; \
 317                         } \
 318                 } \
 319         } while (false)
 320
 321 static int
 322 write_manifest(gzFile f, const struct manifest *mf)
 323 {
 324         WRITE_INT(4, MAGIC);
 325         WRITE_INT(1, MANIFEST_VERSION);
 326         WRITE_INT(1, 16);
 327         WRITE_INT(2, 0);
 328
 329         WRITE_INT(4, mf->n_files);
 330         for (uint32_t i = 0; i < mf->n_files; i++) {
 331                 WRITE_STR(mf->files[i]);
 332         }
 333
 334         WRITE_INT(4, mf->n_file_infos);
 335         for (uint32_t i = 0; i < mf->n_file_infos; i++) {
 336                 WRITE_INT(4, mf->file_infos[i].index);
 337                 WRITE_BYTES(mf->hash_size, mf->file_infos[i].hash);
 338                 WRITE_INT(4, mf->file_infos[i].size);
 339                 WRITE_INT(8, mf->file_infos[i].mtime);
 340                 WRITE_INT(8, mf->file_infos[i].ctime);
 341         }
 342
 343         WRITE_INT(4, mf->n_objects);
 344         for (uint32_t i = 0; i < mf->n_objects; i++) {
 345                 WRITE_INT(4, mf->objects[i].n_file_info_indexes);
 346                 for (uint32_t j = 0; j < mf->objects[i].n_file_info_indexes; j++) {
 347                         WRITE_INT(4, mf->objects[i].file_info_indexes[j]);
 348                 }
 349                 WRITE_BYTES(mf->hash_size, mf->objects[i].hash.hash);
 350                 WRITE_INT(4, mf->objects[i].hash.size);
 351         }
 352
 353         return 1;
 354
 355 error:
 356         cc_log("Error writing to manifest file");
 357         return 0;
 358 }
 359
 360 static int
 361 verify_object(struct conf *conf, struct manifest *mf, struct object *obj,
 362               struct hashtable *stated_files, struct hashtable *hashed_files)
 363 {
 364         for (uint32_t i = 0; i < obj->n_file_info_indexes; i++) {
 365                 struct file_info *fi = &mf->file_infos[obj->file_info_indexes[i]];
 366                 char *path = mf->files[fi->index];
 367                 struct file_stats *st = hashtable_search(stated_files, path);
 368                 if (!st) {
 369                         struct stat file_stat;
 370                         if (x_stat(path, &file_stat) != 0) {
 371                                 return 0;
 372                         }
 373                         st = x_malloc(sizeof(*st));
 374                         st->size = file_stat.st_size;
 375                         st->mtime = file_stat.st_mtime;
 376                         st->ctime = file_stat.st_ctime;
 377                         hashtable_insert(stated_files, x_strdup(path), st);
 378                 }
 379
 380                 if (fi->size != st->size) {
 381                         return 0;
 382                 }
 383
 384                 // Clang stores the mtime of the included files in the precompiled header,
 385                 // and will error out if that header is later used without rebuilding.
 386                 if ((guessed_compiler == GUESSED_CLANG
 387                      || guessed_compiler == GUESSED_UNKNOWN)
 388                     && output_is_precompiled_header
 389                     && fi->mtime != st->mtime) {
 390                         cc_log("Precompiled header includes %s, which has a new mtime", path);
 391                         return 0;
 392                 }
 393
 394                 if (conf->sloppiness & SLOPPY_FILE_STAT_MATCHES) {
 395                         if (!(conf->sloppiness & SLOPPY_FILE_STAT_MATCHES_CTIME)) {
 396                                 if (fi->mtime == st->mtime && fi->ctime == st->ctime) {
 397                                         cc_log("mtime/ctime hit for %s", path);
 398                                         continue;
 399                                 } else {
 400                                         cc_log("mtime/ctime miss for %s", path);
 401                                 }
 402                         } else {
 403                                 if (fi->mtime == st->mtime) {
 404                                         cc_log("mtime hit for %s", path);
 405                                         continue;
 406                                 } else {
 407                                         cc_log("mtime miss for %s", path);
 408                                 }
 409                         }
 410                 }
 411
 412                 struct file_hash *actual = hashtable_search(hashed_files, path);
 413                 if (!actual) {
 414                         struct hash *hash = hash_init();
 415                         int result = hash_source_code_file(conf, hash, path);
 416                         if (result & HASH_SOURCE_CODE_ERROR) {
 417                                 cc_log("Failed hashing %s", path);
 418                                 hash_free(hash);
 419                                 return 0;
 420                         }
 421                         if (result & HASH_SOURCE_CODE_FOUND_TIME) {
 422                                 hash_free(hash);
 423                                 return 0;
 424                         }
 425                         actual = x_malloc(sizeof(*actual));
 426                         hash_result_as_bytes(hash, actual->hash);
 427                         actual->size = hash_input_size(hash);
 428                         hashtable_insert(hashed_files, x_strdup(path), actual);
 429                         hash_free(hash);
 430                 }
 431                 if (memcmp(fi->hash, actual->hash, mf->hash_size) != 0
 432                     || fi->size != actual->size) {
 433                         return 0;
 434                 }
 435         }
 436
 437         return 1;
 438 }
 439
 440 static struct hashtable *
 441 create_string_index_map(char **strings, uint32_t len)
 442 {
 443         struct hashtable *h =
 444                 create_hashtable(1000, hash_from_string, strings_equal);
 445         for (uint32_t i = 0; i < len; i++) {
 446                 uint32_t *index = x_malloc(sizeof(*index));
 447                 *index = i;
 448                 hashtable_insert(h, x_strdup(strings[i]), index);
 449         }
 450         return h;
 451 }
 452
 453 static struct hashtable *
 454 create_file_info_index_map(struct file_info *infos, uint32_t len)
 455 {
 456         struct hashtable *h =
 457                 create_hashtable(1000, hash_from_file_info, file_infos_equal);
 458         for (uint32_t i = 0; i < len; i++) {
 459                 struct file_info *fi = x_malloc(sizeof(*fi));
 460                 *fi = infos[i];
 461                 uint32_t *index = x_malloc(sizeof(*index));
 462                 *index = i;
 463                 hashtable_insert(h, fi, index);
 464         }
 465         return h;
 466 }
 467
 468 static uint32_t
 469 get_include_file_index(struct manifest *mf, char *path,
 470                        struct hashtable *mf_files)
 471 {
 472         uint32_t *index = hashtable_search(mf_files, path);
 473         if (index) {
 474                 return *index;
 475         }
 476
 477         uint32_t n = mf->n_files;
 478         mf->files = x_realloc(mf->files, (n + 1) * sizeof(*mf->files));
 479         mf->n_files++;
 480         mf->files[n] = x_strdup(path);
 481         return n;
 482 }
 483
 484 static uint32_t
 485 get_file_hash_index(struct manifest *mf,
 486                     char *path,
 487                     struct file_hash *file_hash,
 488                     struct hashtable *mf_files,
 489                     struct hashtable *mf_file_infos)
 490 {
 491         struct file_info fi;
 492         fi.index = get_include_file_index(mf, path, mf_files);
 493         memcpy(fi.hash, file_hash->hash, sizeof(fi.hash));
 494         fi.size = file_hash->size;
 495
 496         // file_stat.st_{m,c}time has a resolution of 1 second, so we can cache the
 497         // file's mtime and ctime only if they're at least one second older than
 498         // time_of_compilation.
 499         //
 500         // st->ctime may be 0, so we have to check time_of_compilation against
 501         // MAX(mtime, ctime).
 502
 503         struct stat file_stat;
 504         if (stat(path, &file_stat) != -1
 505             && time_of_compilation > MAX(file_stat.st_mtime, file_stat.st_ctime)) {
 506                 fi.mtime = file_stat.st_mtime;
 507                 fi.ctime = file_stat.st_ctime;
 508         } else {
 509                 fi.mtime = -1;
 510                 fi.ctime = -1;
 511         }
 512
 513         uint32_t *fi_index = hashtable_search(mf_file_infos, &fi);
 514         if (fi_index) {
 515                 return *fi_index;
 516         }
 517
 518         uint32_t n = mf->n_file_infos;
 519         mf->file_infos = x_realloc(mf->file_infos, (n + 1) * sizeof(*mf->file_infos));
 520         mf->n_file_infos++;
 521         mf->file_infos[n] = fi;
 522         return n;
 523 }
 524
 525 static void
 526 add_file_info_indexes(uint32_t *indexes, uint32_t size,
 527                       struct manifest *mf, struct hashtable *included_files)
 528 {
 529         if (size == 0) {
 530                 return;
 531         }
 532
 533         // path --> index
 534         struct hashtable *mf_files =
 535                 create_string_index_map(mf->files, mf->n_files);
 536         // struct file_info --> index
 537         struct hashtable *mf_file_infos =
 538                 create_file_info_index_map(mf->file_infos, mf->n_file_infos);
 539         struct hashtable_itr *iter = hashtable_iterator(included_files);
 540         uint32_t i = 0;
 541         do {
 542                 char *path = hashtable_iterator_key(iter);
 543                 struct file_hash *file_hash = hashtable_iterator_value(iter);
 544                 indexes[i] = get_file_hash_index(mf, path, file_hash, mf_files,
 545                                                  mf_file_infos);
 546                 i++;
 547         } while (hashtable_iterator_advance(iter));
 548         assert(i == size);
 549
 550         hashtable_destroy(mf_file_infos, 1);
 551         hashtable_destroy(mf_files, 1);
 552 }
 553
 554 static void
 555 add_object_entry(struct manifest *mf,
 556                  struct file_hash *object_hash,
 557                  struct hashtable *included_files)
 558 {
 559         uint32_t n_objs = mf->n_objects;
 560         mf->objects = x_realloc(mf->objects, (n_objs + 1) * sizeof(*mf->objects));
 561         mf->n_objects++;
 562         struct object *obj = &mf->objects[n_objs];
 563
 564         uint32_t n_fii = hashtable_count(included_files);
 565         obj->n_file_info_indexes = n_fii;
 566         obj->file_info_indexes = x_malloc(n_fii * sizeof(*obj->file_info_indexes));
 567         add_file_info_indexes(obj->file_info_indexes, n_fii, mf, included_files);
 568         memcpy(obj->hash.hash, object_hash->hash, mf->hash_size);
 569         obj->hash.size = object_hash->size;
 570 }
 571
 572 // Try to get the object hash from a manifest file. Caller frees. Returns NULL
 573 // on failure.
 574 struct file_hash *
 575 manifest_get(struct conf *conf, const char *manifest_path)
 576 {
 577         gzFile f = NULL;
 578         struct manifest *mf = NULL;
 579         struct hashtable *hashed_files = NULL; // path --> struct file_hash
 580         struct hashtable *stated_files = NULL; // path --> struct file_stats
 581         struct file_hash *fh = NULL;
 582
 583         int fd = open(manifest_path, O_RDONLY | O_BINARY);
 584         if (fd == -1) {
 585                 // Cache miss.
 586                 cc_log("No such manifest file");
 587                 goto out;
 588         }
 589         f = gzdopen(fd, "rb");
 590         if (!f) {
 591                 close(fd);
 592                 cc_log("Failed to gzdopen manifest file");
 593                 goto out;
 594         }
 595         mf = read_manifest(f);
 596         if (!mf) {
 597                 cc_log("Error reading manifest file");
 598                 goto out;
 599         }
 600
 601         hashed_files = create_hashtable(1000, hash_from_string, strings_equal);
 602         stated_files = create_hashtable(1000, hash_from_string, strings_equal);
 603
 604         // Check newest object first since it's a bit more likely to match.
 605         for (uint32_t i = mf->n_objects; i > 0; i--) {
 606                 if (verify_object(conf, mf, &mf->objects[i - 1],
 607                                   stated_files, hashed_files)) {
 608                         fh = x_malloc(sizeof(*fh));
 609                         *fh = mf->objects[i - 1].hash;
 610                         goto out;
 611                 }
 612         }
 613
 614 out:
 615         if (hashed_files) {
 616                 hashtable_destroy(hashed_files, 1);
 617         }
 618         if (stated_files) {
 619                 hashtable_destroy(stated_files, 1);
 620         }
 621         if (f) {
 622                 gzclose(f);
 623         }
 624         if (mf) {
 625                 free_manifest(mf);
 626         }
 627         return fh;
 628 }
 629
 630 // Put the object name into a manifest file given a set of included files.
 631 // Returns true on success, otherwise false.
 632 bool
 633 manifest_put(const char *manifest_path, struct file_hash *object_hash,
 634              struct hashtable *included_files)
 635 {
 636         int ret = 0;
 637         gzFile f2 = NULL;
 638         struct manifest *mf = NULL;
 639         char *tmp_file = NULL;
 640
 641         // We don't bother to acquire a lock when writing the manifest to disk. A
 642         // race between two processes will only result in one lost entry, which is
 643         // not a big deal, and it's also very unlikely.
 644
 645         int fd1 = open(manifest_path, O_RDONLY | O_BINARY);
 646         if (fd1 == -1) {
 647                 // New file.
 648                 mf = create_empty_manifest();
 649         } else {
 650                 gzFile f1 = gzdopen(fd1, "rb");
 651                 if (!f1) {
 652                         cc_log("Failed to gzdopen manifest file");
 653                         close(fd1);
 654                         goto out;
 655                 }
 656                 mf = read_manifest(f1);
 657                 gzclose(f1);
 658                 if (!mf) {
 659                         cc_log("Failed to read manifest file; deleting it");
 660                         x_unlink(manifest_path);
 661                         mf = create_empty_manifest();
 662                 }
 663         }
 664
 665         if (mf->n_objects > MAX_MANIFEST_ENTRIES) {
 666                 // Normally, there shouldn't be many object entries in the manifest since
 667                 // new entries are added only if an include file has changed but not the
 668                 // source file, and you typically change source files more often than
 669                 // header files. However, it's certainly possible to imagine cases where
 670                 // the manifest will grow large (for instance, a generated header file that
 671                 // changes for every build), and this must be taken care of since
 672                 // processing an ever growing manifest eventually will take too much time.
 673                 // A good way of solving this would be to maintain the object entries in
 674                 // LRU order and discarding the old ones. An easy way is to throw away all
 675                 // entries when there are too many. Let's do that for now.
 676                 cc_log("More than %u entries in manifest file; discarding",
 677                        MAX_MANIFEST_ENTRIES);
 678                 free_manifest(mf);
 679                 mf = create_empty_manifest();
 680         } else if (mf->n_file_infos > MAX_MANIFEST_FILE_INFO_ENTRIES) {
 681                 // Rarely, file_info entries can grow large in pathological cases where
 682                 // many included files change, but the main file does not. This also puts
 683                 // an upper bound on the number of file_info entries.
 684                 cc_log("More than %u file_info entries in manifest file; discarding",
 685                        MAX_MANIFEST_FILE_INFO_ENTRIES);
 686                 free_manifest(mf);
 687                 mf = create_empty_manifest();
 688         }
 689
 690         tmp_file = format("%s.tmp", manifest_path);
 691         int fd2 = create_tmp_fd(&tmp_file);
 692         f2 = gzdopen(fd2, "wb");
 693         if (!f2) {
 694                 cc_log("Failed to gzdopen %s", tmp_file);
 695                 goto out;
 696         }
 697
 698         add_object_entry(mf, object_hash, included_files);
 699         if (write_manifest(f2, mf)) {
 700                 gzclose(f2);
 701                 f2 = NULL;
 702                 if (x_rename(tmp_file, manifest_path) == 0) {
 703                         ret = 1;
 704                 } else {
 705                         cc_log("Failed to rename %s to %s", tmp_file, manifest_path);
 706                         goto out;
 707                 }
 708         } else {
 709                 cc_log("Failed to write manifest file");
 710                 goto out;
 711         }
 712
 713 out:
 714         if (mf) {
 715                 free_manifest(mf);
 716         }
 717         if (tmp_file) {
 718                 free(tmp_file);
 719         }
 720         if (f2) {
 721                 gzclose(f2);
 722         }
 723         return ret;
 724 }
 725
 726 bool
 727 manifest_dump(const char *manifest_path, FILE *stream)
 728 {
 729         struct manifest *mf = NULL;
 730         gzFile f = NULL;
 731         bool ret = false;
 732
 733         int fd = open(manifest_path, O_RDONLY | O_BINARY);
 734         if (fd == -1) {
 735                 fprintf(stderr, "No such manifest file: %s\n", manifest_path);
 736                 goto out;
 737         }
 738         f = gzdopen(fd, "rb");
 739         if (!f) {
 740                 fprintf(stderr, "Failed to dzopen manifest file\n");
 741                 close(fd);
 742                 goto out;
 743         }
 744         mf = read_manifest(f);
 745         if (!mf) {
 746                 fprintf(stderr, "Error reading manifest file\n");
 747                 goto out;
 748         }
 749
 750         fprintf(stream, "Magic: %c%c%c%c\n",
 751                 (MAGIC >> 24) & 0xFF,
 752                 (MAGIC >> 16) & 0xFF,
 753                 (MAGIC >> 8) & 0xFF,
 754                 MAGIC & 0xFF);
 755         fprintf(stream, "Version: %u\n", mf->version);
 756         fprintf(stream, "Hash size: %u\n", (unsigned)mf->hash_size);
 757         fprintf(stream, "Reserved field: %u\n", (unsigned)mf->reserved);
 758         fprintf(stream, "File paths (%u):\n", (unsigned)mf->n_files);
 759         for (unsigned i = 0; i < mf->n_files; ++i) {
 760                 fprintf(stream, "  %u: %s\n", i, mf->files[i]);
 761         }
 762         fprintf(stream, "File infos (%u):\n", (unsigned)mf->n_file_infos);
 763         for (unsigned i = 0; i < mf->n_file_infos; ++i) {
 764                 char *hash;
 765                 fprintf(stream, "  %u:\n", i);
 766                 fprintf(stream, "    Path index: %u\n", mf->file_infos[i].index);
 767                 hash = format_hash_as_string(mf->file_infos[i].hash, -1);
 768                 fprintf(stream, "    Hash: %s\n", hash);
 769                 free(hash);
 770                 fprintf(stream, "    Size: %u\n", mf->file_infos[i].size);
 771                 fprintf(stream, "    Mtime: %lld\n", (long long)mf->file_infos[i].mtime);
 772                 fprintf(stream, "    Ctime: %lld\n", (long long)mf->file_infos[i].ctime);
 773         }
 774         fprintf(stream, "Results (%u):\n", (unsigned)mf->n_objects);
 775         for (unsigned i = 0; i < mf->n_objects; ++i) {
 776                 char *hash;
 777                 fprintf(stream, "  %u:\n", i);
 778                 fprintf(stream, "    File info indexes:");
 779                 for (unsigned j = 0; j < mf->objects[i].n_file_info_indexes; ++j) {
 780                         fprintf(stream, " %u", mf->objects[i].file_info_indexes[j]);
 781                 }
 782                 fprintf(stream, "\n");
 783                 hash = format_hash_as_string(mf->objects[i].hash.hash, -1);
 784                 fprintf(stream, "    Hash: %s\n", hash);
 785                 free(hash);
 786                 fprintf(stream, "    Size: %u\n", (unsigned)mf->objects[i].hash.size);
 787         }
 788
 789         ret = true;
 790
 791 out:
 792         if (mf) {
 793                 free_manifest(mf);
 794         }
 795         if (f) {
 796                 gzclose(f);
 797         }
 798         return ret;
 799 }