Imported Upstream version 2.25.1
[platform/upstream/git.git] / packfile.c
index 7cd45aa..7e7c04e 100644 (file)
@@ -6,7 +6,6 @@
 #include "mergesort.h"
 #include "packfile.h"
 #include "delta.h"
-#include "list.h"
 #include "streaming.h"
 #include "sha1-lookup.h"
 #include "commit.h"
 #include "tree-walk.h"
 #include "tree.h"
 #include "object-store.h"
+#include "midx.h"
+#include "commit-graph.h"
+#include "promisor-remote.h"
 
 char *odb_pack_name(struct strbuf *buf,
-                   const unsigned char *sha1,
+                   const unsigned char *hash,
                    const char *ext)
 {
        strbuf_reset(buf);
        strbuf_addf(buf, "%s/pack/pack-%s.%s", get_object_directory(),
-                   sha1_to_hex(sha1), ext);
+                   hash_to_hex(hash), ext);
        return buf->buf;
 }
 
@@ -79,10 +81,8 @@ void pack_report(void)
 static int check_packed_git_idx(const char *path, struct packed_git *p)
 {
        void *idx_map;
-       struct pack_idx_header *hdr;
        size_t idx_size;
-       uint32_t version, nr, i, *index;
-       int fd = git_open(path);
+       int fd = git_open(path), ret;
        struct stat st;
        const unsigned int hashsz = the_hash_algo->rawsz;
 
@@ -100,16 +100,32 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)
        idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
        close(fd);
 
-       hdr = idx_map;
+       ret = load_idx(path, hashsz, idx_map, idx_size, p);
+
+       if (ret)
+               munmap(idx_map, idx_size);
+
+       return ret;
+}
+
+int load_idx(const char *path, const unsigned int hashsz, void *idx_map,
+            size_t idx_size, struct packed_git *p)
+{
+       struct pack_idx_header *hdr = idx_map;
+       uint32_t version, nr, i, *index;
+
+       if (idx_size < 4 * 256 + hashsz + hashsz)
+               return error("index file %s is too small", path);
+       if (idx_map == NULL)
+               return error("empty data");
+
        if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
                version = ntohl(hdr->idx_version);
-               if (version < 2 || version > 2) {
-                       munmap(idx_map, idx_size);
+               if (version < 2 || version > 2)
                        return error("index file %s is version %"PRIu32
                                     " and is not supported by this binary"
                                     " (try upgrading GIT to a newer version)",
                                     path, version);
-               }
        } else
                version = 1;
 
@@ -119,10 +135,8 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)
                index += 2;  /* skip index header */
        for (i = 0; i < 256; i++) {
                uint32_t n = ntohl(index[i]);
-               if (n < nr) {
-                       munmap(idx_map, idx_size);
+               if (n < nr)
                        return error("non-monotonic index %s", path);
-               }
                nr = n;
        }
 
@@ -134,10 +148,8 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)
                 *  - hash of the packfile
                 *  - file checksum
                 */
-               if (idx_size != 4*256 + nr * (hashsz + 4) + hashsz + hashsz) {
-                       munmap(idx_map, idx_size);
+               if (idx_size != 4 * 256 + nr * (hashsz + 4) + hashsz + hashsz)
                        return error("wrong index v1 file size in %s", path);
-               }
        } else if (version == 2) {
                /*
                 * Minimum size:
@@ -156,20 +168,16 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)
                unsigned long max_size = min_size;
                if (nr)
                        max_size += (nr - 1)*8;
-               if (idx_size < min_size || idx_size > max_size) {
-                       munmap(idx_map, idx_size);
+               if (idx_size < min_size || idx_size > max_size)
                        return error("wrong index v2 file size in %s", path);
-               }
                if (idx_size != min_size &&
                    /*
                     * make sure we can deal with large pack offsets.
                     * 31-bit signed offset won't be enough, neither
                     * 32-bit unsigned one will be.
                     */
-                   (sizeof(off_t) <= 4)) {
-                       munmap(idx_map, idx_size);
+                   (sizeof(off_t) <= 4))
                        return error("pack too large for current definition of off_t in %s", path);
-               }
        }
 
        p->index_version = version;
@@ -196,6 +204,23 @@ int open_pack_index(struct packed_git *p)
        return ret;
 }
 
+uint32_t get_pack_fanout(struct packed_git *p, uint32_t value)
+{
+       const uint32_t *level1_ofs = p->index_data;
+
+       if (!level1_ofs) {
+               if (open_pack_index(p))
+                       return 0;
+               level1_ofs = p->index_data;
+       }
+
+       if (p->index_version > 1) {
+               level1_ofs += 2;
+       }
+
+       return ntohl(level1_ofs[value]);
+}
+
 static struct packed_git *alloc_packed_git(int extra)
 {
        struct packed_git *p = xmalloc(st_add(sizeof(*p), extra));
@@ -211,7 +236,7 @@ struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
        struct packed_git *p = alloc_packed_git(alloc);
 
        memcpy(p->pack_name, path, alloc); /* includes NUL */
-       hashcpy(p->sha1, sha1);
+       hashcpy(p->hash, sha1);
        if (check_packed_git_idx(idx_path, p)) {
                free(p);
                return NULL;
@@ -262,13 +287,6 @@ static int unuse_one_window(struct packed_git *current)
        return 0;
 }
 
-void release_pack_memory(size_t need)
-{
-       size_t cur = pack_mapped;
-       while (need >= (cur - pack_mapped) && unuse_one_window(NULL))
-               ; /* nothing */
-}
-
 void close_pack_windows(struct packed_git *p)
 {
        while (p->windows) {
@@ -285,7 +303,7 @@ void close_pack_windows(struct packed_git *p)
        }
 }
 
-static int close_pack_fd(struct packed_git *p)
+int close_pack_fd(struct packed_git *p)
 {
        if (p->pack_fd < 0)
                return 0;
@@ -312,7 +330,7 @@ void close_pack(struct packed_git *p)
        close_pack_index(p);
 }
 
-void close_all_packs(struct raw_object_store *o)
+void close_object_store(struct raw_object_store *o)
 {
        struct packed_git *p;
 
@@ -321,6 +339,41 @@ void close_all_packs(struct raw_object_store *o)
                        BUG("want to close pack marked 'do-not-close'");
                else
                        close_pack(p);
+
+       if (o->multi_pack_index) {
+               close_midx(o->multi_pack_index);
+               o->multi_pack_index = NULL;
+       }
+
+       close_commit_graph(o);
+}
+
+void unlink_pack_path(const char *pack_name, int force_delete)
+{
+       static const char *exts[] = {".pack", ".idx", ".keep", ".bitmap", ".promisor"};
+       int i;
+       struct strbuf buf = STRBUF_INIT;
+       size_t plen;
+
+       strbuf_addstr(&buf, pack_name);
+       strip_suffix_mem(buf.buf, &buf.len, ".pack");
+       plen = buf.len;
+
+       if (!force_delete) {
+               strbuf_addstr(&buf, ".keep");
+               if (!access(buf.buf, F_OK)) {
+                       strbuf_release(&buf);
+                       return;
+               }
+       }
+
+       for (i = 0; i < ARRAY_SIZE(exts); i++) {
+               strbuf_setlen(&buf, plen);
+               strbuf_addstr(&buf, exts[i]);
+               unlink(buf.buf);
+       }
+
+       strbuf_release(&buf);
 }
 
 /*
@@ -437,6 +490,16 @@ static unsigned int get_max_fd_limit(void)
 #endif
 }
 
+const char *pack_basename(struct packed_git *p)
+{
+       const char *ret = strrchr(p->pack_name, '/');
+       if (ret)
+               ret = ret + 1; /* skip past slash */
+       else
+               ret = p->pack_name; /* we only have a base */
+       return ret;
+}
+
 /*
  * Do not call this directly as this leaks p->pack_fd on error return;
  * call open_packed_git() instead.
@@ -447,12 +510,22 @@ static int open_packed_git_1(struct packed_git *p)
        struct pack_header hdr;
        unsigned char hash[GIT_MAX_RAWSZ];
        unsigned char *idx_hash;
-       long fd_flag;
        ssize_t read_result;
        const unsigned hashsz = the_hash_algo->rawsz;
 
-       if (!p->index_data && open_pack_index(p))
-               return error("packfile %s index unavailable", p->pack_name);
+       if (!p->index_data) {
+               struct multi_pack_index *m;
+               const char *pack_name = pack_basename(p);
+
+               for (m = the_repository->objects->multi_pack_index;
+                    m; m = m->next) {
+                       if (midx_contains_pack(m, pack_name))
+                               break;
+               }
+
+               if (!m && open_pack_index(p))
+                       return error("packfile %s index unavailable", p->pack_name);
+       }
 
        if (!pack_max_fds) {
                unsigned int max_fds = get_max_fd_limit();
@@ -480,16 +553,6 @@ static int open_packed_git_1(struct packed_git *p)
        } else if (p->pack_size != st.st_size)
                return error("packfile %s size changed", p->pack_name);
 
-       /* We leave these file descriptors open with sliding mmap;
-        * there is no point keeping them open across exec(), though.
-        */
-       fd_flag = fcntl(p->pack_fd, F_GETFD, 0);
-       if (fd_flag < 0)
-               return error("cannot determine file descriptor flags");
-       fd_flag |= FD_CLOEXEC;
-       if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
-               return error("cannot set FD_CLOEXEC");
-
        /* Verify we recognize this pack file format. */
        read_result = read_in_full(p->pack_fd, &hdr, sizeof(hdr));
        if (read_result < 0)
@@ -503,21 +566,24 @@ static int open_packed_git_1(struct packed_git *p)
                        " supported (try upgrading GIT to a newer version)",
                        p->pack_name, ntohl(hdr.hdr_version));
 
+       /* Skip index checking if in multi-pack-index */
+       if (!p->index_data)
+               return 0;
+
        /* Verify the pack matches its index. */
        if (p->num_objects != ntohl(hdr.hdr_entries))
                return error("packfile %s claims to have %"PRIu32" objects"
                             " while index indicates %"PRIu32" objects",
                             p->pack_name, ntohl(hdr.hdr_entries),
                             p->num_objects);
-       if (lseek(p->pack_fd, p->pack_size - hashsz, SEEK_SET) == -1)
-               return error("end of packfile %s is unavailable", p->pack_name);
-       read_result = read_in_full(p->pack_fd, hash, hashsz);
+       read_result = pread_in_full(p->pack_fd, hash, hashsz,
+                                       p->pack_size - hashsz);
        if (read_result < 0)
                return error_errno("error reading from %s", p->pack_name);
        if (read_result != hashsz)
                return error("packfile %s signature is unavailable", p->pack_name);
        idx_hash = ((unsigned char *)p->index_data) + p->index_size - hashsz * 2;
-       if (hashcmp(hash, idx_hash))
+       if (!hasheq(hash, idx_hash))
                return error("packfile %s does not match index", p->pack_name);
        return 0;
 }
@@ -586,7 +652,7 @@ unsigned char *use_pack(struct packed_git *p,
                        while (packed_git_limit < pack_mapped
                                && unuse_one_window(p))
                                ; /* nothing */
-                       win->base = xmmap(NULL, win->len,
+                       win->base = xmmap_gently(NULL, win->len,
                                PROT_READ, MAP_PRIVATE,
                                p->pack_fd, win->offset);
                        if (win->base == MAP_FAILED)
@@ -625,23 +691,12 @@ void unuse_pack(struct pack_window **w_cursor)
        }
 }
 
-static void try_to_free_pack_memory(size_t size)
-{
-       release_pack_memory(size);
-}
-
 struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
 {
-       static int have_set_try_to_free_routine;
        struct stat st;
        size_t alloc;
        struct packed_git *p;
 
-       if (!have_set_try_to_free_routine) {
-               have_set_try_to_free_routine = 1;
-               set_try_to_free_routine(try_to_free_pack_memory);
-       }
-
        /*
         * Make sure a corresponding .pack file exists and that
         * the index looks sane.
@@ -678,8 +733,8 @@ struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
        p->pack_local = local;
        p->mtime = st.st_mtime;
        if (path_len < the_hash_algo->hexsz ||
-           get_sha1_hex(path + path_len - the_hash_algo->hexsz, p->sha1))
-               hashclr(p->sha1);
+           get_sha1_hex(path + path_len - the_hash_algo->hexsz, p->hash))
+               hashclr(p->hash);
        return p;
 }
 
@@ -690,6 +745,9 @@ void install_packed_git(struct repository *r, struct packed_git *pack)
 
        pack->next = r->objects->packed_git;
        r->objects->packed_git = pack;
+
+       hashmap_entry_init(&pack->packmap_ent, strhash(pack->pack_name));
+       hashmap_add(&r->objects->pack_map, &pack->packmap_ent);
 }
 
 void (*report_garbage)(unsigned seen_bits, const char *path);
@@ -738,13 +796,14 @@ static void report_pack_garbage(struct string_list *list)
        report_helper(list, seen_bits, first, list->nr);
 }
 
-static void prepare_packed_git_one(struct repository *r, char *objdir, int local)
+void for_each_file_in_pack_dir(const char *objdir,
+                              each_file_in_pack_dir_fn fn,
+                              void *data)
 {
        struct strbuf path = STRBUF_INIT;
        size_t dirnamelen;
        DIR *dir;
        struct dirent *de;
-       struct string_list garbage = STRING_LIST_INIT_DUP;
 
        strbuf_addstr(&path, objdir);
        strbuf_addstr(&path, "/pack");
@@ -759,53 +818,85 @@ static void prepare_packed_git_one(struct repository *r, char *objdir, int local
        strbuf_addch(&path, '/');
        dirnamelen = path.len;
        while ((de = readdir(dir)) != NULL) {
-               struct packed_git *p;
-               size_t base_len;
-
                if (is_dot_or_dotdot(de->d_name))
                        continue;
 
                strbuf_setlen(&path, dirnamelen);
                strbuf_addstr(&path, de->d_name);
 
-               base_len = path.len;
-               if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
-                       /* Don't reopen a pack we already have. */
-                       for (p = r->objects->packed_git; p;
-                            p = p->next) {
-                               size_t len;
-                               if (strip_suffix(p->pack_name, ".pack", &len) &&
-                                   len == base_len &&
-                                   !memcmp(p->pack_name, path.buf, len))
-                                       break;
-                       }
-                       if (p == NULL &&
-                           /*
-                            * See if it really is a valid .idx file with
-                            * corresponding .pack file that we can map.
-                            */
-                           (p = add_packed_git(path.buf, path.len, local)) != NULL)
-                               install_packed_git(r, p);
-               }
-
-               if (!report_garbage)
-                       continue;
-
-               if (ends_with(de->d_name, ".idx") ||
-                   ends_with(de->d_name, ".pack") ||
-                   ends_with(de->d_name, ".bitmap") ||
-                   ends_with(de->d_name, ".keep") ||
-                   ends_with(de->d_name, ".promisor"))
-                       string_list_append(&garbage, path.buf);
-               else
-                       report_garbage(PACKDIR_FILE_GARBAGE, path.buf);
+               fn(path.buf, path.len, de->d_name, data);
        }
+
        closedir(dir);
-       report_pack_garbage(&garbage);
-       string_list_clear(&garbage, 0);
        strbuf_release(&path);
 }
 
+struct prepare_pack_data {
+       struct repository *r;
+       struct string_list *garbage;
+       int local;
+       struct multi_pack_index *m;
+};
+
+static void prepare_pack(const char *full_name, size_t full_name_len,
+                        const char *file_name, void *_data)
+{
+       struct prepare_pack_data *data = (struct prepare_pack_data *)_data;
+       struct packed_git *p;
+       size_t base_len = full_name_len;
+
+       if (strip_suffix_mem(full_name, &base_len, ".idx") &&
+           !(data->m && midx_contains_pack(data->m, file_name))) {
+               struct hashmap_entry hent;
+               char *pack_name = xstrfmt("%.*s.pack", (int)base_len, full_name);
+               unsigned int hash = strhash(pack_name);
+               hashmap_entry_init(&hent, hash);
+
+               /* Don't reopen a pack we already have. */
+               if (!hashmap_get(&data->r->objects->pack_map, &hent, pack_name)) {
+                       p = add_packed_git(full_name, full_name_len, data->local);
+                       if (p)
+                               install_packed_git(data->r, p);
+               }
+               free(pack_name);
+       }
+
+       if (!report_garbage)
+               return;
+
+       if (!strcmp(file_name, "multi-pack-index"))
+               return;
+       if (ends_with(file_name, ".idx") ||
+           ends_with(file_name, ".pack") ||
+           ends_with(file_name, ".bitmap") ||
+           ends_with(file_name, ".keep") ||
+           ends_with(file_name, ".promisor"))
+               string_list_append(data->garbage, full_name);
+       else
+               report_garbage(PACKDIR_FILE_GARBAGE, full_name);
+}
+
+static void prepare_packed_git_one(struct repository *r, char *objdir, int local)
+{
+       struct prepare_pack_data data;
+       struct string_list garbage = STRING_LIST_INIT_DUP;
+
+       data.m = r->objects->multi_pack_index;
+
+       /* look for the multi-pack-index for this object directory */
+       while (data.m && strcmp(data.m->object_dir, objdir))
+               data.m = data.m->next;
+
+       data.r = r;
+       data.garbage = &garbage;
+       data.local = local;
+
+       for_each_file_in_pack_dir(objdir, prepare_pack, &data);
+
+       report_pack_garbage(data.garbage);
+       string_list_clear(data.garbage, 0);
+}
+
 static void prepare_packed_git(struct repository *r);
 /*
  * Give a fast, rough count of the number of objects in the repository. This
@@ -814,22 +905,25 @@ static void prepare_packed_git(struct repository *r);
  * all unreachable objects about to be pruned, in which case they're not really
  * interesting as a measure of repo size in the first place.
  */
-unsigned long approximate_object_count(void)
+unsigned long repo_approximate_object_count(struct repository *r)
 {
-       if (!the_repository->objects->approximate_object_count_valid) {
+       if (!r->objects->approximate_object_count_valid) {
                unsigned long count;
+               struct multi_pack_index *m;
                struct packed_git *p;
 
-               prepare_packed_git(the_repository);
+               prepare_packed_git(r);
                count = 0;
-               for (p = the_repository->objects->packed_git; p; p = p->next) {
+               for (m = get_multi_pack_index(r); m; m = m->next)
+                       count += m->num_objects;
+               for (p = r->objects->packed_git; p; p = p->next) {
                        if (open_pack_index(p))
                                continue;
                        count += p->num_objects;
                }
-               the_repository->objects->approximate_object_count = count;
+               r->objects->approximate_object_count = count;
        }
-       return the_repository->objects->approximate_object_count;
+       return r->objects->approximate_object_count;
 }
 
 static void *get_next_packed_git(const void *p)
@@ -889,21 +983,30 @@ static void prepare_packed_git_mru(struct repository *r)
 
 static void prepare_packed_git(struct repository *r)
 {
-       struct alternate_object_database *alt;
+       struct object_directory *odb;
 
        if (r->objects->packed_git_initialized)
                return;
-       prepare_packed_git_one(r, r->objects->objectdir, 1);
+
        prepare_alt_odb(r);
-       for (alt = r->objects->alt_odb_list; alt; alt = alt->next)
-               prepare_packed_git_one(r, alt->path, 0);
+       for (odb = r->objects->odb; odb; odb = odb->next) {
+               int local = (odb == r->objects->odb);
+               prepare_multi_pack_index_one(r, odb->path, local);
+               prepare_packed_git_one(r, odb->path, local);
+       }
        rearrange_packed_git(r);
+
        prepare_packed_git_mru(r);
        r->objects->packed_git_initialized = 1;
 }
 
 void reprepare_packed_git(struct repository *r)
 {
+       struct object_directory *odb;
+
+       for (odb = r->objects->odb; odb; odb = odb->next)
+               odb_clear_loose_cache(odb);
+
        r->objects->approximate_object_count_valid = 0;
        r->objects->packed_git_initialized = 0;
        prepare_packed_git(r);
@@ -915,6 +1018,26 @@ struct packed_git *get_packed_git(struct repository *r)
        return r->objects->packed_git;
 }
 
+struct multi_pack_index *get_multi_pack_index(struct repository *r)
+{
+       prepare_packed_git(r);
+       return r->objects->multi_pack_index;
+}
+
+struct packed_git *get_all_packs(struct repository *r)
+{
+       struct multi_pack_index *m;
+
+       prepare_packed_git(r);
+       for (m = r->objects->multi_pack_index; m; m = m->next) {
+               uint32_t i;
+               for (i = 0; i < m->num_packs; i++)
+                       prepare_midx_pack(r, m, i);
+       }
+
+       return r->objects->packed_git;
+}
+
 struct list_head *get_packed_git_mru(struct repository *r)
 {
        prepare_packed_git(r);
@@ -1014,25 +1137,27 @@ int unpack_object_header(struct packed_git *p,
 void mark_bad_packed_object(struct packed_git *p, const unsigned char *sha1)
 {
        unsigned i;
+       const unsigned hashsz = the_hash_algo->rawsz;
        for (i = 0; i < p->num_bad_objects; i++)
-               if (!hashcmp(sha1, p->bad_object_sha1 + GIT_SHA1_RAWSZ * i))
+               if (hasheq(sha1, p->bad_object_sha1 + hashsz * i))
                        return;
        p->bad_object_sha1 = xrealloc(p->bad_object_sha1,
                                      st_mult(GIT_MAX_RAWSZ,
                                              st_add(p->num_bad_objects, 1)));
-       hashcpy(p->bad_object_sha1 + GIT_SHA1_RAWSZ * p->num_bad_objects, sha1);
+       hashcpy(p->bad_object_sha1 + hashsz * p->num_bad_objects, sha1);
        p->num_bad_objects++;
 }
 
-const struct packed_git *has_packed_and_bad(const unsigned char *sha1)
+const struct packed_git *has_packed_and_bad(struct repository *r,
+                                           const unsigned char *sha1)
 {
        struct packed_git *p;
        unsigned i;
 
-       for (p = the_repository->objects->packed_git; p; p = p->next)
+       for (p = r->objects->packed_git; p; p = p->next)
                for (i = 0; i < p->num_bad_objects; i++)
-                       if (!hashcmp(sha1,
-                                    p->bad_object_sha1 + the_hash_algo->rawsz * i))
+                       if (hasheq(sha1,
+                                  p->bad_object_sha1 + the_hash_algo->rawsz * i))
                                return p;
        return NULL;
 }
@@ -1146,7 +1271,7 @@ static enum object_type packed_to_object_type(struct repository *r,
                if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) {
                        poi_stack_alloc = alloc_nr(poi_stack_nr);
                        ALLOC_ARRAY(poi_stack, poi_stack_alloc);
-                       memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr);
+                       COPY_ARRAY(poi_stack, small_poi_stack, poi_stack_nr);
                } else {
                        ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc);
                }
@@ -1207,7 +1332,7 @@ struct delta_base_cache_key {
 };
 
 struct delta_base_cache_entry {
-       struct hashmap hash;
+       struct hashmap_entry ent;
        struct delta_base_cache_key key;
        struct list_head lru;
        void *data;
@@ -1227,7 +1352,7 @@ static unsigned int pack_entry_hash(struct packed_git *p, off_t base_offset)
 static struct delta_base_cache_entry *
 get_delta_base_cache_entry(struct packed_git *p, off_t base_offset)
 {
-       struct hashmap_entry entry;
+       struct hashmap_entry entry, *e;
        struct delta_base_cache_key key;
 
        if (!delta_base_cache.cmpfn)
@@ -1236,7 +1361,8 @@ get_delta_base_cache_entry(struct packed_git *p, off_t base_offset)
        hashmap_entry_init(&entry, pack_entry_hash(p, base_offset));
        key.p = p;
        key.base_offset = base_offset;
-       return hashmap_get(&delta_base_cache, &entry, &key);
+       e = hashmap_get(&delta_base_cache, &entry, &key);
+       return e ? container_of(e, struct delta_base_cache_entry, ent) : NULL;
 }
 
 static int delta_base_cache_key_eq(const struct delta_base_cache_key *a,
@@ -1246,11 +1372,16 @@ static int delta_base_cache_key_eq(const struct delta_base_cache_key *a,
 }
 
 static int delta_base_cache_hash_cmp(const void *unused_cmp_data,
-                                    const void *va, const void *vb,
+                                    const struct hashmap_entry *va,
+                                    const struct hashmap_entry *vb,
                                     const void *vkey)
 {
-       const struct delta_base_cache_entry *a = va, *b = vb;
+       const struct delta_base_cache_entry *a, *b;
        const struct delta_base_cache_key *key = vkey;
+
+       a = container_of(va, const struct delta_base_cache_entry, ent);
+       b = container_of(vb, const struct delta_base_cache_entry, ent);
+
        if (key)
                return !delta_base_cache_key_eq(&a->key, key);
        else
@@ -1269,7 +1400,7 @@ static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
  */
 static void detach_delta_base_cache_entry(struct delta_base_cache_entry *ent)
 {
-       hashmap_remove(&delta_base_cache, ent, &ent->key);
+       hashmap_remove(&delta_base_cache, &ent->ent, &ent->key);
        list_del(&ent->lru);
        delta_base_cached -= ent->size;
        free(ent);
@@ -1333,8 +1464,8 @@ static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
 
        if (!delta_base_cache.cmpfn)
                hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, NULL, 0);
-       hashmap_entry_init(ent, pack_entry_hash(p, base_offset));
-       hashmap_add(&delta_base_cache, ent);
+       hashmap_entry_init(&ent->ent, pack_entry_hash(p, base_offset));
+       hashmap_add(&delta_base_cache, &ent->ent);
 }
 
 int packed_object_info(struct repository *r, struct packed_git *p,
@@ -1556,8 +1687,8 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset,
                    && delta_stack == small_delta_stack) {
                        delta_stack_alloc = alloc_nr(delta_stack_nr);
                        ALLOC_ARRAY(delta_stack, delta_stack_alloc);
-                       memcpy(delta_stack, small_delta_stack,
-                              sizeof(*delta_stack)*delta_stack_nr);
+                       COPY_ARRAY(delta_stack, small_delta_stack,
+                                  delta_stack_nr);
                } else {
                        ALLOC_GROW(delta_stack, delta_stack_nr+1, delta_stack_alloc);
                }
@@ -1830,8 +1961,8 @@ static int fill_pack_entry(const struct object_id *oid,
        if (p->num_bad_objects) {
                unsigned i;
                for (i = 0; i < p->num_bad_objects; i++)
-                       if (!hashcmp(oid->hash,
-                                    p->bad_object_sha1 + the_hash_algo->rawsz * i))
+                       if (hasheq(oid->hash,
+                                  p->bad_object_sha1 + the_hash_algo->rawsz * i))
                                return 0;
        }
 
@@ -1856,14 +1987,20 @@ static int fill_pack_entry(const struct object_id *oid,
 int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e)
 {
        struct list_head *pos;
+       struct multi_pack_index *m;
 
        prepare_packed_git(r);
-       if (!r->objects->packed_git)
+       if (!r->objects->packed_git && !r->objects->multi_pack_index)
                return 0;
 
+       for (m = r->objects->multi_pack_index; m; m = m->next) {
+               if (fill_midx_entry(r, oid, e, m))
+                       return 1;
+       }
+
        list_for_each(pos, &r->objects->packed_git_mru) {
                struct packed_git *p = list_entry(pos, struct packed_git, mru);
-               if (fill_pack_entry(oid, e, p)) {
+               if (!p->multi_pack_index && fill_pack_entry(oid, e, p)) {
                        list_move(&p->mru, &r->objects->packed_git_mru);
                        return 1;
                }
@@ -1885,33 +2022,47 @@ int has_pack_index(const unsigned char *sha1)
        return 1;
 }
 
-int for_each_object_in_pack(struct packed_git *p, each_packed_object_fn cb, void *data)
+int for_each_object_in_pack(struct packed_git *p,
+                           each_packed_object_fn cb, void *data,
+                           enum for_each_object_flags flags)
 {
        uint32_t i;
        int r = 0;
 
+       if (flags & FOR_EACH_OBJECT_PACK_ORDER) {
+               if (load_pack_revindex(p))
+                       return -1;
+       }
+
        for (i = 0; i < p->num_objects; i++) {
+               uint32_t pos;
                struct object_id oid;
 
-               if (!nth_packed_object_oid(&oid, p, i))
+               if (flags & FOR_EACH_OBJECT_PACK_ORDER)
+                       pos = p->revindex[i].nr;
+               else
+                       pos = i;
+
+               if (!nth_packed_object_oid(&oid, p, pos))
                        return error("unable to get sha1 of object %u in %s",
-                                    i, p->pack_name);
+                                    pos, p->pack_name);
 
-               r = cb(&oid, p, i, data);
+               r = cb(&oid, p, pos, data);
                if (r)
                        break;
        }
        return r;
 }
 
-int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags)
+int for_each_packed_object(each_packed_object_fn cb, void *data,
+                          enum for_each_object_flags flags)
 {
        struct packed_git *p;
        int r = 0;
        int pack_errors = 0;
 
        prepare_packed_git(the_repository);
-       for (p = the_repository->objects->packed_git; p; p = p->next) {
+       for (p = get_all_packs(the_repository); p; p = p->next) {
                if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local)
                        continue;
                if ((flags & FOR_EACH_OBJECT_PROMISOR_ONLY) &&
@@ -1921,7 +2072,7 @@ int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags)
                        pack_errors = 1;
                        continue;
                }
-               r = for_each_object_in_pack(p, cb, data);
+               r = for_each_object_in_pack(p, cb, data, flags);
                if (r)
                        break;
        }
@@ -1934,7 +2085,7 @@ static int add_promisor_object(const struct object_id *oid,
                               void *set_)
 {
        struct oidset *set = set_;
-       struct object *obj = parse_object(oid);
+       struct object *obj = parse_object(the_repository, oid);
        if (!obj)
                return 1;
 
@@ -1955,7 +2106,7 @@ static int add_promisor_object(const struct object_id *oid,
                         */
                        return 0;
                while (tree_entry_gently(&desc, &entry))
-                       oidset_insert(set, entry.oid);
+                       oidset_insert(set, &entry.oid);
        } else if (obj->type == OBJ_COMMIT) {
                struct commit *commit = (struct commit *) obj;
                struct commit_list *parents = commit->parents;
@@ -1965,7 +2116,7 @@ static int add_promisor_object(const struct object_id *oid,
                        oidset_insert(set, &parents->item->object.oid);
        } else if (obj->type == OBJ_TAG) {
                struct tag *tag = (struct tag *) obj;
-               oidset_insert(set, &tag->tagged->oid);
+               oidset_insert(set, get_tagged_oid(tag));
        }
        return 0;
 }
@@ -1976,7 +2127,7 @@ int is_promisor_object(const struct object_id *oid)
        static int promisor_objects_prepared;
 
        if (!promisor_objects_prepared) {
-               if (repository_format_partial_clone) {
+               if (has_promisor_remote()) {
                        for_each_packed_object(add_promisor_object,
                                               &promisor_objects,
                                               FOR_EACH_OBJECT_PROMISOR_ONLY);