#include "streaming.h"
#include "thread-utils.h"
#include "pack-bitmap.h"
+#include "delta-islands.h"
#include "reachable.h"
-#include "sha1-array.h"
+#include "oid-array.h"
#include "argv-array.h"
#include "list.h"
#include "packfile.h"
#include "object-store.h"
#include "dir.h"
+#include "midx.h"
+#include "trace2.h"
+#include "shallow.h"
#define IN_PACK(obj) oe_in_pack(&to_pack, obj)
#define SIZE(obj) oe_size(&to_pack, obj)
#define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj)
#define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj)
#define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val)
+#define SET_DELTA_EXT(obj, oid) oe_set_delta_ext(&to_pack, obj, oid)
#define SET_DELTA_SIZE(obj, val) oe_set_delta_size(&to_pack, obj, val)
#define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val)
#define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val)
static struct pack_idx_entry **written_list;
static uint32_t nr_result, nr_written, nr_seen;
+static struct bitmap_index *bitmap_git;
+static uint32_t write_layer;
static int non_empty;
static int reuse_delta = 1, reuse_object = 1;
static int depth = 50;
static int delta_search_threads;
static int pack_to_stdout;
+static int sparse;
+static int thin;
static int num_preferred_base;
static struct progress *progress_state;
static struct packed_git *reuse_packfile;
static uint32_t reuse_packfile_objects;
-static off_t reuse_packfile_offset;
+static struct bitmap *reuse_packfile_bitmap;
static int use_bitmap_index_default = 1;
static int use_bitmap_index = -1;
-static int write_bitmap_index;
-static uint16_t write_bitmap_options;
+static int allow_pack_reuse = 1;
+static enum {
+ WRITE_BITMAP_FALSE = 0,
+ WRITE_BITMAP_QUIET,
+ WRITE_BITMAP_TRUE,
+} write_bitmap_index;
+static uint16_t write_bitmap_options = BITMAP_OPT_HASH_CACHE;
static int exclude_promisor_objects;
+static int use_delta_islands;
+
static unsigned long delta_cache_size = 0;
static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE;
static unsigned long cache_max_small_delta_size = 1000;
delta_buf = diff_delta(base_buf, base_size,
buf, size, &delta_size, 0);
/*
- * We succesfully computed this delta once but dropped it for
+ * We successfully computed this delta once but dropped it for
* memory reasons. Something is very wrong if this time we
* recompute and create a different delta.
*/
if (!usable_delta) {
if (oe_type(entry) == OBJ_BLOB &&
oe_size_greater_than(&to_pack, entry, big_file_threshold) &&
- (st = open_istream(&entry->idx.oid, &type, &size, NULL)) != NULL)
+ (st = open_istream(the_repository, &entry->idx.oid, &type,
+ &size, NULL)) != NULL)
buf = NULL;
else {
buf = read_object_file(&entry->idx.oid, &type, &size);
void *cb_data)
{
struct object_id peeled;
- struct object_entry *entry = packlist_find(&to_pack, oid->hash, NULL);
+ struct object_entry *entry = packlist_find(&to_pack, oid);
if (entry)
entry->tagged = 1;
if (!peel_ref(path, &peeled)) {
- entry = packlist_find(&to_pack, peeled.hash, NULL);
+ entry = packlist_find(&to_pack, &peeled);
if (entry)
entry->tagged = 1;
}
unsigned int *endp,
struct object_entry *e)
{
- if (e->filled)
+ if (e->filled || oe_layer(&to_pack, e) != write_layer)
return;
wo[(*endp)++] = e;
e->filled = 1;
add_descendants_to_write_order(wo, endp, root);
}
-static struct object_entry **compute_write_order(void)
+static void compute_layer_order(struct object_entry **wo, unsigned int *wo_end)
{
- unsigned int i, wo_end, last_untagged;
-
- struct object_entry **wo;
+ unsigned int i, last_untagged;
struct object_entry *objects = to_pack.objects;
for (i = 0; i < to_pack.nr_objects; i++) {
- objects[i].tagged = 0;
- objects[i].filled = 0;
- SET_DELTA_CHILD(&objects[i], NULL);
- SET_DELTA_SIBLING(&objects[i], NULL);
- }
-
- /*
- * Fully connect delta_child/delta_sibling network.
- * Make sure delta_sibling is sorted in the original
- * recency order.
- */
- for (i = to_pack.nr_objects; i > 0;) {
- struct object_entry *e = &objects[--i];
- if (!DELTA(e))
- continue;
- /* Mark me as the first child */
- e->delta_sibling_idx = DELTA(e)->delta_child_idx;
- SET_DELTA_CHILD(DELTA(e), e);
- }
-
- /*
- * Mark objects that are at the tip of tags.
- */
- for_each_tag_ref(mark_tagged, NULL);
-
- /*
- * Give the objects in the original recency order until
- * we see a tagged tip.
- */
- ALLOC_ARRAY(wo, to_pack.nr_objects);
- for (i = wo_end = 0; i < to_pack.nr_objects; i++) {
if (objects[i].tagged)
break;
- add_to_write_order(wo, &wo_end, &objects[i]);
+ add_to_write_order(wo, wo_end, &objects[i]);
}
last_untagged = i;
*/
for (; i < to_pack.nr_objects; i++) {
if (objects[i].tagged)
- add_to_write_order(wo, &wo_end, &objects[i]);
+ add_to_write_order(wo, wo_end, &objects[i]);
}
/*
if (oe_type(&objects[i]) != OBJ_COMMIT &&
oe_type(&objects[i]) != OBJ_TAG)
continue;
- add_to_write_order(wo, &wo_end, &objects[i]);
+ add_to_write_order(wo, wo_end, &objects[i]);
}
/*
for (i = last_untagged; i < to_pack.nr_objects; i++) {
if (oe_type(&objects[i]) != OBJ_TREE)
continue;
- add_to_write_order(wo, &wo_end, &objects[i]);
+ add_to_write_order(wo, wo_end, &objects[i]);
}
/*
* Finally all the rest in really tight order
*/
for (i = last_untagged; i < to_pack.nr_objects; i++) {
- if (!objects[i].filled)
- add_family_to_write_order(wo, &wo_end, &objects[i]);
+ if (!objects[i].filled && oe_layer(&to_pack, &objects[i]) == write_layer)
+ add_family_to_write_order(wo, wo_end, &objects[i]);
+ }
+}
+
+static struct object_entry **compute_write_order(void)
+{
+ uint32_t max_layers = 1;
+ unsigned int i, wo_end;
+
+ struct object_entry **wo;
+ struct object_entry *objects = to_pack.objects;
+
+ for (i = 0; i < to_pack.nr_objects; i++) {
+ objects[i].tagged = 0;
+ objects[i].filled = 0;
+ SET_DELTA_CHILD(&objects[i], NULL);
+ SET_DELTA_SIBLING(&objects[i], NULL);
}
+ /*
+ * Fully connect delta_child/delta_sibling network.
+ * Make sure delta_sibling is sorted in the original
+ * recency order.
+ */
+ for (i = to_pack.nr_objects; i > 0;) {
+ struct object_entry *e = &objects[--i];
+ if (!DELTA(e))
+ continue;
+ /* Mark me as the first child */
+ e->delta_sibling_idx = DELTA(e)->delta_child_idx;
+ SET_DELTA_CHILD(DELTA(e), e);
+ }
+
+ /*
+ * Mark objects that are at the tip of tags.
+ */
+ for_each_tag_ref(mark_tagged, NULL);
+
+ if (use_delta_islands)
+ max_layers = compute_pack_layers(&to_pack);
+
+ ALLOC_ARRAY(wo, to_pack.nr_objects);
+ wo_end = 0;
+
+ for (; write_layer < max_layers; ++write_layer)
+ compute_layer_order(wo, &wo_end);
+
if (wo_end != to_pack.nr_objects)
die(_("ordered %u objects, expected %"PRIu32),
wo_end, to_pack.nr_objects);
return wo;
}
-static off_t write_reused_pack(struct hashfile *f)
+
+/*
+ * A reused set of objects. All objects in a chunk have the same
+ * relative position in the original packfile and the generated
+ * packfile.
+ */
+
+static struct reused_chunk {
+ /* The offset of the first object of this chunk in the original
+ * packfile. */
+ off_t original;
+ /* The offset of the first object of this chunk in the generated
+ * packfile minus "original". */
+ off_t difference;
+} *reused_chunks;
+static int reused_chunks_nr;
+static int reused_chunks_alloc;
+
+static void record_reused_object(off_t where, off_t offset)
{
- unsigned char buffer[8192];
- off_t to_write, total;
- int fd;
+ if (reused_chunks_nr && reused_chunks[reused_chunks_nr-1].difference == offset)
+ return;
- if (!is_pack_valid(reuse_packfile))
- die(_("packfile is invalid: %s"), reuse_packfile->pack_name);
+ ALLOC_GROW(reused_chunks, reused_chunks_nr + 1,
+ reused_chunks_alloc);
+ reused_chunks[reused_chunks_nr].original = where;
+ reused_chunks[reused_chunks_nr].difference = offset;
+ reused_chunks_nr++;
+}
- fd = git_open(reuse_packfile->pack_name);
- if (fd < 0)
- die_errno(_("unable to open packfile for reuse: %s"),
- reuse_packfile->pack_name);
+/*
+ * Binary search to find the chunk that "where" is in. Note
+ * that we're not looking for an exact match, just the first
+ * chunk that contains it (which implicitly ends at the start
+ * of the next chunk.
+ */
+static off_t find_reused_offset(off_t where)
+{
+ int lo = 0, hi = reused_chunks_nr;
+ while (lo < hi) {
+ int mi = lo + ((hi - lo) / 2);
+ if (where == reused_chunks[mi].original)
+ return reused_chunks[mi].difference;
+ if (where < reused_chunks[mi].original)
+ hi = mi;
+ else
+ lo = mi + 1;
+ }
- if (lseek(fd, sizeof(struct pack_header), SEEK_SET) == -1)
- die_errno(_("unable to seek in reused packfile"));
+ /*
+ * The first chunk starts at zero, so we can't have gone below
+ * there.
+ */
+ assert(lo);
+ return reused_chunks[lo-1].difference;
+}
- if (reuse_packfile_offset < 0)
- reuse_packfile_offset = reuse_packfile->pack_size - the_hash_algo->rawsz;
+static void write_reused_pack_one(size_t pos, struct hashfile *out,
+ struct pack_window **w_curs)
+{
+ off_t offset, next, cur;
+ enum object_type type;
+ unsigned long size;
- total = to_write = reuse_packfile_offset - sizeof(struct pack_header);
+ offset = reuse_packfile->revindex[pos].offset;
+ next = reuse_packfile->revindex[pos + 1].offset;
- while (to_write) {
- int read_pack = xread(fd, buffer, sizeof(buffer));
+ record_reused_object(offset, offset - hashfile_total(out));
- if (read_pack <= 0)
- die_errno(_("unable to read from reused packfile"));
+ cur = offset;
+ type = unpack_object_header(reuse_packfile, w_curs, &cur, &size);
+ assert(type >= 0);
- if (read_pack > to_write)
- read_pack = to_write;
+ if (type == OBJ_OFS_DELTA) {
+ off_t base_offset;
+ off_t fixup;
- hashwrite(f, buffer, read_pack);
- to_write -= read_pack;
+ unsigned char header[MAX_PACK_OBJECT_HEADER];
+ unsigned len;
+
+ base_offset = get_delta_base(reuse_packfile, w_curs, &cur, type, offset);
+ assert(base_offset != 0);
+
+ /* Convert to REF_DELTA if we must... */
+ if (!allow_ofs_delta) {
+ int base_pos = find_revindex_position(reuse_packfile, base_offset);
+ struct object_id base_oid;
+
+ nth_packed_object_id(&base_oid, reuse_packfile,
+ reuse_packfile->revindex[base_pos].nr);
+
+ len = encode_in_pack_object_header(header, sizeof(header),
+ OBJ_REF_DELTA, size);
+ hashwrite(out, header, len);
+ hashwrite(out, base_oid.hash, the_hash_algo->rawsz);
+ copy_pack_data(out, reuse_packfile, w_curs, cur, next - cur);
+ return;
+ }
+
+ /* Otherwise see if we need to rewrite the offset... */
+ fixup = find_reused_offset(offset) -
+ find_reused_offset(base_offset);
+ if (fixup) {
+ unsigned char ofs_header[10];
+ unsigned i, ofs_len;
+ off_t ofs = offset - base_offset - fixup;
+
+ len = encode_in_pack_object_header(header, sizeof(header),
+ OBJ_OFS_DELTA, size);
+
+ i = sizeof(ofs_header) - 1;
+ ofs_header[i] = ofs & 127;
+ while (ofs >>= 7)
+ ofs_header[--i] = 128 | (--ofs & 127);
+
+ ofs_len = sizeof(ofs_header) - i;
+
+ hashwrite(out, header, len);
+ hashwrite(out, ofs_header + sizeof(ofs_header) - ofs_len, ofs_len);
+ copy_pack_data(out, reuse_packfile, w_curs, cur, next - cur);
+ return;
+ }
+
+ /* ...otherwise we have no fixup, and can write it verbatim */
+ }
+
+ copy_pack_data(out, reuse_packfile, w_curs, offset, next - offset);
+}
+
+static size_t write_reused_pack_verbatim(struct hashfile *out,
+ struct pack_window **w_curs)
+{
+ size_t pos = 0;
+
+ while (pos < reuse_packfile_bitmap->word_alloc &&
+ reuse_packfile_bitmap->words[pos] == (eword_t)~0)
+ pos++;
+
+ if (pos) {
+ off_t to_write;
+
+ written = (pos * BITS_IN_EWORD);
+ to_write = reuse_packfile->revindex[written].offset
+ - sizeof(struct pack_header);
+
+ /* We're recording one chunk, not one object. */
+ record_reused_object(sizeof(struct pack_header), 0);
+ hashflush(out);
+ copy_pack_data(out, reuse_packfile, w_curs,
+ sizeof(struct pack_header), to_write);
- /*
- * We don't know the actual number of objects written,
- * only how many bytes written, how many bytes total, and
- * how many objects total. So we can fake it by pretending all
- * objects we are writing are the same size. This gives us a
- * smooth progress meter, and at the end it matches the true
- * answer.
- */
- written = reuse_packfile_objects *
- (((double)(total - to_write)) / total);
display_progress(progress_state, written);
}
+ return pos;
+}
+
+static void write_reused_pack(struct hashfile *f)
+{
+ size_t i = 0;
+ uint32_t offset;
+ struct pack_window *w_curs = NULL;
- close(fd);
- written = reuse_packfile_objects;
- display_progress(progress_state, written);
- return reuse_packfile_offset - sizeof(struct pack_header);
+ if (allow_ofs_delta)
+ i = write_reused_pack_verbatim(f, &w_curs);
+
+ for (; i < reuse_packfile_bitmap->word_alloc; ++i) {
+ eword_t word = reuse_packfile_bitmap->words[i];
+ size_t pos = (i * BITS_IN_EWORD);
+
+ for (offset = 0; offset < BITS_IN_EWORD; ++offset) {
+ if ((word >> offset) == 0)
+ break;
+
+ offset += ewah_bit_ctz64(word >> offset);
+ write_reused_pack_one(pos + offset, f, &w_curs);
+ display_progress(progress_state, ++written);
+ }
+ }
+
+ unuse_pack(&w_curs);
}
static const char no_split_warning[] = N_(
offset = write_pack_header(f, nr_remaining);
if (reuse_packfile) {
- off_t packfile_size;
assert(pack_to_stdout);
-
- packfile_size = write_reused_pack(f);
- offset += packfile_size;
+ write_reused_pack(f);
+ offset = hashfile_total(f);
}
nr_written = 0;
nr_written, oid.hash, offset);
close(fd);
if (write_bitmap_index) {
- warning(_(no_split_warning));
+ if (write_bitmap_index != WRITE_BITMAP_QUIET)
+ warning(_(no_split_warning));
write_bitmap_index = 0;
}
}
if (written != nr_result)
die(_("wrote %"PRIu32" objects while expecting %"PRIu32),
written, nr_result);
+ trace2_data_intmax("pack-objects", the_repository,
+ "write_pack_file/wrote", nr_result);
}
static int no_try_delta(const char *path)
if (!check)
check = attr_check_initl("delta", NULL);
- git_check_attr(&the_index, path, check);
+ git_check_attr(the_repository->index, path, check);
if (ATTR_FALSE(check->items[0].value))
return 1;
return 0;
* few lines later when we want to add the new entry.
*/
static int have_duplicate_entry(const struct object_id *oid,
- int exclude,
- uint32_t *index_pos)
+ int exclude)
{
struct object_entry *entry;
- entry = packlist_find(&to_pack, oid->hash, index_pos);
+ if (reuse_packfile_bitmap &&
+ bitmap_walk_contains(bitmap_git, reuse_packfile_bitmap, oid))
+ return 1;
+
+ entry = packlist_find(&to_pack, oid);
if (!entry)
return 0;
{
int want;
struct list_head *pos;
+ struct multi_pack_index *m;
if (!exclude && local && has_loose_object_nonlocal(oid))
return 0;
if (want != -1)
return want;
}
+
+ for (m = get_multi_pack_index(the_repository); m; m = m->next) {
+ struct pack_entry e;
+ if (fill_midx_entry(the_repository, oid, &e, m)) {
+ struct packed_git *p = e.p;
+ off_t offset;
+
+ if (p == *found_pack)
+ offset = *found_offset;
+ else
+ offset = find_pack_entry_one(oid->hash, p);
+
+ if (offset) {
+ if (!*found_pack) {
+ if (!is_pack_valid(p))
+ continue;
+ *found_offset = offset;
+ *found_pack = p;
+ }
+ want = want_found_object(exclude, p);
+ if (want != -1)
+ return want;
+ }
+ }
+ }
+
list_for_each(pos, get_packed_git_mru(the_repository)) {
struct packed_git *p = list_entry(pos, struct packed_git, mru);
off_t offset;
uint32_t hash,
int exclude,
int no_try_delta,
- uint32_t index_pos,
struct packed_git *found_pack,
off_t found_offset)
{
struct object_entry *entry;
- entry = packlist_alloc(&to_pack, oid->hash, index_pos);
+ entry = packlist_alloc(&to_pack, oid);
entry->hash = hash;
oe_set_type(entry, type);
if (exclude)
{
struct packed_git *found_pack = NULL;
off_t found_offset = 0;
- uint32_t index_pos;
display_progress(progress_state, ++nr_seen);
- if (have_duplicate_entry(oid, exclude, &index_pos))
+ if (have_duplicate_entry(oid, exclude))
return 0;
if (!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) {
/* The pack is missing an object, so it will not have closure */
if (write_bitmap_index) {
- warning(_(no_closure_warning));
+ if (write_bitmap_index != WRITE_BITMAP_QUIET)
+ warning(_(no_closure_warning));
write_bitmap_index = 0;
}
return 0;
create_object_entry(oid, type, pack_name_hash(name),
exclude, name && no_try_delta(name),
- index_pos, found_pack, found_offset);
+ found_pack, found_offset);
return 1;
}
int flags, uint32_t name_hash,
struct packed_git *pack, off_t offset)
{
- uint32_t index_pos;
-
display_progress(progress_state, ++nr_seen);
- if (have_duplicate_entry(oid, 0, &index_pos))
+ if (have_duplicate_entry(oid, 0))
return 0;
if (!want_object_in_pack(oid, 0, &pack, &offset))
return 0;
- create_object_entry(oid, type, name_hash, 0, 0, index_pos, pack, offset);
+ create_object_entry(oid, type, name_hash, 0, 0, pack, offset);
return 1;
}
*/
for (neigh = 0; neigh < 8; neigh++) {
ent = pbase_tree_cache[my_ix];
- if (ent && !oidcmp(&ent->oid, oid)) {
+ if (ent && oideq(&ent->oid, oid)) {
ent->ref++;
return ent;
}
if (cmp < 0)
return;
if (name[cmplen] != '/') {
- add_object_entry(entry.oid,
+ add_object_entry(&entry.oid,
object_type(entry.mode),
fullname, 1);
return;
const char *down = name+cmplen+1;
int downlen = name_cmp_len(down);
- tree = pbase_tree_get(entry.oid);
+ tree = pbase_tree_get(&entry.oid);
if (!tree)
return;
init_tree_desc(&sub, tree->tree_data, tree->tree_size);
if (window <= num_preferred_base++)
return;
- data = read_object_with_reference(oid, tree_type, &size, &tree_oid);
+ data = read_object_with_reference(the_repository, oid,
+ tree_type, &size, &tree_oid);
if (!data)
return;
for (it = pbase_tree; it; it = it->next) {
- if (!oidcmp(&it->pcache.oid, &tree_oid)) {
+ if (oideq(&it->pcache.oid, &tree_oid)) {
free(data);
return;
}
done_pbase_paths_num = done_pbase_paths_alloc = 0;
}
+/*
+ * Return 1 iff the object specified by "delta" can be sent
+ * literally as a delta against the base in "base_sha1". If
+ * so, then *base_out will point to the entry in our packing
+ * list, or NULL if we must use the external-base list.
+ *
+ * Depth value does not matter - find_deltas() will
+ * never consider reused delta as the base object to
+ * deltify other objects against, in order to avoid
+ * circular deltas.
+ */
+static int can_reuse_delta(const struct object_id *base_oid,
+ struct object_entry *delta,
+ struct object_entry **base_out)
+{
+ struct object_entry *base;
+
+ /*
+ * First see if we're already sending the base (or it's explicitly in
+ * our "excluded" list).
+ */
+ base = packlist_find(&to_pack, base_oid);
+ if (base) {
+ if (!in_same_island(&delta->idx.oid, &base->idx.oid))
+ return 0;
+ *base_out = base;
+ return 1;
+ }
+
+ /*
+ * Otherwise, reachability bitmaps may tell us if the receiver has it,
+ * even if it was buried too deep in history to make it into the
+ * packing list.
+ */
+ if (thin && bitmap_has_oid_in_uninteresting(bitmap_git, base_oid)) {
+ if (use_delta_islands) {
+ if (!in_same_island(&delta->idx.oid, base_oid))
+ return 0;
+ }
+ *base_out = NULL;
+ return 1;
+ }
+
+ return 0;
+}
+
static void check_object(struct object_entry *entry)
{
unsigned long canonical_size;
if (IN_PACK(entry)) {
struct packed_git *p = IN_PACK(entry);
struct pack_window *w_curs = NULL;
- const unsigned char *base_ref = NULL;
+ int have_base = 0;
+ struct object_id base_ref;
struct object_entry *base_entry;
unsigned long used, used_0;
unsigned long avail;
unuse_pack(&w_curs);
return;
case OBJ_REF_DELTA:
- if (reuse_delta && !entry->preferred_base)
- base_ref = use_pack(p, &w_curs,
- entry->in_pack_offset + used, NULL);
+ if (reuse_delta && !entry->preferred_base) {
+ oidread(&base_ref,
+ use_pack(p, &w_curs,
+ entry->in_pack_offset + used,
+ NULL));
+ have_base = 1;
+ }
entry->in_pack_header_size = used + the_hash_algo->rawsz;
break;
case OBJ_OFS_DELTA:
revidx = find_pack_revindex(p, ofs);
if (!revidx)
goto give_up;
- base_ref = nth_packed_object_sha1(p, revidx->nr);
+ if (!nth_packed_object_id(&base_ref, p, revidx->nr))
+ have_base = 1;
}
entry->in_pack_header_size = used + used_0;
break;
}
- if (base_ref && (base_entry = packlist_find(&to_pack, base_ref, NULL))) {
- /*
- * If base_ref was set above that means we wish to
- * reuse delta data, and we even found that base
- * in the list of objects we want to pack. Goodie!
- *
- * Depth value does not matter - find_deltas() will
- * never consider reused delta as the base object to
- * deltify other objects against, in order to avoid
- * circular deltas.
- */
+ if (have_base &&
+ can_reuse_delta(&base_ref, entry, &base_entry)) {
oe_set_type(entry, entry->in_pack_type);
SET_SIZE(entry, in_pack_size); /* delta size */
- SET_DELTA(entry, base_entry);
SET_DELTA_SIZE(entry, in_pack_size);
- entry->delta_sibling_idx = base_entry->delta_child_idx;
- SET_DELTA_CHILD(base_entry, entry);
+
+ if (base_entry) {
+ SET_DELTA(entry, base_entry);
+ entry->delta_sibling_idx = base_entry->delta_child_idx;
+ SET_DELTA_CHILD(base_entry, entry);
+ } else {
+ SET_DELTA_EXT(entry, &base_ref);
+ }
+
unuse_pack(&w_curs);
return;
}
/*
* No choice but to fall back to the recursive delta walk
- * with sha1_object_info() to find about the object type
+ * with oid_object_info() to find about the object type
* at this point...
*/
give_up:
if (packed_object_info(the_repository, IN_PACK(entry), entry->in_pack_offset, &oi) < 0) {
/*
* We failed to get the info from this pack for some reason;
- * fall back to sha1_object_info, which may find another copy.
+ * fall back to oid_object_info, which may find another copy.
* And if that fails, the error will be recorded in oe_type(entry)
* and dealt with in prepare_pack().
*/
{
const struct object_entry *a = *(struct object_entry **)_a;
const struct object_entry *b = *(struct object_entry **)_b;
- enum object_type a_type = oe_type(a);
- enum object_type b_type = oe_type(b);
- unsigned long a_size = SIZE(a);
- unsigned long b_size = SIZE(b);
+ const enum object_type a_type = oe_type(a);
+ const enum object_type b_type = oe_type(b);
+ const unsigned long a_size = SIZE(a);
+ const unsigned long b_size = SIZE(b);
if (a_type > b_type)
return -1;
return -1;
if (a->preferred_base < b->preferred_base)
return 1;
+ if (use_delta_islands) {
+ const int island_cmp = island_delta_cmp(&a->idx.oid, &b->idx.oid);
+ if (island_cmp)
+ return island_cmp;
+ }
if (a_size > b_size)
return -1;
if (a_size < b_size)
return 0;
}
-#ifndef NO_PTHREADS
-
-/* Protect access to object database */
-static pthread_mutex_t read_mutex;
-#define read_lock() pthread_mutex_lock(&read_mutex)
-#define read_unlock() pthread_mutex_unlock(&read_mutex)
-
/* Protect delta_cache_size */
static pthread_mutex_t cache_mutex;
#define cache_lock() pthread_mutex_lock(&cache_mutex)
* ahead in the list because they can be stolen and would need
* progress_mutex for protection.
*/
-#else
-
-#define read_lock() (void)0
-#define read_unlock() (void)0
-#define cache_lock() (void)0
-#define cache_unlock() (void)0
-#define progress_lock() (void)0
-#define progress_unlock() (void)0
-
-#endif
/*
* Return the size of the object without doing any delta
unsigned long used, avail, size;
if (e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) {
- read_lock();
+ packing_data_lock(&to_pack);
if (oid_object_info(the_repository, &e->idx.oid, &size) < 0)
die(_("unable to get size of %s"),
oid_to_hex(&e->idx.oid));
- read_unlock();
+ packing_data_unlock(&to_pack);
return size;
}
if (!p)
BUG("when e->type is a delta, it must belong to a pack");
- read_lock();
+ packing_data_lock(&to_pack);
w_curs = NULL;
buf = use_pack(p, &w_curs, e->in_pack_offset, &avail);
used = unpack_object_header_buffer(buf, avail, &type, &size);
oid_to_hex(&e->idx.oid));
unuse_pack(&w_curs);
- read_unlock();
+ packing_data_unlock(&to_pack);
return size;
}
if (trg_size < src_size / 32)
return 0;
+ if (!in_same_island(&trg->entry->idx.oid, &src->entry->idx.oid))
+ return 0;
+
/* Load data if not already done */
if (!trg->data) {
- read_lock();
+ packing_data_lock(&to_pack);
trg->data = read_object_file(&trg_entry->idx.oid, &type, &sz);
- read_unlock();
+ packing_data_unlock(&to_pack);
if (!trg->data)
die(_("object %s cannot be read"),
oid_to_hex(&trg_entry->idx.oid));
if (sz != trg_size)
- die(_("object %s inconsistent object length (%lu vs %lu)"),
- oid_to_hex(&trg_entry->idx.oid), sz,
- trg_size);
+ die(_("object %s inconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"),
+ oid_to_hex(&trg_entry->idx.oid), (uintmax_t)sz,
+ (uintmax_t)trg_size);
*mem_usage += sz;
}
if (!src->data) {
- read_lock();
+ packing_data_lock(&to_pack);
src->data = read_object_file(&src_entry->idx.oid, &type, &sz);
- read_unlock();
+ packing_data_unlock(&to_pack);
if (!src->data) {
if (src_entry->preferred_base) {
static int warned = 0;
oid_to_hex(&src_entry->idx.oid));
}
if (sz != src_size)
- die(_("object %s inconsistent object length (%lu vs %lu)"),
- oid_to_hex(&src_entry->idx.oid), sz,
- src_size);
+ die(_("object %s inconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"),
+ oid_to_hex(&src_entry->idx.oid), (uintmax_t)sz,
+ (uintmax_t)src_size);
*mem_usage += sz;
}
if (!src->index) {
struct object_entry *child = DELTA_CHILD(me);
unsigned int m = n;
while (child) {
- unsigned int c = check_delta_limit(child, n + 1);
+ const unsigned int c = check_delta_limit(child, n + 1);
if (m < c)
m = c;
child = DELTA_SIBLING(child);
while (window_memory_limit &&
mem_usage > window_memory_limit &&
count > 1) {
- uint32_t tail = (idx + window - count) % window;
+ const uint32_t tail = (idx + window - count) % window;
mem_usage -= free_unpacked(array + tail);
count--;
}
free(array);
}
-#ifndef NO_PTHREADS
-
-static void try_to_free_from_threads(size_t size)
-{
- read_lock();
- release_pack_memory(size);
- read_unlock();
-}
-
-static try_to_free_t old_try_to_free_routine;
-
/*
* The main object list is split into smaller lists, each is handed to
* one worker.
*/
static void init_threaded_search(void)
{
- init_recursive_mutex(&read_mutex);
pthread_mutex_init(&cache_mutex, NULL);
pthread_mutex_init(&progress_mutex, NULL);
pthread_cond_init(&progress_cond, NULL);
- old_try_to_free_routine = set_try_to_free_routine(try_to_free_from_threads);
}
static void cleanup_threaded_search(void)
{
- set_try_to_free_routine(old_try_to_free_routine);
pthread_cond_destroy(&progress_cond);
- pthread_mutex_destroy(&read_mutex);
pthread_mutex_destroy(&cache_mutex);
pthread_mutex_destroy(&progress_mutex);
}
free(p);
}
-#else
-#define ll_find_deltas(l, s, w, d, p) find_deltas(l, &s, w, d, p)
-#endif
+static int obj_is_packed(const struct object_id *oid)
+{
+ return packlist_find(&to_pack, oid) ||
+ (reuse_packfile_bitmap &&
+ bitmap_walk_contains(bitmap_git, reuse_packfile_bitmap, oid));
+}
static void add_tag_chain(const struct object_id *oid)
{
* it was included via bitmaps, we would not have parsed it
* previously).
*/
- if (packlist_find(&to_pack, oid->hash, NULL))
+ if (obj_is_packed(oid))
return;
tag = lookup_tag(the_repository, oid);
if (starts_with(path, "refs/tags/") && /* is a tag? */
!peel_ref(path, &peeled) && /* peelable? */
- packlist_find(&to_pack, peeled.hash, NULL)) /* object packed? */
+ obj_is_packed(&peeled)) /* object packed? */
add_tag_chain(oid);
return 0;
}
uint32_t i, nr_deltas;
unsigned n;
+ if (use_delta_islands)
+ resolve_tree_islands(the_repository, progress, &to_pack);
+
get_object_details();
/*
if (nr_deltas && n > 1) {
unsigned nr_done = 0;
+
if (progress)
progress_state = start_progress(_("Compressing objects"),
nr_deltas);
use_bitmap_index_default = git_config_bool(k, v);
return 0;
}
+ if (!strcmp(k, "pack.allowpackreuse")) {
+ allow_pack_reuse = git_config_bool(k, v);
+ return 0;
+ }
if (!strcmp(k, "pack.threads")) {
delta_search_threads = git_config_int(k, v);
if (delta_search_threads < 0)
die(_("invalid number of threads specified (%d)"),
delta_search_threads);
-#ifdef NO_PTHREADS
- if (delta_search_threads != 1) {
+ if (!HAVE_THREADS && delta_search_threads != 1) {
warning(_("no threads support, ignoring %s"), k);
delta_search_threads = 0;
}
-#endif
return 0;
}
if (!strcmp(k, "pack.indexversion")) {
if (write_bitmap_index)
index_commit_for_bitmap(commit);
+
+ if (use_delta_islands)
+ propagate_island_marks(commit);
}
static void show_object(struct object *obj, const char *name, void *data)
add_preferred_base_object(name);
add_object_entry(&obj->oid, obj->type, name, 0);
obj->flags |= OBJECT_ADDED;
+
+ if (use_delta_islands) {
+ const char *p;
+ unsigned depth;
+ struct object_entry *ent;
+
+ /* the empty string is a root tree, which is depth 0 */
+ depth = *name ? 1 : 0;
+ for (p = strchr(name, '/'); p; p = strchr(p + 1, '/'))
+ depth++;
+
+ ent = packlist_find(&to_pack, &obj->oid);
+ if (ent && depth > oe_tree_depth(&to_pack, ent))
+ oe_set_tree_depth(&to_pack, ent, depth);
+ }
}
static void show_object__ma_allow_any(struct object *obj, const char *name, void *data)
return oidcmp(&a->object->oid, &b->object->oid);
}
-static void add_objects_in_unpacked_packs(struct rev_info *revs)
+static void add_objects_in_unpacked_packs(void)
{
struct packed_git *p;
struct in_pack in_pack;
memset(&in_pack, 0, sizeof(in_pack));
- for (p = get_packed_git(the_repository); p; p = p->next) {
+ for (p = get_all_packs(the_repository); p; p = p->next) {
struct object_id oid;
struct object *o;
in_pack.alloc);
for (i = 0; i < p->num_objects; i++) {
- nth_packed_object_oid(&oid, p, i);
- o = lookup_unknown_object(oid.hash);
+ nth_packed_object_id(&oid, p, i);
+ o = lookup_unknown_object(&oid);
if (!(o->flags & OBJECT_ADDED))
mark_in_pack_object(o, p, &in_pack);
o->flags |= OBJECT_ADDED;
struct packed_git *p;
p = (last_found != (void *)1) ? last_found :
- get_packed_git(the_repository);
+ get_all_packs(the_repository);
while (p) {
if ((!p->pack_local || p->pack_keep ||
return 1;
}
if (p == last_found)
- p = get_packed_git(the_repository);
+ p = get_all_packs(the_repository);
else
p = p->next;
if (p == last_found)
return 1;
}
-static void loosen_unused_packed_objects(struct rev_info *revs)
+static void loosen_unused_packed_objects(void)
{
struct packed_git *p;
uint32_t i;
struct object_id oid;
- for (p = get_packed_git(the_repository); p; p = p->next) {
+ for (p = get_all_packs(the_repository); p; p = p->next) {
if (!p->pack_local || p->pack_keep || p->pack_keep_in_core)
continue;
die(_("cannot open pack index"));
for (i = 0; i < p->num_objects; i++) {
- nth_packed_object_oid(&oid, p, i);
- if (!packlist_find(&to_pack, oid.hash, NULL) &&
+ nth_packed_object_id(&oid, p, i);
+ if (!packlist_find(&to_pack, &oid) &&
!has_sha1_pack_kept_or_nonlocal(&oid) &&
!loosened_object_can_be_discarded(&oid, p->mtime))
if (force_object_loose(&oid, p->mtime))
*/
static int pack_options_allow_reuse(void)
{
- return pack_to_stdout &&
- allow_ofs_delta &&
+ return allow_pack_reuse &&
+ pack_to_stdout &&
!ignore_packed_keep_on_disk &&
!ignore_packed_keep_in_core &&
(!local || !have_non_local_packs) &&
static int get_object_list_from_bitmap(struct rev_info *revs)
{
- struct bitmap_index *bitmap_git;
- if (!(bitmap_git = prepare_bitmap_walk(revs)))
+ if (!(bitmap_git = prepare_bitmap_walk(revs, &filter_options)))
return -1;
if (pack_options_allow_reuse() &&
bitmap_git,
&reuse_packfile,
&reuse_packfile_objects,
- &reuse_packfile_offset)) {
+ &reuse_packfile_bitmap)) {
assert(reuse_packfile_objects);
nr_result += reuse_packfile_objects;
display_progress(progress_state, nr_result);
}
- traverse_bitmap_commit_list(bitmap_git, &add_object_entry_from_bitmap);
- free_bitmap_index(bitmap_git);
+ traverse_bitmap_commit_list(bitmap_git, revs,
+ &add_object_entry_from_bitmap);
return 0;
}
static void get_object_list(int ac, const char **av)
{
struct rev_info revs;
+ struct setup_revision_opt s_r_opt = {
+ .allow_exclude_promisor_objects = 1,
+ };
char line[1000];
int flags = 0;
+ int save_warning;
- init_revisions(&revs, NULL);
+ repo_init_revisions(the_repository, &revs, NULL);
save_commit_buffer = 0;
- revs.allow_exclude_promisor_objects_opt = 1;
- setup_revisions(ac, av, &revs, NULL);
+ setup_revisions(ac, av, &revs, &s_r_opt);
/* make sure shallows are read */
is_repository_shallow(the_repository);
+ save_warning = warn_on_object_refname_ambiguity;
+ warn_on_object_refname_ambiguity = 0;
+
while (fgets(line, sizeof(line), stdin) != NULL) {
int len = strlen(line);
if (len && line[len - 1] == '\n')
die(_("bad revision '%s'"), line);
}
+ warn_on_object_refname_ambiguity = save_warning;
+
if (use_bitmap_index && !get_object_list_from_bitmap(&revs))
return;
+ if (use_delta_islands)
+ load_delta_islands(the_repository, progress);
+
if (prepare_revision_walk(&revs))
die(_("revision walk setup failed"));
- mark_edges_uninteresting(&revs, show_edge);
+ mark_edges_uninteresting(&revs, show_edge, sparse);
if (!fn_show_object)
fn_show_object = show_object;
}
if (keep_unreachable)
- add_objects_in_unpacked_packs(&revs);
+ add_objects_in_unpacked_packs();
if (pack_loose_unreachable)
add_unreachable_loose_objects();
if (unpack_unreachable)
- loosen_unused_packed_objects(&revs);
+ loosen_unused_packed_objects();
oid_array_clear(&recent_objects);
}
if (!names->nr)
return;
- for (p = get_packed_git(the_repository); p; p = p->next) {
+ for (p = get_all_packs(the_repository); p; p = p->next) {
const char *name = basename(p->pack_name);
int i;
{
char *c;
const char *val = arg;
+
+ BUG_ON_OPT_NEG(unset);
+
pack_idx_opts.version = strtoul(val, &c, 10);
if (pack_idx_opts.version > 2)
die(_("unsupported index version %s"), val);
int cmd_pack_objects(int argc, const char **argv, const char *prefix)
{
int use_internal_rev_list = 0;
- int thin = 0;
int shallow = 0;
int all_progress_implied = 0;
struct argv_array rp = ARGV_ARRAY_INIT;
OPT_BOOL(0, "all-progress-implied",
&all_progress_implied,
N_("similar to --all-progress when progress meter is shown")),
- { OPTION_CALLBACK, 0, "index-version", NULL, N_("<version>[,<offset>]"),
+ OPT_CALLBACK_F(0, "index-version", NULL, N_("<version>[,<offset>]"),
N_("write the pack index file in the specified idx format version"),
- 0, option_parse_index_version },
+ PARSE_OPT_NONEG, option_parse_index_version),
OPT_MAGNITUDE(0, "max-pack-size", &pack_size_limit,
N_("maximum size of each output pack file")),
OPT_BOOL(0, "local", &local,
N_("keep unreachable objects")),
OPT_BOOL(0, "pack-loose-unreachable", &pack_loose_unreachable,
N_("pack loose unreachable objects")),
- { OPTION_CALLBACK, 0, "unpack-unreachable", NULL, N_("time"),
+ OPT_CALLBACK_F(0, "unpack-unreachable", NULL, N_("time"),
N_("unpack unreachable objects newer than <time>"),
- PARSE_OPT_OPTARG, option_parse_unpack_unreachable },
+ PARSE_OPT_OPTARG, option_parse_unpack_unreachable),
+ OPT_BOOL(0, "sparse", &sparse,
+ N_("use the sparse reachability algorithm")),
OPT_BOOL(0, "thin", &thin,
N_("create thin packs")),
OPT_BOOL(0, "shallow", &shallow,
N_("do not hide commits by grafts"), 0),
OPT_BOOL(0, "use-bitmap-index", &use_bitmap_index,
N_("use a bitmap index if available to speed up counting objects")),
- OPT_BOOL(0, "write-bitmap-index", &write_bitmap_index,
- N_("write a bitmap index together with the pack index")),
+ OPT_SET_INT(0, "write-bitmap-index", &write_bitmap_index,
+ N_("write a bitmap index together with the pack index"),
+ WRITE_BITMAP_TRUE),
+ OPT_SET_INT_F(0, "write-bitmap-index-quiet",
+ &write_bitmap_index,
+ N_("write a bitmap index if possible"),
+ WRITE_BITMAP_QUIET, PARSE_OPT_HIDDEN),
OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),
- { OPTION_CALLBACK, 0, "missing", NULL, N_("action"),
+ OPT_CALLBACK_F(0, "missing", NULL, N_("action"),
N_("handling for missing objects"), PARSE_OPT_NONEG,
- option_parse_missing_action },
+ option_parse_missing_action),
OPT_BOOL(0, "exclude-promisor-objects", &exclude_promisor_objects,
N_("do not pack objects in promisor packfiles")),
+ OPT_BOOL(0, "delta-islands", &use_delta_islands,
+ N_("respect islands during delta compression")),
OPT_END(),
};
read_replace_refs = 0;
+ sparse = git_env_bool("GIT_TEST_PACK_SPARSE", -1);
+ prepare_repo_settings(the_repository);
+ if (sparse < 0)
+ sparse = the_repository->settings.pack_use_sparse;
+
reset_pack_idx_option(&pack_idx_opts);
git_config(git_pack_config, NULL);
if (!delta_search_threads) /* --threads=0 means autodetect */
delta_search_threads = online_cpus();
-#ifdef NO_PTHREADS
- if (delta_search_threads != 1)
+ if (!HAVE_THREADS && delta_search_threads != 1)
warning(_("no threads support, ignoring --threads"));
-#endif
if (!pack_to_stdout && !pack_size_limit)
pack_size_limit = pack_size_limit_cfg;
if (pack_to_stdout && pack_size_limit)
if (filter_options.choice) {
if (!pack_to_stdout)
die(_("cannot use --filter without --stdout"));
- use_bitmap_index = 0;
}
/*
if (pack_to_stdout || !rev_list_all)
write_bitmap_index = 0;
+ if (use_delta_islands)
+ argv_array_push(&rp, "--topo-order");
+
if (progress && all_progress_implied)
progress = 2;
add_extra_kept_packs(&keep_pack_list);
if (ignore_packed_keep_on_disk) {
struct packed_git *p;
- for (p = get_packed_git(the_repository); p; p = p->next)
+ for (p = get_all_packs(the_repository); p; p = p->next)
if (p->pack_local && p->pack_keep)
break;
if (!p) /* no keep-able packs found */
* it also covers non-local objects
*/
struct packed_git *p;
- for (p = get_packed_git(the_repository); p; p = p->next) {
+ for (p = get_all_packs(the_repository); p; p = p->next) {
if (!p->pack_local) {
have_non_local_packs = 1;
break;
}
}
- prepare_packing_data(&to_pack);
+ trace2_region_enter("pack-objects", "enumerate-objects",
+ the_repository);
+ prepare_packing_data(the_repository, &to_pack);
if (progress)
progress_state = start_progress(_("Enumerating objects"), 0);
if (include_tag && nr_result)
for_each_ref(add_ref_tag, NULL);
stop_progress(&progress_state);
+ trace2_region_leave("pack-objects", "enumerate-objects",
+ the_repository);
if (non_empty && !nr_result)
return 0;
- if (nr_result)
+ if (nr_result) {
+ trace2_region_enter("pack-objects", "prepare-pack",
+ the_repository);
prepare_pack(window, depth);
+ trace2_region_leave("pack-objects", "prepare-pack",
+ the_repository);
+ }
+
+ trace2_region_enter("pack-objects", "write-pack-file", the_repository);
write_pack_file();
+ trace2_region_leave("pack-objects", "write-pack-file", the_repository);
+
if (progress)
fprintf_ln(stderr,
_("Total %"PRIu32" (delta %"PRIu32"),"
- " reused %"PRIu32" (delta %"PRIu32")"),
- written, written_delta, reused, reused_delta);
+ " reused %"PRIu32" (delta %"PRIu32"),"
+ " pack-reused %"PRIu32),
+ written, written_delta, reused, reused_delta,
+ reuse_packfile_objects);
return 0;
}
projects / platform / upstream / git.git / blobdiff