fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int deepen_since_ok;
  25 static int deepen_not_ok;
  26 static int fetch_fsck_objects = -1;
  27 static int transfer_fsck_objects = -1;
  28 static int agent_supported;
  29 static struct lock_file shallow_lock;
  30 static const char *alternate_shallow_file;
  31
  32 /* Remember to update object flag allocation in object.h */
  33 #define COMPLETE        (1U << 0)
  34 #define COMMON          (1U << 1)
  35 #define COMMON_REF      (1U << 2)
  36 #define SEEN            (1U << 3)
  37 #define POPPED          (1U << 4)
  38
  39 static int marked;
  40
  41 /*
  42  * After sending this many "have"s if we do not get any new ACK , we
  43  * give up traversing our history.
  44  */
  45 #define MAX_IN_VAIN 256
  46
  47 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  48 static int non_common_revs, multi_ack, use_sideband;
  49 /* Allow specifying sha1 if it is a ref tip. */
  50 #define ALLOW_TIP_SHA1  01
  51 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  52 #define ALLOW_REACHABLE_SHA1    02
  53 static unsigned int allow_unadvertised_object_request;
  54
  55 __attribute__((format (printf, 2, 3)))
  56 static inline void print_verbose(const struct fetch_pack_args *args,
  57                                  const char *fmt, ...)
  58 {
  59         va_list params;
  60
  61         if (!args->verbose)
  62                 return;
  63
  64         va_start(params, fmt);
  65         vfprintf(stderr, fmt, params);
  66         va_end(params);
  67         fputc('\n', stderr);
  68 }
  69
  70 static void rev_list_push(struct commit *commit, int mark)
  71 {
  72         if (!(commit->object.flags & mark)) {
  73                 commit->object.flags |= mark;
  74
  75                 if (parse_commit(commit))
  76                         return;
  77
  78                 prio_queue_put(&rev_list, commit);
  79
  80                 if (!(commit->object.flags & COMMON))
  81                         non_common_revs++;
  82         }
  83 }
  84
  85 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
  86 {
  87         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  88
  89         if (o && o->type == OBJ_COMMIT)
  90                 rev_list_push((struct commit *)o, SEEN);
  91
  92         return 0;
  93 }
  94
  95 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
  96                                    int flag, void *cb_data)
  97 {
  98         return rev_list_insert_ref(refname, oid->hash);
  99 }
 100
 101 static int clear_marks(const char *refname, const struct object_id *oid,
 102                        int flag, void *cb_data)
 103 {
 104         struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
 105
 106         if (o && o->type == OBJ_COMMIT)
 107                 clear_commit_marks((struct commit *)o,
 108                                    COMMON | COMMON_REF | SEEN | POPPED);
 109         return 0;
 110 }
 111
 112 /*
 113    This function marks a rev and its ancestors as common.
 114    In some cases, it is desirable to mark only the ancestors (for example
 115    when only the server does not yet know that they are common).
 116 */
 117
 118 static void mark_common(struct commit *commit,
 119                 int ancestors_only, int dont_parse)
 120 {
 121         if (commit != NULL && !(commit->object.flags & COMMON)) {
 122                 struct object *o = (struct object *)commit;
 123
 124                 if (!ancestors_only)
 125                         o->flags |= COMMON;
 126
 127                 if (!(o->flags & SEEN))
 128                         rev_list_push(commit, SEEN);
 129                 else {
 130                         struct commit_list *parents;
 131
 132                         if (!ancestors_only && !(o->flags & POPPED))
 133                                 non_common_revs--;
 134                         if (!o->parsed && !dont_parse)
 135                                 if (parse_commit(commit))
 136                                         return;
 137
 138                         for (parents = commit->parents;
 139                                         parents;
 140                                         parents = parents->next)
 141                                 mark_common(parents->item, 0, dont_parse);
 142                 }
 143         }
 144 }
 145
 146 /*
 147   Get the next rev to send, ignoring the common.
 148 */
 149
 150 static const unsigned char *get_rev(void)
 151 {
 152         struct commit *commit = NULL;
 153
 154         while (commit == NULL) {
 155                 unsigned int mark;
 156                 struct commit_list *parents;
 157
 158                 if (rev_list.nr == 0 || non_common_revs == 0)
 159                         return NULL;
 160
 161                 commit = prio_queue_get(&rev_list);
 162                 parse_commit(commit);
 163                 parents = commit->parents;
 164
 165                 commit->object.flags |= POPPED;
 166                 if (!(commit->object.flags & COMMON))
 167                         non_common_revs--;
 168
 169                 if (commit->object.flags & COMMON) {
 170                         /* do not send "have", and ignore ancestors */
 171                         commit = NULL;
 172                         mark = COMMON | SEEN;
 173                 } else if (commit->object.flags & COMMON_REF)
 174                         /* send "have", and ignore ancestors */
 175                         mark = COMMON | SEEN;
 176                 else
 177                         /* send "have", also for its ancestors */
 178                         mark = SEEN;
 179
 180                 while (parents) {
 181                         if (!(parents->item->object.flags & SEEN))
 182                                 rev_list_push(parents->item, mark);
 183                         if (mark & COMMON)
 184                                 mark_common(parents->item, 1, 0);
 185                         parents = parents->next;
 186                 }
 187         }
 188
 189         return commit->object.oid.hash;
 190 }
 191
 192 enum ack_type {
 193         NAK = 0,
 194         ACK,
 195         ACK_continue,
 196         ACK_common,
 197         ACK_ready
 198 };
 199
 200 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 201 {
 202         if (args->stateless_rpc && args->deepen) {
 203                 /* If we sent a depth we will get back "duplicate"
 204                  * shallow and unshallow commands every time there
 205                  * is a block of have lines exchanged.
 206                  */
 207                 char *line;
 208                 while ((line = packet_read_line(fd, NULL))) {
 209                         if (starts_with(line, "shallow "))
 210                                 continue;
 211                         if (starts_with(line, "unshallow "))
 212                                 continue;
 213                         die(_("git fetch-pack: expected shallow list"));
 214                 }
 215         }
 216 }
 217
 218 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 219 {
 220         int len;
 221         char *line = packet_read_line(fd, &len);
 222         const char *arg;
 223
 224         if (!len)
 225                 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
 226         if (!strcmp(line, "NAK"))
 227                 return NAK;
 228         if (skip_prefix(line, "ACK ", &arg)) {
 229                 if (!get_sha1_hex(arg, result_sha1)) {
 230                         arg += 40;
 231                         len -= arg - line;
 232                         if (len < 1)
 233                                 return ACK;
 234                         if (strstr(arg, "continue"))
 235                                 return ACK_continue;
 236                         if (strstr(arg, "common"))
 237                                 return ACK_common;
 238                         if (strstr(arg, "ready"))
 239                                 return ACK_ready;
 240                         return ACK;
 241                 }
 242         }
 243         die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 244 }
 245
 246 static void send_request(struct fetch_pack_args *args,
 247                          int fd, struct strbuf *buf)
 248 {
 249         if (args->stateless_rpc) {
 250                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 251                 packet_flush(fd);
 252         } else
 253                 write_or_die(fd, buf->buf, buf->len);
 254 }
 255
 256 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 257 {
 258         rev_list_insert_ref(NULL, ref->old_oid.hash);
 259 }
 260
 261 #define INITIAL_FLUSH 16
 262 #define PIPESAFE_FLUSH 32
 263 #define LARGE_FLUSH 16384
 264
 265 static int next_flush(struct fetch_pack_args *args, int count)
 266 {
 267         if (args->stateless_rpc) {
 268                 if (count < LARGE_FLUSH)
 269                         count <<= 1;
 270                 else
 271                         count = count * 11 / 10;
 272         } else {
 273                 if (count < PIPESAFE_FLUSH)
 274                         count <<= 1;
 275                 else
 276                         count += PIPESAFE_FLUSH;
 277         }
 278         return count;
 279 }
 280
 281 static int find_common(struct fetch_pack_args *args,
 282                        int fd[2], unsigned char *result_sha1,
 283                        struct ref *refs)
 284 {
 285         int fetching;
 286         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 287         const unsigned char *sha1;
 288         unsigned in_vain = 0;
 289         int got_continue = 0;
 290         int got_ready = 0;
 291         struct strbuf req_buf = STRBUF_INIT;
 292         size_t state_len = 0;
 293
 294         if (args->stateless_rpc && multi_ack == 1)
 295                 die(_("--stateless-rpc requires multi_ack_detailed"));
 296         if (marked)
 297                 for_each_ref(clear_marks, NULL);
 298         marked = 1;
 299
 300         for_each_ref(rev_list_insert_ref_oid, NULL);
 301         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 302
 303         fetching = 0;
 304         for ( ; refs ; refs = refs->next) {
 305                 unsigned char *remote = refs->old_oid.hash;
 306                 const char *remote_hex;
 307                 struct object *o;
 308
 309                 /*
 310                  * If that object is complete (i.e. it is an ancestor of a
 311                  * local ref), we tell them we have it but do not have to
 312                  * tell them about its ancestors, which they already know
 313                  * about.
 314                  *
 315                  * We use lookup_object here because we are only
 316                  * interested in the case we *know* the object is
 317                  * reachable and we have already scanned it.
 318                  */
 319                 if (((o = lookup_object(remote)) != NULL) &&
 320                                 (o->flags & COMPLETE)) {
 321                         continue;
 322                 }
 323
 324                 remote_hex = sha1_to_hex(remote);
 325                 if (!fetching) {
 326                         struct strbuf c = STRBUF_INIT;
 327                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 328                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 329                         if (no_done)            strbuf_addstr(&c, " no-done");
 330                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 331                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 332                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 333                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 334                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 335                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 336                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 337                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 338                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 339                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 340                                                             git_user_agent_sanitized());
 341                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 342                         strbuf_release(&c);
 343                 } else
 344                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 345                 fetching++;
 346         }
 347
 348         if (!fetching) {
 349                 strbuf_release(&req_buf);
 350                 packet_flush(fd[1]);
 351                 return 1;
 352         }
 353
 354         if (is_repository_shallow())
 355                 write_shallow_commits(&req_buf, 1, NULL);
 356         if (args->depth > 0)
 357                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 358         if (args->deepen_since) {
 359                 unsigned long max_age = approxidate(args->deepen_since);
 360                 packet_buf_write(&req_buf, "deepen-since %lu", max_age);
 361         }
 362         if (args->deepen_not) {
 363                 int i;
 364                 for (i = 0; i < args->deepen_not->nr; i++) {
 365                         struct string_list_item *s = args->deepen_not->items + i;
 366                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 367                 }
 368         }
 369         packet_buf_flush(&req_buf);
 370         state_len = req_buf.len;
 371
 372         if (args->deepen) {
 373                 char *line;
 374                 const char *arg;
 375                 unsigned char sha1[20];
 376
 377                 send_request(args, fd[1], &req_buf);
 378                 while ((line = packet_read_line(fd[0], NULL))) {
 379                         if (skip_prefix(line, "shallow ", &arg)) {
 380                                 if (get_sha1_hex(arg, sha1))
 381                                         die(_("invalid shallow line: %s"), line);
 382                                 register_shallow(sha1);
 383                                 continue;
 384                         }
 385                         if (skip_prefix(line, "unshallow ", &arg)) {
 386                                 if (get_sha1_hex(arg, sha1))
 387                                         die(_("invalid unshallow line: %s"), line);
 388                                 if (!lookup_object(sha1))
 389                                         die(_("object not found: %s"), line);
 390                                 /* make sure that it is parsed as shallow */
 391                                 if (!parse_object(sha1))
 392                                         die(_("error in object: %s"), line);
 393                                 if (unregister_shallow(sha1))
 394                                         die(_("no shallow found: %s"), line);
 395                                 continue;
 396                         }
 397                         die(_("expected shallow/unshallow, got %s"), line);
 398                 }
 399         } else if (!args->stateless_rpc)
 400                 send_request(args, fd[1], &req_buf);
 401
 402         if (!args->stateless_rpc) {
 403                 /* If we aren't using the stateless-rpc interface
 404                  * we don't need to retain the headers.
 405                  */
 406                 strbuf_setlen(&req_buf, 0);
 407                 state_len = 0;
 408         }
 409
 410         flushes = 0;
 411         retval = -1;
 412         while ((sha1 = get_rev())) {
 413                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 414                 print_verbose(args, "have %s", sha1_to_hex(sha1));
 415                 in_vain++;
 416                 if (flush_at <= ++count) {
 417                         int ack;
 418
 419                         packet_buf_flush(&req_buf);
 420                         send_request(args, fd[1], &req_buf);
 421                         strbuf_setlen(&req_buf, state_len);
 422                         flushes++;
 423                         flush_at = next_flush(args, count);
 424
 425                         /*
 426                          * We keep one window "ahead" of the other side, and
 427                          * will wait for an ACK only on the next one
 428                          */
 429                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 430                                 continue;
 431
 432                         consume_shallow_list(args, fd[0]);
 433                         do {
 434                                 ack = get_ack(fd[0], result_sha1);
 435                                 if (ack)
 436                                         print_verbose(args, _("got %s %d %s"), "ack",
 437                                                       ack, sha1_to_hex(result_sha1));
 438                                 switch (ack) {
 439                                 case ACK:
 440                                         flushes = 0;
 441                                         multi_ack = 0;
 442                                         retval = 0;
 443                                         goto done;
 444                                 case ACK_common:
 445                                 case ACK_ready:
 446                                 case ACK_continue: {
 447                                         struct commit *commit =
 448                                                 lookup_commit(result_sha1);
 449                                         if (!commit)
 450                                                 die(_("invalid commit %s"), sha1_to_hex(result_sha1));
 451                                         if (args->stateless_rpc
 452                                          && ack == ACK_common
 453                                          && !(commit->object.flags & COMMON)) {
 454                                                 /* We need to replay the have for this object
 455                                                  * on the next RPC request so the peer knows
 456                                                  * it is in common with us.
 457                                                  */
 458                                                 const char *hex = sha1_to_hex(result_sha1);
 459                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 460                                                 state_len = req_buf.len;
 461                                                 /*
 462                                                  * Reset in_vain because an ack
 463                                                  * for this commit has not been
 464                                                  * seen.
 465                                                  */
 466                                                 in_vain = 0;
 467                                         } else if (!args->stateless_rpc
 468                                                    || ack != ACK_common)
 469                                                 in_vain = 0;
 470                                         mark_common(commit, 0, 1);
 471                                         retval = 0;
 472                                         got_continue = 1;
 473                                         if (ack == ACK_ready) {
 474                                                 clear_prio_queue(&rev_list);
 475                                                 got_ready = 1;
 476                                         }
 477                                         break;
 478                                         }
 479                                 }
 480                         } while (ack);
 481                         flushes--;
 482                         if (got_continue && MAX_IN_VAIN < in_vain) {
 483                                 print_verbose(args, _("giving up"));
 484                                 break; /* give up */
 485                         }
 486                 }
 487         }
 488 done:
 489         if (!got_ready || !no_done) {
 490                 packet_buf_write(&req_buf, "done\n");
 491                 send_request(args, fd[1], &req_buf);
 492         }
 493         print_verbose(args, _("done"));
 494         if (retval != 0) {
 495                 multi_ack = 0;
 496                 flushes++;
 497         }
 498         strbuf_release(&req_buf);
 499
 500         if (!got_ready || !no_done)
 501                 consume_shallow_list(args, fd[0]);
 502         while (flushes || multi_ack) {
 503                 int ack = get_ack(fd[0], result_sha1);
 504                 if (ack) {
 505                         print_verbose(args, _("got %s (%d) %s"), "ack",
 506                                       ack, sha1_to_hex(result_sha1));
 507                         if (ack == ACK)
 508                                 return 0;
 509                         multi_ack = 1;
 510                         continue;
 511                 }
 512                 flushes--;
 513         }
 514         /* it is no error to fetch into a completely empty repo */
 515         return count ? retval : 0;
 516 }
 517
 518 static struct commit_list *complete;
 519
 520 static int mark_complete(const unsigned char *sha1)
 521 {
 522         struct object *o = parse_object(sha1);
 523
 524         while (o && o->type == OBJ_TAG) {
 525                 struct tag *t = (struct tag *) o;
 526                 if (!t->tagged)
 527                         break; /* broken repository */
 528                 o->flags |= COMPLETE;
 529                 o = parse_object(t->tagged->oid.hash);
 530         }
 531         if (o && o->type == OBJ_COMMIT) {
 532                 struct commit *commit = (struct commit *)o;
 533                 if (!(commit->object.flags & COMPLETE)) {
 534                         commit->object.flags |= COMPLETE;
 535                         commit_list_insert(commit, &complete);
 536                 }
 537         }
 538         return 0;
 539 }
 540
 541 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 542                              int flag, void *cb_data)
 543 {
 544         return mark_complete(oid->hash);
 545 }
 546
 547 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 548                                          unsigned long cutoff)
 549 {
 550         while (complete && cutoff <= complete->item->date) {
 551                 print_verbose(args, _("Marking %s as complete"),
 552                               oid_to_hex(&complete->item->object.oid));
 553                 pop_most_recent_commit(&complete, COMPLETE);
 554         }
 555 }
 556
 557 static void filter_refs(struct fetch_pack_args *args,
 558                         struct ref **refs,
 559                         struct ref **sought, int nr_sought)
 560 {
 561         struct ref *newlist = NULL;
 562         struct ref **newtail = &newlist;
 563         struct ref *ref, *next;
 564         int i;
 565
 566         i = 0;
 567         for (ref = *refs; ref; ref = next) {
 568                 int keep = 0;
 569                 next = ref->next;
 570
 571                 if (starts_with(ref->name, "refs/") &&
 572                     check_refname_format(ref->name, 0))
 573                         ; /* trash */
 574                 else {
 575                         while (i < nr_sought) {
 576                                 int cmp = strcmp(ref->name, sought[i]->name);
 577                                 if (cmp < 0)
 578                                         break; /* definitely do not have it */
 579                                 else if (cmp == 0) {
 580                                         keep = 1; /* definitely have it */
 581                                         sought[i]->match_status = REF_MATCHED;
 582                                 }
 583                                 i++;
 584                         }
 585                 }
 586
 587                 if (!keep && args->fetch_all &&
 588                     (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 589                         keep = 1;
 590
 591                 if (keep) {
 592                         *newtail = ref;
 593                         ref->next = NULL;
 594                         newtail = &ref->next;
 595                 } else {
 596                         free(ref);
 597                 }
 598         }
 599
 600         /* Append unmatched requests to the list */
 601         for (i = 0; i < nr_sought; i++) {
 602                 unsigned char sha1[20];
 603
 604                 ref = sought[i];
 605                 if (ref->match_status != REF_NOT_MATCHED)
 606                         continue;
 607                 if (get_sha1_hex(ref->name, sha1) ||
 608                     ref->name[40] != '\0' ||
 609                     hashcmp(sha1, ref->old_oid.hash))
 610                         continue;
 611
 612                 if ((allow_unadvertised_object_request &
 613                     (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 614                         ref->match_status = REF_MATCHED;
 615                         *newtail = copy_ref(ref);
 616                         newtail = &(*newtail)->next;
 617                 } else {
 618                         ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 619                 }
 620         }
 621         *refs = newlist;
 622 }
 623
 624 static void mark_alternate_complete(const struct ref *ref, void *unused)
 625 {
 626         mark_complete(ref->old_oid.hash);
 627 }
 628
 629 static int everything_local(struct fetch_pack_args *args,
 630                             struct ref **refs,
 631                             struct ref **sought, int nr_sought)
 632 {
 633         struct ref *ref;
 634         int retval;
 635         unsigned long cutoff = 0;
 636
 637         save_commit_buffer = 0;
 638
 639         for (ref = *refs; ref; ref = ref->next) {
 640                 struct object *o;
 641
 642                 if (!has_object_file(&ref->old_oid))
 643                         continue;
 644
 645                 o = parse_object(ref->old_oid.hash);
 646                 if (!o)
 647                         continue;
 648
 649                 /* We already have it -- which may mean that we were
 650                  * in sync with the other side at some time after
 651                  * that (it is OK if we guess wrong here).
 652                  */
 653                 if (o->type == OBJ_COMMIT) {
 654                         struct commit *commit = (struct commit *)o;
 655                         if (!cutoff || cutoff < commit->date)
 656                                 cutoff = commit->date;
 657                 }
 658         }
 659
 660         if (!args->deepen) {
 661                 for_each_ref(mark_complete_oid, NULL);
 662                 for_each_alternate_ref(mark_alternate_complete, NULL);
 663                 commit_list_sort_by_date(&complete);
 664                 if (cutoff)
 665                         mark_recent_complete_commits(args, cutoff);
 666         }
 667
 668         /*
 669          * Mark all complete remote refs as common refs.
 670          * Don't mark them common yet; the server has to be told so first.
 671          */
 672         for (ref = *refs; ref; ref = ref->next) {
 673                 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 674                                              NULL, 0);
 675
 676                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 677                         continue;
 678
 679                 if (!(o->flags & SEEN)) {
 680                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 681
 682                         mark_common((struct commit *)o, 1, 1);
 683                 }
 684         }
 685
 686         filter_refs(args, refs, sought, nr_sought);
 687
 688         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 689                 const unsigned char *remote = ref->old_oid.hash;
 690                 struct object *o;
 691
 692                 o = lookup_object(remote);
 693                 if (!o || !(o->flags & COMPLETE)) {
 694                         retval = 0;
 695                         print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
 696                                       ref->name);
 697                         continue;
 698                 }
 699                 print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote),
 700                               ref->name);
 701         }
 702         return retval;
 703 }
 704
 705 static int sideband_demux(int in, int out, void *data)
 706 {
 707         int *xd = data;
 708         int ret;
 709
 710         ret = recv_sideband("fetch-pack", xd[0], out);
 711         close(out);
 712         return ret;
 713 }
 714
 715 static int get_pack(struct fetch_pack_args *args,
 716                     int xd[2], char **pack_lockfile)
 717 {
 718         struct async demux;
 719         int do_keep = args->keep_pack;
 720         const char *cmd_name;
 721         struct pack_header header;
 722         int pass_header = 0;
 723         struct child_process cmd = CHILD_PROCESS_INIT;
 724         int ret;
 725
 726         memset(&demux, 0, sizeof(demux));
 727         if (use_sideband) {
 728                 /* xd[] is talking with upload-pack; subprocess reads from
 729                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 730                  * through demux->out.
 731                  */
 732                 demux.proc = sideband_demux;
 733                 demux.data = xd;
 734                 demux.out = -1;
 735                 demux.isolate_sigpipe = 1;
 736                 if (start_async(&demux))
 737                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 738         }
 739         else
 740                 demux.out = xd[0];
 741
 742         if (!args->keep_pack && unpack_limit) {
 743
 744                 if (read_pack_header(demux.out, &header))
 745                         die(_("protocol error: bad pack header"));
 746                 pass_header = 1;
 747                 if (ntohl(header.hdr_entries) < unpack_limit)
 748                         do_keep = 0;
 749                 else
 750                         do_keep = 1;
 751         }
 752
 753         if (alternate_shallow_file) {
 754                 argv_array_push(&cmd.args, "--shallow-file");
 755                 argv_array_push(&cmd.args, alternate_shallow_file);
 756         }
 757
 758         if (do_keep) {
 759                 if (pack_lockfile)
 760                         cmd.out = -1;
 761                 cmd_name = "index-pack";
 762                 argv_array_push(&cmd.args, cmd_name);
 763                 argv_array_push(&cmd.args, "--stdin");
 764                 if (!args->quiet && !args->no_progress)
 765                         argv_array_push(&cmd.args, "-v");
 766                 if (args->use_thin_pack)
 767                         argv_array_push(&cmd.args, "--fix-thin");
 768                 if (args->lock_pack || unpack_limit) {
 769                         char hostname[256];
 770                         if (gethostname(hostname, sizeof(hostname)))
 771                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 772                         argv_array_pushf(&cmd.args,
 773                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 774                                         (uintmax_t)getpid(), hostname);
 775                 }
 776                 if (args->check_self_contained_and_connected)
 777                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 778         }
 779         else {
 780                 cmd_name = "unpack-objects";
 781                 argv_array_push(&cmd.args, cmd_name);
 782                 if (args->quiet || args->no_progress)
 783                         argv_array_push(&cmd.args, "-q");
 784                 args->check_self_contained_and_connected = 0;
 785         }
 786
 787         if (pass_header)
 788                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 789                                  ntohl(header.hdr_version),
 790                                  ntohl(header.hdr_entries));
 791         if (fetch_fsck_objects >= 0
 792             ? fetch_fsck_objects
 793             : transfer_fsck_objects >= 0
 794             ? transfer_fsck_objects
 795             : 0)
 796                 argv_array_push(&cmd.args, "--strict");
 797
 798         cmd.in = demux.out;
 799         cmd.git_cmd = 1;
 800         if (start_command(&cmd))
 801                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 802         if (do_keep && pack_lockfile) {
 803                 *pack_lockfile = index_pack_lockfile(cmd.out);
 804                 close(cmd.out);
 805         }
 806
 807         if (!use_sideband)
 808                 /* Closed by start_command() */
 809                 xd[0] = -1;
 810
 811         ret = finish_command(&cmd);
 812         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 813                 args->self_contained_and_connected =
 814                         args->check_self_contained_and_connected &&
 815                         ret == 0;
 816         else
 817                 die(_("%s failed"), cmd_name);
 818         if (use_sideband && finish_async(&demux))
 819                 die(_("error in sideband demultiplexer"));
 820         return 0;
 821 }
 822
 823 static int cmp_ref_by_name(const void *a_, const void *b_)
 824 {
 825         const struct ref *a = *((const struct ref **)a_);
 826         const struct ref *b = *((const struct ref **)b_);
 827         return strcmp(a->name, b->name);
 828 }
 829
 830 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 831                                  int fd[2],
 832                                  const struct ref *orig_ref,
 833                                  struct ref **sought, int nr_sought,
 834                                  struct shallow_info *si,
 835                                  char **pack_lockfile)
 836 {
 837         struct ref *ref = copy_ref_list(orig_ref);
 838         unsigned char sha1[20];
 839         const char *agent_feature;
 840         int agent_len;
 841
 842         sort_ref_list(&ref, ref_compare_name);
 843         QSORT(sought, nr_sought, cmp_ref_by_name);
 844
 845         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 846                 die(_("Server does not support shallow clients"));
 847         if (args->depth > 0 || args->deepen_since || args->deepen_not)
 848                 args->deepen = 1;
 849         if (server_supports("multi_ack_detailed")) {
 850                 print_verbose(args, _("Server supports multi_ack_detailed"));
 851                 multi_ack = 2;
 852                 if (server_supports("no-done")) {
 853                         print_verbose(args, _("Server supports no-done"));
 854                         if (args->stateless_rpc)
 855                                 no_done = 1;
 856                 }
 857         }
 858         else if (server_supports("multi_ack")) {
 859                 print_verbose(args, _("Server supports multi_ack"));
 860                 multi_ack = 1;
 861         }
 862         if (server_supports("side-band-64k")) {
 863                 print_verbose(args, _("Server supports side-band-64k"));
 864                 use_sideband = 2;
 865         }
 866         else if (server_supports("side-band")) {
 867                 print_verbose(args, _("Server supports side-band"));
 868                 use_sideband = 1;
 869         }
 870         if (server_supports("allow-tip-sha1-in-want")) {
 871                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
 872                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 873         }
 874         if (server_supports("allow-reachable-sha1-in-want")) {
 875                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
 876                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 877         }
 878         if (!server_supports("thin-pack"))
 879                 args->use_thin_pack = 0;
 880         if (!server_supports("no-progress"))
 881                 args->no_progress = 0;
 882         if (!server_supports("include-tag"))
 883                 args->include_tag = 0;
 884         if (server_supports("ofs-delta"))
 885                 print_verbose(args, _("Server supports ofs-delta"));
 886         else
 887                 prefer_ofs_delta = 0;
 888
 889         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 890                 agent_supported = 1;
 891                 if (agent_len)
 892                         print_verbose(args, _("Server version is %.*s"),
 893                                       agent_len, agent_feature);
 894         }
 895         if (server_supports("deepen-since"))
 896                 deepen_since_ok = 1;
 897         else if (args->deepen_since)
 898                 die(_("Server does not support --shallow-since"));
 899         if (server_supports("deepen-not"))
 900                 deepen_not_ok = 1;
 901         else if (args->deepen_not)
 902                 die(_("Server does not support --shallow-exclude"));
 903         if (!server_supports("deepen-relative") && args->deepen_relative)
 904                 die(_("Server does not support --deepen"));
 905
 906         if (everything_local(args, &ref, sought, nr_sought)) {
 907                 packet_flush(fd[1]);
 908                 goto all_done;
 909         }
 910         if (find_common(args, fd, sha1, ref) < 0)
 911                 if (!args->keep_pack)
 912                         /* When cloning, it is not unusual to have
 913                          * no common commit.
 914                          */
 915                         warning(_("no common commits"));
 916
 917         if (args->stateless_rpc)
 918                 packet_flush(fd[1]);
 919         if (args->deepen)
 920                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 921                                         NULL);
 922         else if (si->nr_ours || si->nr_theirs)
 923                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 924         else
 925                 alternate_shallow_file = NULL;
 926         if (get_pack(args, fd, pack_lockfile))
 927                 die(_("git fetch-pack: fetch failed."));
 928
 929  all_done:
 930         return ref;
 931 }
 932
 933 static void fetch_pack_config(void)
 934 {
 935         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 936         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 937         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 938         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 939         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 940
 941         git_config(git_default_config, NULL);
 942 }
 943
 944 static void fetch_pack_setup(void)
 945 {
 946         static int did_setup;
 947         if (did_setup)
 948                 return;
 949         fetch_pack_config();
 950         if (0 <= transfer_unpack_limit)
 951                 unpack_limit = transfer_unpack_limit;
 952         else if (0 <= fetch_unpack_limit)
 953                 unpack_limit = fetch_unpack_limit;
 954         did_setup = 1;
 955 }
 956
 957 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 958 {
 959         struct string_list names = STRING_LIST_INIT_NODUP;
 960         int src, dst;
 961
 962         for (src = dst = 0; src < nr; src++) {
 963                 struct string_list_item *item;
 964                 item = string_list_insert(&names, ref[src]->name);
 965                 if (item->util)
 966                         continue; /* already have it */
 967                 item->util = ref[src];
 968                 if (src != dst)
 969                         ref[dst] = ref[src];
 970                 dst++;
 971         }
 972         for (src = dst; src < nr; src++)
 973                 ref[src] = NULL;
 974         string_list_clear(&names, 0);
 975         return dst;
 976 }
 977
 978 static void update_shallow(struct fetch_pack_args *args,
 979                            struct ref **sought, int nr_sought,
 980                            struct shallow_info *si)
 981 {
 982         struct sha1_array ref = SHA1_ARRAY_INIT;
 983         int *status;
 984         int i;
 985
 986         if (args->deepen && alternate_shallow_file) {
 987                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 988                         unlink_or_warn(git_path_shallow());
 989                         rollback_lock_file(&shallow_lock);
 990                 } else
 991                         commit_lock_file(&shallow_lock);
 992                 return;
 993         }
 994
 995         if (!si->shallow || !si->shallow->nr)
 996                 return;
 997
 998         if (args->cloning) {
 999                 /*
1000                  * remote is shallow, but this is a clone, there are
1001                  * no objects in repo to worry about. Accept any
1002                  * shallow points that exist in the pack (iow in repo
1003                  * after get_pack() and reprepare_packed_git())
1004                  */
1005                 struct sha1_array extra = SHA1_ARRAY_INIT;
1006                 unsigned char (*sha1)[20] = si->shallow->sha1;
1007                 for (i = 0; i < si->shallow->nr; i++)
1008                         if (has_sha1_file(sha1[i]))
1009                                 sha1_array_append(&extra, sha1[i]);
1010                 if (extra.nr) {
1011                         setup_alternate_shallow(&shallow_lock,
1012                                                 &alternate_shallow_file,
1013                                                 &extra);
1014                         commit_lock_file(&shallow_lock);
1015                 }
1016                 sha1_array_clear(&extra);
1017                 return;
1018         }
1019
1020         if (!si->nr_ours && !si->nr_theirs)
1021                 return;
1022
1023         remove_nonexistent_theirs_shallow(si);
1024         if (!si->nr_ours && !si->nr_theirs)
1025                 return;
1026         for (i = 0; i < nr_sought; i++)
1027                 sha1_array_append(&ref, sought[i]->old_oid.hash);
1028         si->ref = &ref;
1029
1030         if (args->update_shallow) {
1031                 /*
1032                  * remote is also shallow, .git/shallow may be updated
1033                  * so all refs can be accepted. Make sure we only add
1034                  * shallow roots that are actually reachable from new
1035                  * refs.
1036                  */
1037                 struct sha1_array extra = SHA1_ARRAY_INIT;
1038                 unsigned char (*sha1)[20] = si->shallow->sha1;
1039                 assign_shallow_commits_to_refs(si, NULL, NULL);
1040                 if (!si->nr_ours && !si->nr_theirs) {
1041                         sha1_array_clear(&ref);
1042                         return;
1043                 }
1044                 for (i = 0; i < si->nr_ours; i++)
1045                         sha1_array_append(&extra, sha1[si->ours[i]]);
1046                 for (i = 0; i < si->nr_theirs; i++)
1047                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1048                 setup_alternate_shallow(&shallow_lock,
1049                                         &alternate_shallow_file,
1050                                         &extra);
1051                 commit_lock_file(&shallow_lock);
1052                 sha1_array_clear(&extra);
1053                 sha1_array_clear(&ref);
1054                 return;
1055         }
1056
1057         /*
1058          * remote is also shallow, check what ref is safe to update
1059          * without updating .git/shallow
1060          */
1061         status = xcalloc(nr_sought, sizeof(*status));
1062         assign_shallow_commits_to_refs(si, NULL, status);
1063         if (si->nr_ours || si->nr_theirs) {
1064                 for (i = 0; i < nr_sought; i++)
1065                         if (status[i])
1066                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1067         }
1068         free(status);
1069         sha1_array_clear(&ref);
1070 }
1071
1072 struct ref *fetch_pack(struct fetch_pack_args *args,
1073                        int fd[], struct child_process *conn,
1074                        const struct ref *ref,
1075                        const char *dest,
1076                        struct ref **sought, int nr_sought,
1077                        struct sha1_array *shallow,
1078                        char **pack_lockfile)
1079 {
1080         struct ref *ref_cpy;
1081         struct shallow_info si;
1082
1083         fetch_pack_setup();
1084         if (nr_sought)
1085                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1086
1087         if (!ref) {
1088                 packet_flush(fd[1]);
1089                 die(_("no matching remote head"));
1090         }
1091         prepare_shallow_info(&si, shallow);
1092         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1093                                 &si, pack_lockfile);
1094         reprepare_packed_git();
1095         update_shallow(args, sought, nr_sought, &si);
1096         clear_shallow_info(&si);
1097         return ref_cpy;
1098 }
1099
1100 int report_unmatched_refs(struct ref **sought, int nr_sought)
1101 {
1102         int i, ret = 0;
1103
1104         for (i = 0; i < nr_sought; i++) {
1105                 if (!sought[i])
1106                         continue;
1107                 switch (sought[i]->match_status) {
1108                 case REF_MATCHED:
1109                         continue;
1110                 case REF_NOT_MATCHED:
1111                         error(_("no such remote ref %s"), sought[i]->name);
1112                         break;
1113                 case REF_UNADVERTISED_NOT_ALLOWED:
1114                         error(_("Server does not allow request for unadvertised object %s"),
1115                               sought[i]->name);
1116                         break;
1117                 }
1118                 ret = 1;
1119         }
1120         return ret;
1121 }