builtin/fetch-pack.c

   1 #include "builtin.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "transport.h"
  13 #include "version.h"
  14
  15 static int transfer_unpack_limit = -1;
  16 static int fetch_unpack_limit = -1;
  17 static int unpack_limit = 100;
  18 static int prefer_ofs_delta = 1;
  19 static int no_done;
  20 static int fetch_fsck_objects = -1;
  21 static int transfer_fsck_objects = -1;
  22 static int agent_supported;
  23 static struct fetch_pack_args args = {
  24         /* .uploadpack = */ "git-upload-pack",
  25 };
  26
  27 static const char fetch_pack_usage[] =
  28 "git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
  29 "[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
  30 "[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
  31
  32 #define COMPLETE        (1U << 0)
  33 #define COMMON          (1U << 1)
  34 #define COMMON_REF      (1U << 2)
  35 #define SEEN            (1U << 3)
  36 #define POPPED          (1U << 4)
  37
  38 static int marked;
  39
  40 /*
  41  * After sending this many "have"s if we do not get any new ACK , we
  42  * give up traversing our history.
  43  */
  44 #define MAX_IN_VAIN 256
  45
  46 static struct commit_list *rev_list;
  47 static int non_common_revs, multi_ack, use_sideband;
  48
  49 static void rev_list_push(struct commit *commit, int mark)
  50 {
  51         if (!(commit->object.flags & mark)) {
  52                 commit->object.flags |= mark;
  53
  54                 if (!(commit->object.parsed))
  55                         if (parse_commit(commit))
  56                                 return;
  57
  58                 commit_list_insert_by_date(commit, &rev_list);
  59
  60                 if (!(commit->object.flags & COMMON))
  61                         non_common_revs++;
  62         }
  63 }
  64
  65 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  66 {
  67         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  68
  69         if (o && o->type == OBJ_COMMIT)
  70                 rev_list_push((struct commit *)o, SEEN);
  71
  72         return 0;
  73 }
  74
  75 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  76 {
  77         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  78
  79         if (o && o->type == OBJ_COMMIT)
  80                 clear_commit_marks((struct commit *)o,
  81                                    COMMON | COMMON_REF | SEEN | POPPED);
  82         return 0;
  83 }
  84
  85 /*
  86    This function marks a rev and its ancestors as common.
  87    In some cases, it is desirable to mark only the ancestors (for example
  88    when only the server does not yet know that they are common).
  89 */
  90
  91 static void mark_common(struct commit *commit,
  92                 int ancestors_only, int dont_parse)
  93 {
  94         if (commit != NULL && !(commit->object.flags & COMMON)) {
  95                 struct object *o = (struct object *)commit;
  96
  97                 if (!ancestors_only)
  98                         o->flags |= COMMON;
  99
 100                 if (!(o->flags & SEEN))
 101                         rev_list_push(commit, SEEN);
 102                 else {
 103                         struct commit_list *parents;
 104
 105                         if (!ancestors_only && !(o->flags & POPPED))
 106                                 non_common_revs--;
 107                         if (!o->parsed && !dont_parse)
 108                                 if (parse_commit(commit))
 109                                         return;
 110
 111                         for (parents = commit->parents;
 112                                         parents;
 113                                         parents = parents->next)
 114                                 mark_common(parents->item, 0, dont_parse);
 115                 }
 116         }
 117 }
 118
 119 /*
 120   Get the next rev to send, ignoring the common.
 121 */
 122
 123 static const unsigned char *get_rev(void)
 124 {
 125         struct commit *commit = NULL;
 126
 127         while (commit == NULL) {
 128                 unsigned int mark;
 129                 struct commit_list *parents;
 130
 131                 if (rev_list == NULL || non_common_revs == 0)
 132                         return NULL;
 133
 134                 commit = rev_list->item;
 135                 if (!commit->object.parsed)
 136                         parse_commit(commit);
 137                 parents = commit->parents;
 138
 139                 commit->object.flags |= POPPED;
 140                 if (!(commit->object.flags & COMMON))
 141                         non_common_revs--;
 142
 143                 if (commit->object.flags & COMMON) {
 144                         /* do not send "have", and ignore ancestors */
 145                         commit = NULL;
 146                         mark = COMMON | SEEN;
 147                 } else if (commit->object.flags & COMMON_REF)
 148                         /* send "have", and ignore ancestors */
 149                         mark = COMMON | SEEN;
 150                 else
 151                         /* send "have", also for its ancestors */
 152                         mark = SEEN;
 153
 154                 while (parents) {
 155                         if (!(parents->item->object.flags & SEEN))
 156                                 rev_list_push(parents->item, mark);
 157                         if (mark & COMMON)
 158                                 mark_common(parents->item, 1, 0);
 159                         parents = parents->next;
 160                 }
 161
 162                 rev_list = rev_list->next;
 163         }
 164
 165         return commit->object.sha1;
 166 }
 167
 168 enum ack_type {
 169         NAK = 0,
 170         ACK,
 171         ACK_continue,
 172         ACK_common,
 173         ACK_ready
 174 };
 175
 176 static void consume_shallow_list(int fd)
 177 {
 178         if (args.stateless_rpc && args.depth > 0) {
 179                 /* If we sent a depth we will get back "duplicate"
 180                  * shallow and unshallow commands every time there
 181                  * is a block of have lines exchanged.
 182                  */
 183                 char line[1000];
 184                 while (packet_read_line(fd, line, sizeof(line))) {
 185                         if (!prefixcmp(line, "shallow "))
 186                                 continue;
 187                         if (!prefixcmp(line, "unshallow "))
 188                                 continue;
 189                         die("git fetch-pack: expected shallow list");
 190                 }
 191         }
 192 }
 193
 194 struct write_shallow_data {
 195         struct strbuf *out;
 196         int use_pack_protocol;
 197         int count;
 198 };
 199
 200 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 201 {
 202         struct write_shallow_data *data = cb_data;
 203         const char *hex = sha1_to_hex(graft->sha1);
 204         data->count++;
 205         if (data->use_pack_protocol)
 206                 packet_buf_write(data->out, "shallow %s", hex);
 207         else {
 208                 strbuf_addstr(data->out, hex);
 209                 strbuf_addch(data->out, '\n');
 210         }
 211         return 0;
 212 }
 213
 214 static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 215 {
 216         struct write_shallow_data data;
 217         data.out = out;
 218         data.use_pack_protocol = use_pack_protocol;
 219         data.count = 0;
 220         for_each_commit_graft(write_one_shallow, &data);
 221         return data.count;
 222 }
 223
 224 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 225 {
 226         static char line[1000];
 227         int len = packet_read_line(fd, line, sizeof(line));
 228
 229         if (!len)
 230                 die("git fetch-pack: expected ACK/NAK, got EOF");
 231         if (line[len-1] == '\n')
 232                 line[--len] = 0;
 233         if (!strcmp(line, "NAK"))
 234                 return NAK;
 235         if (!prefixcmp(line, "ACK ")) {
 236                 if (!get_sha1_hex(line+4, result_sha1)) {
 237                         if (strstr(line+45, "continue"))
 238                                 return ACK_continue;
 239                         if (strstr(line+45, "common"))
 240                                 return ACK_common;
 241                         if (strstr(line+45, "ready"))
 242                                 return ACK_ready;
 243                         return ACK;
 244                 }
 245         }
 246         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 247 }
 248
 249 static void send_request(int fd, struct strbuf *buf)
 250 {
 251         if (args.stateless_rpc) {
 252                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 253                 packet_flush(fd);
 254         } else
 255                 safe_write(fd, buf->buf, buf->len);
 256 }
 257
 258 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 259 {
 260         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 261 }
 262
 263 #define INITIAL_FLUSH 16
 264 #define PIPESAFE_FLUSH 32
 265 #define LARGE_FLUSH 1024
 266
 267 static int next_flush(int count)
 268 {
 269         int flush_limit = args.stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 270
 271         if (count < flush_limit)
 272                 count <<= 1;
 273         else
 274                 count += flush_limit;
 275         return count;
 276 }
 277
 278 static int find_common(int fd[2], unsigned char *result_sha1,
 279                        struct ref *refs)
 280 {
 281         int fetching;
 282         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 283         const unsigned char *sha1;
 284         unsigned in_vain = 0;
 285         int got_continue = 0;
 286         int got_ready = 0;
 287         struct strbuf req_buf = STRBUF_INIT;
 288         size_t state_len = 0;
 289
 290         if (args.stateless_rpc && multi_ack == 1)
 291                 die("--stateless-rpc requires multi_ack_detailed");
 292         if (marked)
 293                 for_each_ref(clear_marks, NULL);
 294         marked = 1;
 295
 296         for_each_ref(rev_list_insert_ref, NULL);
 297         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 298
 299         fetching = 0;
 300         for ( ; refs ; refs = refs->next) {
 301                 unsigned char *remote = refs->old_sha1;
 302                 const char *remote_hex;
 303                 struct object *o;
 304
 305                 /*
 306                  * If that object is complete (i.e. it is an ancestor of a
 307                  * local ref), we tell them we have it but do not have to
 308                  * tell them about its ancestors, which they already know
 309                  * about.
 310                  *
 311                  * We use lookup_object here because we are only
 312                  * interested in the case we *know* the object is
 313                  * reachable and we have already scanned it.
 314                  */
 315                 if (((o = lookup_object(remote)) != NULL) &&
 316                                 (o->flags & COMPLETE)) {
 317                         continue;
 318                 }
 319
 320                 remote_hex = sha1_to_hex(remote);
 321                 if (!fetching) {
 322                         struct strbuf c = STRBUF_INIT;
 323                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 324                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 325                         if (no_done)            strbuf_addstr(&c, " no-done");
 326                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 327                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 328                         if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack");
 329                         if (args.no_progress)   strbuf_addstr(&c, " no-progress");
 330                         if (args.include_tag)   strbuf_addstr(&c, " include-tag");
 331                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 332                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 333                                                             git_user_agent_sanitized());
 334                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 335                         strbuf_release(&c);
 336                 } else
 337                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 338                 fetching++;
 339         }
 340
 341         if (!fetching) {
 342                 strbuf_release(&req_buf);
 343                 packet_flush(fd[1]);
 344                 return 1;
 345         }
 346
 347         if (is_repository_shallow())
 348                 write_shallow_commits(&req_buf, 1);
 349         if (args.depth > 0)
 350                 packet_buf_write(&req_buf, "deepen %d", args.depth);
 351         packet_buf_flush(&req_buf);
 352         state_len = req_buf.len;
 353
 354         if (args.depth > 0) {
 355                 char line[1024];
 356                 unsigned char sha1[20];
 357
 358                 send_request(fd[1], &req_buf);
 359                 while (packet_read_line(fd[0], line, sizeof(line))) {
 360                         if (!prefixcmp(line, "shallow ")) {
 361                                 if (get_sha1_hex(line + 8, sha1))
 362                                         die("invalid shallow line: %s", line);
 363                                 register_shallow(sha1);
 364                                 continue;
 365                         }
 366                         if (!prefixcmp(line, "unshallow ")) {
 367                                 if (get_sha1_hex(line + 10, sha1))
 368                                         die("invalid unshallow line: %s", line);
 369                                 if (!lookup_object(sha1))
 370                                         die("object not found: %s", line);
 371                                 /* make sure that it is parsed as shallow */
 372                                 if (!parse_object(sha1))
 373                                         die("error in object: %s", line);
 374                                 if (unregister_shallow(sha1))
 375                                         die("no shallow found: %s", line);
 376                                 continue;
 377                         }
 378                         die("expected shallow/unshallow, got %s", line);
 379                 }
 380         } else if (!args.stateless_rpc)
 381                 send_request(fd[1], &req_buf);
 382
 383         if (!args.stateless_rpc) {
 384                 /* If we aren't using the stateless-rpc interface
 385                  * we don't need to retain the headers.
 386                  */
 387                 strbuf_setlen(&req_buf, 0);
 388                 state_len = 0;
 389         }
 390
 391         flushes = 0;
 392         retval = -1;
 393         while ((sha1 = get_rev())) {
 394                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 395                 if (args.verbose)
 396                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 397                 in_vain++;
 398                 if (flush_at <= ++count) {
 399                         int ack;
 400
 401                         packet_buf_flush(&req_buf);
 402                         send_request(fd[1], &req_buf);
 403                         strbuf_setlen(&req_buf, state_len);
 404                         flushes++;
 405                         flush_at = next_flush(count);
 406
 407                         /*
 408                          * We keep one window "ahead" of the other side, and
 409                          * will wait for an ACK only on the next one
 410                          */
 411                         if (!args.stateless_rpc && count == INITIAL_FLUSH)
 412                                 continue;
 413
 414                         consume_shallow_list(fd[0]);
 415                         do {
 416                                 ack = get_ack(fd[0], result_sha1);
 417                                 if (args.verbose && ack)
 418                                         fprintf(stderr, "got ack %d %s\n", ack,
 419                                                         sha1_to_hex(result_sha1));
 420                                 switch (ack) {
 421                                 case ACK:
 422                                         flushes = 0;
 423                                         multi_ack = 0;
 424                                         retval = 0;
 425                                         goto done;
 426                                 case ACK_common:
 427                                 case ACK_ready:
 428                                 case ACK_continue: {
 429                                         struct commit *commit =
 430                                                 lookup_commit(result_sha1);
 431                                         if (!commit)
 432                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 433                                         if (args.stateless_rpc
 434                                          && ack == ACK_common
 435                                          && !(commit->object.flags & COMMON)) {
 436                                                 /* We need to replay the have for this object
 437                                                  * on the next RPC request so the peer knows
 438                                                  * it is in common with us.
 439                                                  */
 440                                                 const char *hex = sha1_to_hex(result_sha1);
 441                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 442                                                 state_len = req_buf.len;
 443                                         }
 444                                         mark_common(commit, 0, 1);
 445                                         retval = 0;
 446                                         in_vain = 0;
 447                                         got_continue = 1;
 448                                         if (ack == ACK_ready) {
 449                                                 rev_list = NULL;
 450                                                 got_ready = 1;
 451                                         }
 452                                         break;
 453                                         }
 454                                 }
 455                         } while (ack);
 456                         flushes--;
 457                         if (got_continue && MAX_IN_VAIN < in_vain) {
 458                                 if (args.verbose)
 459                                         fprintf(stderr, "giving up\n");
 460                                 break; /* give up */
 461                         }
 462                 }
 463         }
 464 done:
 465         if (!got_ready || !no_done) {
 466                 packet_buf_write(&req_buf, "done\n");
 467                 send_request(fd[1], &req_buf);
 468         }
 469         if (args.verbose)
 470                 fprintf(stderr, "done\n");
 471         if (retval != 0) {
 472                 multi_ack = 0;
 473                 flushes++;
 474         }
 475         strbuf_release(&req_buf);
 476
 477         consume_shallow_list(fd[0]);
 478         while (flushes || multi_ack) {
 479                 int ack = get_ack(fd[0], result_sha1);
 480                 if (ack) {
 481                         if (args.verbose)
 482                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 483                                         sha1_to_hex(result_sha1));
 484                         if (ack == ACK)
 485                                 return 0;
 486                         multi_ack = 1;
 487                         continue;
 488                 }
 489                 flushes--;
 490         }
 491         /* it is no error to fetch into a completely empty repo */
 492         return count ? retval : 0;
 493 }
 494
 495 static struct commit_list *complete;
 496
 497 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 498 {
 499         struct object *o = parse_object(sha1);
 500
 501         while (o && o->type == OBJ_TAG) {
 502                 struct tag *t = (struct tag *) o;
 503                 if (!t->tagged)
 504                         break; /* broken repository */
 505                 o->flags |= COMPLETE;
 506                 o = parse_object(t->tagged->sha1);
 507         }
 508         if (o && o->type == OBJ_COMMIT) {
 509                 struct commit *commit = (struct commit *)o;
 510                 if (!(commit->object.flags & COMPLETE)) {
 511                         commit->object.flags |= COMPLETE;
 512                         commit_list_insert_by_date(commit, &complete);
 513                 }
 514         }
 515         return 0;
 516 }
 517
 518 static void mark_recent_complete_commits(unsigned long cutoff)
 519 {
 520         while (complete && cutoff <= complete->item->date) {
 521                 if (args.verbose)
 522                         fprintf(stderr, "Marking %s as complete\n",
 523                                 sha1_to_hex(complete->item->object.sha1));
 524                 pop_most_recent_commit(&complete, COMPLETE);
 525         }
 526 }
 527
 528 static int non_matching_ref(struct string_list_item *item, void *unused)
 529 {
 530         if (item->util) {
 531                 item->util = NULL;
 532                 return 0;
 533         }
 534         else
 535                 return 1;
 536 }
 537
 538 static void filter_refs(struct ref **refs, struct string_list *sought)
 539 {
 540         struct ref *newlist = NULL;
 541         struct ref **newtail = &newlist;
 542         struct ref *ref, *next;
 543         int sought_pos;
 544
 545         sought_pos = 0;
 546         for (ref = *refs; ref; ref = next) {
 547                 int keep = 0;
 548                 next = ref->next;
 549                 if (!memcmp(ref->name, "refs/", 5) &&
 550                     check_refname_format(ref->name + 5, 0))
 551                         ; /* trash */
 552                 else {
 553                         while (sought_pos < sought->nr) {
 554                                 int cmp = strcmp(ref->name, sought->items[sought_pos].string);
 555                                 if (cmp < 0)
 556                                         break; /* definitely do not have it */
 557                                 else if (cmp == 0) {
 558                                         keep = 1; /* definitely have it */
 559                                         sought->items[sought_pos++].util = "matched";
 560                                         break;
 561                                 }
 562                                 else
 563                                         sought_pos++; /* might have it; keep looking */
 564                         }
 565                 }
 566
 567                 if (! keep && args.fetch_all &&
 568                     (!args.depth || prefixcmp(ref->name, "refs/tags/")))
 569                         keep = 1;
 570
 571                 if (keep) {
 572                         *newtail = ref;
 573                         ref->next = NULL;
 574                         newtail = &ref->next;
 575                 } else {
 576                         free(ref);
 577                 }
 578         }
 579
 580         filter_string_list(sought, 0, non_matching_ref, NULL);
 581         *refs = newlist;
 582 }
 583
 584 static void mark_alternate_complete(const struct ref *ref, void *unused)
 585 {
 586         mark_complete(NULL, ref->old_sha1, 0, NULL);
 587 }
 588
 589 static int everything_local(struct ref **refs, struct string_list *sought)
 590 {
 591         struct ref *ref;
 592         int retval;
 593         unsigned long cutoff = 0;
 594
 595         save_commit_buffer = 0;
 596
 597         for (ref = *refs; ref; ref = ref->next) {
 598                 struct object *o;
 599
 600                 o = parse_object(ref->old_sha1);
 601                 if (!o)
 602                         continue;
 603
 604                 /* We already have it -- which may mean that we were
 605                  * in sync with the other side at some time after
 606                  * that (it is OK if we guess wrong here).
 607                  */
 608                 if (o->type == OBJ_COMMIT) {
 609                         struct commit *commit = (struct commit *)o;
 610                         if (!cutoff || cutoff < commit->date)
 611                                 cutoff = commit->date;
 612                 }
 613         }
 614
 615         if (!args.depth) {
 616                 for_each_ref(mark_complete, NULL);
 617                 for_each_alternate_ref(mark_alternate_complete, NULL);
 618                 if (cutoff)
 619                         mark_recent_complete_commits(cutoff);
 620         }
 621
 622         /*
 623          * Mark all complete remote refs as common refs.
 624          * Don't mark them common yet; the server has to be told so first.
 625          */
 626         for (ref = *refs; ref; ref = ref->next) {
 627                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 628                                              NULL, 0);
 629
 630                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 631                         continue;
 632
 633                 if (!(o->flags & SEEN)) {
 634                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 635
 636                         mark_common((struct commit *)o, 1, 1);
 637                 }
 638         }
 639
 640         filter_refs(refs, sought);
 641
 642         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 643                 const unsigned char *remote = ref->old_sha1;
 644                 unsigned char local[20];
 645                 struct object *o;
 646
 647                 o = lookup_object(remote);
 648                 if (!o || !(o->flags & COMPLETE)) {
 649                         retval = 0;
 650                         if (!args.verbose)
 651                                 continue;
 652                         fprintf(stderr,
 653                                 "want %s (%s)\n", sha1_to_hex(remote),
 654                                 ref->name);
 655                         continue;
 656                 }
 657
 658                 hashcpy(ref->new_sha1, local);
 659                 if (!args.verbose)
 660                         continue;
 661                 fprintf(stderr,
 662                         "already have %s (%s)\n", sha1_to_hex(remote),
 663                         ref->name);
 664         }
 665         return retval;
 666 }
 667
 668 static int sideband_demux(int in, int out, void *data)
 669 {
 670         int *xd = data;
 671
 672         int ret = recv_sideband("fetch-pack", xd[0], out);
 673         close(out);
 674         return ret;
 675 }
 676
 677 static int get_pack(int xd[2], char **pack_lockfile)
 678 {
 679         struct async demux;
 680         const char *argv[20];
 681         char keep_arg[256];
 682         char hdr_arg[256];
 683         const char **av;
 684         int do_keep = args.keep_pack;
 685         struct child_process cmd;
 686
 687         memset(&demux, 0, sizeof(demux));
 688         if (use_sideband) {
 689                 /* xd[] is talking with upload-pack; subprocess reads from
 690                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 691                  * through demux->out.
 692                  */
 693                 demux.proc = sideband_demux;
 694                 demux.data = xd;
 695                 demux.out = -1;
 696                 if (start_async(&demux))
 697                         die("fetch-pack: unable to fork off sideband"
 698                             " demultiplexer");
 699         }
 700         else
 701                 demux.out = xd[0];
 702
 703         memset(&cmd, 0, sizeof(cmd));
 704         cmd.argv = argv;
 705         av = argv;
 706         *hdr_arg = 0;
 707         if (!args.keep_pack && unpack_limit) {
 708                 struct pack_header header;
 709
 710                 if (read_pack_header(demux.out, &header))
 711                         die("protocol error: bad pack header");
 712                 snprintf(hdr_arg, sizeof(hdr_arg),
 713                          "--pack_header=%"PRIu32",%"PRIu32,
 714                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 715                 if (ntohl(header.hdr_entries) < unpack_limit)
 716                         do_keep = 0;
 717                 else
 718                         do_keep = 1;
 719         }
 720
 721         if (do_keep) {
 722                 if (pack_lockfile)
 723                         cmd.out = -1;
 724                 *av++ = "index-pack";
 725                 *av++ = "--stdin";
 726                 if (!args.quiet && !args.no_progress)
 727                         *av++ = "-v";
 728                 if (args.use_thin_pack)
 729                         *av++ = "--fix-thin";
 730                 if (args.lock_pack || unpack_limit) {
 731                         int s = sprintf(keep_arg,
 732                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 733                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 734                                 strcpy(keep_arg + s, "localhost");
 735                         *av++ = keep_arg;
 736                 }
 737         }
 738         else {
 739                 *av++ = "unpack-objects";
 740                 if (args.quiet || args.no_progress)
 741                         *av++ = "-q";
 742         }
 743         if (*hdr_arg)
 744                 *av++ = hdr_arg;
 745         if (fetch_fsck_objects >= 0
 746             ? fetch_fsck_objects
 747             : transfer_fsck_objects >= 0
 748             ? transfer_fsck_objects
 749             : 0)
 750                 *av++ = "--strict";
 751         *av++ = NULL;
 752
 753         cmd.in = demux.out;
 754         cmd.git_cmd = 1;
 755         if (start_command(&cmd))
 756                 die("fetch-pack: unable to fork off %s", argv[0]);
 757         if (do_keep && pack_lockfile) {
 758                 *pack_lockfile = index_pack_lockfile(cmd.out);
 759                 close(cmd.out);
 760         }
 761
 762         if (finish_command(&cmd))
 763                 die("%s failed", argv[0]);
 764         if (use_sideband && finish_async(&demux))
 765                 die("error in sideband demultiplexer");
 766         return 0;
 767 }
 768
 769 static struct ref *do_fetch_pack(int fd[2],
 770                 const struct ref *orig_ref,
 771                 struct string_list *sought,
 772                 char **pack_lockfile)
 773 {
 774         struct ref *ref = copy_ref_list(orig_ref);
 775         unsigned char sha1[20];
 776         const char *agent_feature;
 777         int agent_len;
 778
 779         sort_ref_list(&ref, ref_compare_name);
 780
 781         if (is_repository_shallow() && !server_supports("shallow"))
 782                 die("Server does not support shallow clients");
 783         if (server_supports("multi_ack_detailed")) {
 784                 if (args.verbose)
 785                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 786                 multi_ack = 2;
 787                 if (server_supports("no-done")) {
 788                         if (args.verbose)
 789                                 fprintf(stderr, "Server supports no-done\n");
 790                         if (args.stateless_rpc)
 791                                 no_done = 1;
 792                 }
 793         }
 794         else if (server_supports("multi_ack")) {
 795                 if (args.verbose)
 796                         fprintf(stderr, "Server supports multi_ack\n");
 797                 multi_ack = 1;
 798         }
 799         if (server_supports("side-band-64k")) {
 800                 if (args.verbose)
 801                         fprintf(stderr, "Server supports side-band-64k\n");
 802                 use_sideband = 2;
 803         }
 804         else if (server_supports("side-band")) {
 805                 if (args.verbose)
 806                         fprintf(stderr, "Server supports side-band\n");
 807                 use_sideband = 1;
 808         }
 809         if (!server_supports("thin-pack"))
 810                 args.use_thin_pack = 0;
 811         if (!server_supports("no-progress"))
 812                 args.no_progress = 0;
 813         if (!server_supports("include-tag"))
 814                 args.include_tag = 0;
 815         if (server_supports("ofs-delta")) {
 816                 if (args.verbose)
 817                         fprintf(stderr, "Server supports ofs-delta\n");
 818         } else
 819                 prefer_ofs_delta = 0;
 820
 821         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 822                 agent_supported = 1;
 823                 if (args.verbose && agent_len)
 824                         fprintf(stderr, "Server version is %.*s\n",
 825                                 agent_len, agent_feature);
 826         }
 827
 828         if (everything_local(&ref, sought)) {
 829                 packet_flush(fd[1]);
 830                 goto all_done;
 831         }
 832         if (find_common(fd, sha1, ref) < 0)
 833                 if (!args.keep_pack)
 834                         /* When cloning, it is not unusual to have
 835                          * no common commit.
 836                          */
 837                         warning("no common commits");
 838
 839         if (args.stateless_rpc)
 840                 packet_flush(fd[1]);
 841         if (get_pack(fd, pack_lockfile))
 842                 die("git fetch-pack: fetch failed.");
 843
 844  all_done:
 845         return ref;
 846 }
 847
 848 static int fetch_pack_config(const char *var, const char *value, void *cb)
 849 {
 850         if (strcmp(var, "fetch.unpacklimit") == 0) {
 851                 fetch_unpack_limit = git_config_int(var, value);
 852                 return 0;
 853         }
 854
 855         if (strcmp(var, "transfer.unpacklimit") == 0) {
 856                 transfer_unpack_limit = git_config_int(var, value);
 857                 return 0;
 858         }
 859
 860         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 861                 prefer_ofs_delta = git_config_bool(var, value);
 862                 return 0;
 863         }
 864
 865         if (!strcmp(var, "fetch.fsckobjects")) {
 866                 fetch_fsck_objects = git_config_bool(var, value);
 867                 return 0;
 868         }
 869
 870         if (!strcmp(var, "transfer.fsckobjects")) {
 871                 transfer_fsck_objects = git_config_bool(var, value);
 872                 return 0;
 873         }
 874
 875         return git_default_config(var, value, cb);
 876 }
 877
 878 static struct lock_file lock;
 879
 880 static void fetch_pack_setup(void)
 881 {
 882         static int did_setup;
 883         if (did_setup)
 884                 return;
 885         git_config(fetch_pack_config, NULL);
 886         if (0 <= transfer_unpack_limit)
 887                 unpack_limit = transfer_unpack_limit;
 888         else if (0 <= fetch_unpack_limit)
 889                 unpack_limit = fetch_unpack_limit;
 890         did_setup = 1;
 891 }
 892
 893 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 894 {
 895         int i, ret;
 896         struct ref *ref = NULL;
 897         const char *dest = NULL;
 898         struct string_list sought = STRING_LIST_INIT_DUP;
 899         int fd[2];
 900         char *pack_lockfile = NULL;
 901         char **pack_lockfile_ptr = NULL;
 902         struct child_process *conn;
 903
 904         packet_trace_identity("fetch-pack");
 905
 906         for (i = 1; i < argc && *argv[i] == '-'; i++) {
 907                 const char *arg = argv[i];
 908
 909                 if (!prefixcmp(arg, "--upload-pack=")) {
 910                         args.uploadpack = arg + 14;
 911                         continue;
 912                 }
 913                 if (!prefixcmp(arg, "--exec=")) {
 914                         args.uploadpack = arg + 7;
 915                         continue;
 916                 }
 917                 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 918                         args.quiet = 1;
 919                         continue;
 920                 }
 921                 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 922                         args.lock_pack = args.keep_pack;
 923                         args.keep_pack = 1;
 924                         continue;
 925                 }
 926                 if (!strcmp("--thin", arg)) {
 927                         args.use_thin_pack = 1;
 928                         continue;
 929                 }
 930                 if (!strcmp("--include-tag", arg)) {
 931                         args.include_tag = 1;
 932                         continue;
 933                 }
 934                 if (!strcmp("--all", arg)) {
 935                         args.fetch_all = 1;
 936                         continue;
 937                 }
 938                 if (!strcmp("--stdin", arg)) {
 939                         args.stdin_refs = 1;
 940                         continue;
 941                 }
 942                 if (!strcmp("-v", arg)) {
 943                         args.verbose = 1;
 944                         continue;
 945                 }
 946                 if (!prefixcmp(arg, "--depth=")) {
 947                         args.depth = strtol(arg + 8, NULL, 0);
 948                         continue;
 949                 }
 950                 if (!strcmp("--no-progress", arg)) {
 951                         args.no_progress = 1;
 952                         continue;
 953                 }
 954                 if (!strcmp("--stateless-rpc", arg)) {
 955                         args.stateless_rpc = 1;
 956                         continue;
 957                 }
 958                 if (!strcmp("--lock-pack", arg)) {
 959                         args.lock_pack = 1;
 960                         pack_lockfile_ptr = &pack_lockfile;
 961                         continue;
 962                 }
 963                 usage(fetch_pack_usage);
 964         }
 965
 966         if (i < argc)
 967                 dest = argv[i++];
 968         else
 969                 usage(fetch_pack_usage);
 970
 971         /*
 972          * Copy refs from cmdline to growable list, then append any
 973          * refs from the standard input:
 974          */
 975         for (; i < argc; i++)
 976                 string_list_append(&sought, xstrdup(argv[i]));
 977         if (args.stdin_refs) {
 978                 if (args.stateless_rpc) {
 979                         /* in stateless RPC mode we use pkt-line to read
 980                          * from stdin, until we get a flush packet
 981                          */
 982                         static char line[1000];
 983                         for (;;) {
 984                                 int n = packet_read_line(0, line, sizeof(line));
 985                                 if (!n)
 986                                         break;
 987                                 if (line[n-1] == '\n')
 988                                         n--;
 989                                 string_list_append(&sought, xmemdupz(line, n));
 990                         }
 991                 }
 992                 else {
 993                         /* read from stdin one ref per line, until EOF */
 994                         struct strbuf line = STRBUF_INIT;
 995                         while (strbuf_getline(&line, stdin, '\n') != EOF)
 996                                 string_list_append(&sought, strbuf_detach(&line, NULL));
 997                         strbuf_release(&line);
 998                 }
 999         }
1000
1001         if (args.stateless_rpc) {
1002                 conn = NULL;
1003                 fd[0] = 0;
1004                 fd[1] = 1;
1005         } else {
1006                 conn = git_connect(fd, dest, args.uploadpack,
1007                                    args.verbose ? CONNECT_VERBOSE : 0);
1008         }
1009
1010         get_remote_heads(fd[0], &ref, 0, NULL);
1011
1012         ref = fetch_pack(&args, fd, conn, ref, dest,
1013                          &sought, pack_lockfile_ptr);
1014         if (pack_lockfile) {
1015                 printf("lock %s\n", pack_lockfile);
1016                 fflush(stdout);
1017         }
1018         close(fd[0]);
1019         close(fd[1]);
1020         if (finish_connect(conn))
1021                 return 1;
1022
1023         ret = !ref || sought.nr;
1024
1025         /*
1026          * If the heads to pull were given, we should have consumed
1027          * all of them by matching the remote.  Otherwise, 'git fetch
1028          * remote no-such-ref' would silently succeed without issuing
1029          * an error.
1030          */
1031         for (i = 0; i < sought.nr; i++)
1032                 error("no such remote ref %s", sought.items[i].string);
1033         while (ref) {
1034                 printf("%s %s\n",
1035                        sha1_to_hex(ref->old_sha1), ref->name);
1036                 ref = ref->next;
1037         }
1038
1039         return ret;
1040 }
1041
1042 struct ref *fetch_pack(struct fetch_pack_args *my_args,
1043                        int fd[], struct child_process *conn,
1044                        const struct ref *ref,
1045                        const char *dest,
1046                        struct string_list *sought,
1047                        char **pack_lockfile)
1048 {
1049         struct stat st;
1050         struct ref *ref_cpy;
1051
1052         fetch_pack_setup();
1053         if (&args != my_args)
1054                 memcpy(&args, my_args, sizeof(args));
1055         if (args.depth > 0) {
1056                 if (stat(git_path("shallow"), &st))
1057                         st.st_mtime = 0;
1058         }
1059
1060         if (sought->nr) {
1061                 sort_string_list(sought);
1062                 string_list_remove_duplicates(sought, 0);
1063         }
1064
1065         if (!ref) {
1066                 packet_flush(fd[1]);
1067                 die("no matching remote head");
1068         }
1069         ref_cpy = do_fetch_pack(fd, ref, sought, pack_lockfile);
1070
1071         if (args.depth > 0) {
1072                 struct cache_time mtime;
1073                 struct strbuf sb = STRBUF_INIT;
1074                 char *shallow = git_path("shallow");
1075                 int fd;
1076
1077                 mtime.sec = st.st_mtime;
1078                 mtime.nsec = ST_MTIME_NSEC(st);
1079                 if (stat(shallow, &st)) {
1080                         if (mtime.sec)
1081                                 die("shallow file was removed during fetch");
1082                 } else if (st.st_mtime != mtime.sec
1083 #ifdef USE_NSEC
1084                                 || ST_MTIME_NSEC(st) != mtime.nsec
1085 #endif
1086                           )
1087                         die("shallow file was changed during fetch");
1088
1089                 fd = hold_lock_file_for_update(&lock, shallow,
1090                                                LOCK_DIE_ON_ERROR);
1091                 if (!write_shallow_commits(&sb, 0)
1092                  || write_in_full(fd, sb.buf, sb.len) != sb.len) {
1093                         unlink_or_warn(shallow);
1094                         rollback_lock_file(&lock);
1095                 } else {
1096                         commit_lock_file(&lock);
1097                 }
1098                 strbuf_release(&sb);
1099         }
1100
1101         reprepare_packed_git();
1102         return ref_cpy;
1103 }