fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "connect.h"
  13 #include "transport.h"
  14 #include "version.h"
  15 #include "prio-queue.h"
  16 #include "sha1-array.h"
  17
  18 static int transfer_unpack_limit = -1;
  19 static int fetch_unpack_limit = -1;
  20 static int unpack_limit = 100;
  21 static int prefer_ofs_delta = 1;
  22 static int no_done;
  23 static int fetch_fsck_objects = -1;
  24 static int transfer_fsck_objects = -1;
  25 static int agent_supported;
  26 static struct lock_file shallow_lock;
  27 static const char *alternate_shallow_file;
  28
  29 /* Remember to update object flag allocation in object.h */
  30 #define COMPLETE        (1U << 0)
  31 #define COMMON          (1U << 1)
  32 #define COMMON_REF      (1U << 2)
  33 #define SEEN            (1U << 3)
  34 #define POPPED          (1U << 4)
  35
  36 static int marked;
  37
  38 /*
  39  * After sending this many "have"s if we do not get any new ACK , we
  40  * give up traversing our history.
  41  */
  42 #define MAX_IN_VAIN 256
  43
  44 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  45 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  46
  47 static void rev_list_push(struct commit *commit, int mark)
  48 {
  49         if (!(commit->object.flags & mark)) {
  50                 commit->object.flags |= mark;
  51
  52                 if (parse_commit(commit))
  53                         return;
  54
  55                 prio_queue_put(&rev_list, commit);
  56
  57                 if (!(commit->object.flags & COMMON))
  58                         non_common_revs++;
  59         }
  60 }
  61
  62 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  63 {
  64         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  65
  66         if (o && o->type == OBJ_COMMIT)
  67                 rev_list_push((struct commit *)o, SEEN);
  68
  69         return 0;
  70 }
  71
  72 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  73 {
  74         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  75
  76         if (o && o->type == OBJ_COMMIT)
  77                 clear_commit_marks((struct commit *)o,
  78                                    COMMON | COMMON_REF | SEEN | POPPED);
  79         return 0;
  80 }
  81
  82 /*
  83    This function marks a rev and its ancestors as common.
  84    In some cases, it is desirable to mark only the ancestors (for example
  85    when only the server does not yet know that they are common).
  86 */
  87
  88 static void mark_common(struct commit *commit,
  89                 int ancestors_only, int dont_parse)
  90 {
  91         if (commit != NULL && !(commit->object.flags & COMMON)) {
  92                 struct object *o = (struct object *)commit;
  93
  94                 if (!ancestors_only)
  95                         o->flags |= COMMON;
  96
  97                 if (!(o->flags & SEEN))
  98                         rev_list_push(commit, SEEN);
  99                 else {
 100                         struct commit_list *parents;
 101
 102                         if (!ancestors_only && !(o->flags & POPPED))
 103                                 non_common_revs--;
 104                         if (!o->parsed && !dont_parse)
 105                                 if (parse_commit(commit))
 106                                         return;
 107
 108                         for (parents = commit->parents;
 109                                         parents;
 110                                         parents = parents->next)
 111                                 mark_common(parents->item, 0, dont_parse);
 112                 }
 113         }
 114 }
 115
 116 /*
 117   Get the next rev to send, ignoring the common.
 118 */
 119
 120 static const unsigned char *get_rev(void)
 121 {
 122         struct commit *commit = NULL;
 123
 124         while (commit == NULL) {
 125                 unsigned int mark;
 126                 struct commit_list *parents;
 127
 128                 if (rev_list.nr == 0 || non_common_revs == 0)
 129                         return NULL;
 130
 131                 commit = prio_queue_get(&rev_list);
 132                 parse_commit(commit);
 133                 parents = commit->parents;
 134
 135                 commit->object.flags |= POPPED;
 136                 if (!(commit->object.flags & COMMON))
 137                         non_common_revs--;
 138
 139                 if (commit->object.flags & COMMON) {
 140                         /* do not send "have", and ignore ancestors */
 141                         commit = NULL;
 142                         mark = COMMON | SEEN;
 143                 } else if (commit->object.flags & COMMON_REF)
 144                         /* send "have", and ignore ancestors */
 145                         mark = COMMON | SEEN;
 146                 else
 147                         /* send "have", also for its ancestors */
 148                         mark = SEEN;
 149
 150                 while (parents) {
 151                         if (!(parents->item->object.flags & SEEN))
 152                                 rev_list_push(parents->item, mark);
 153                         if (mark & COMMON)
 154                                 mark_common(parents->item, 1, 0);
 155                         parents = parents->next;
 156                 }
 157         }
 158
 159         return commit->object.sha1;
 160 }
 161
 162 enum ack_type {
 163         NAK = 0,
 164         ACK,
 165         ACK_continue,
 166         ACK_common,
 167         ACK_ready
 168 };
 169
 170 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 171 {
 172         if (args->stateless_rpc && args->depth > 0) {
 173                 /* If we sent a depth we will get back "duplicate"
 174                  * shallow and unshallow commands every time there
 175                  * is a block of have lines exchanged.
 176                  */
 177                 char *line;
 178                 while ((line = packet_read_line(fd, NULL))) {
 179                         if (starts_with(line, "shallow "))
 180                                 continue;
 181                         if (starts_with(line, "unshallow "))
 182                                 continue;
 183                         die("git fetch-pack: expected shallow list");
 184                 }
 185         }
 186 }
 187
 188 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 189 {
 190         int len;
 191         char *line = packet_read_line(fd, &len);
 192
 193         if (!len)
 194                 die("git fetch-pack: expected ACK/NAK, got EOF");
 195         if (!strcmp(line, "NAK"))
 196                 return NAK;
 197         if (starts_with(line, "ACK ")) {
 198                 if (!get_sha1_hex(line+4, result_sha1)) {
 199                         if (len < 45)
 200                                 return ACK;
 201                         if (strstr(line+45, "continue"))
 202                                 return ACK_continue;
 203                         if (strstr(line+45, "common"))
 204                                 return ACK_common;
 205                         if (strstr(line+45, "ready"))
 206                                 return ACK_ready;
 207                         return ACK;
 208                 }
 209         }
 210         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 211 }
 212
 213 static void send_request(struct fetch_pack_args *args,
 214                          int fd, struct strbuf *buf)
 215 {
 216         if (args->stateless_rpc) {
 217                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 218                 packet_flush(fd);
 219         } else
 220                 write_or_die(fd, buf->buf, buf->len);
 221 }
 222
 223 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 224 {
 225         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 226 }
 227
 228 #define INITIAL_FLUSH 16
 229 #define PIPESAFE_FLUSH 32
 230 #define LARGE_FLUSH 1024
 231
 232 static int next_flush(struct fetch_pack_args *args, int count)
 233 {
 234         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 235
 236         if (count < flush_limit)
 237                 count <<= 1;
 238         else
 239                 count += flush_limit;
 240         return count;
 241 }
 242
 243 static int find_common(struct fetch_pack_args *args,
 244                        int fd[2], unsigned char *result_sha1,
 245                        struct ref *refs)
 246 {
 247         int fetching;
 248         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 249         const unsigned char *sha1;
 250         unsigned in_vain = 0;
 251         int got_continue = 0;
 252         int got_ready = 0;
 253         struct strbuf req_buf = STRBUF_INIT;
 254         size_t state_len = 0;
 255
 256         if (args->stateless_rpc && multi_ack == 1)
 257                 die("--stateless-rpc requires multi_ack_detailed");
 258         if (marked)
 259                 for_each_ref(clear_marks, NULL);
 260         marked = 1;
 261
 262         for_each_ref(rev_list_insert_ref, NULL);
 263         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 264
 265         fetching = 0;
 266         for ( ; refs ; refs = refs->next) {
 267                 unsigned char *remote = refs->old_sha1;
 268                 const char *remote_hex;
 269                 struct object *o;
 270
 271                 /*
 272                  * If that object is complete (i.e. it is an ancestor of a
 273                  * local ref), we tell them we have it but do not have to
 274                  * tell them about its ancestors, which they already know
 275                  * about.
 276                  *
 277                  * We use lookup_object here because we are only
 278                  * interested in the case we *know* the object is
 279                  * reachable and we have already scanned it.
 280                  */
 281                 if (((o = lookup_object(remote)) != NULL) &&
 282                                 (o->flags & COMPLETE)) {
 283                         continue;
 284                 }
 285
 286                 remote_hex = sha1_to_hex(remote);
 287                 if (!fetching) {
 288                         struct strbuf c = STRBUF_INIT;
 289                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 290                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 291                         if (no_done)            strbuf_addstr(&c, " no-done");
 292                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 293                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 294                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 295                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 296                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 297                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 298                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 299                                                             git_user_agent_sanitized());
 300                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 301                         strbuf_release(&c);
 302                 } else
 303                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 304                 fetching++;
 305         }
 306
 307         if (!fetching) {
 308                 strbuf_release(&req_buf);
 309                 packet_flush(fd[1]);
 310                 return 1;
 311         }
 312
 313         if (is_repository_shallow())
 314                 write_shallow_commits(&req_buf, 1, NULL);
 315         if (args->depth > 0)
 316                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 317         packet_buf_flush(&req_buf);
 318         state_len = req_buf.len;
 319
 320         if (args->depth > 0) {
 321                 char *line;
 322                 const char *arg;
 323                 unsigned char sha1[20];
 324
 325                 send_request(args, fd[1], &req_buf);
 326                 while ((line = packet_read_line(fd[0], NULL))) {
 327                         if (skip_prefix(line, "shallow ", &arg)) {
 328                                 if (get_sha1_hex(arg, sha1))
 329                                         die("invalid shallow line: %s", line);
 330                                 register_shallow(sha1);
 331                                 continue;
 332                         }
 333                         if (skip_prefix(line, "unshallow ", &arg)) {
 334                                 if (get_sha1_hex(arg, sha1))
 335                                         die("invalid unshallow line: %s", line);
 336                                 if (!lookup_object(sha1))
 337                                         die("object not found: %s", line);
 338                                 /* make sure that it is parsed as shallow */
 339                                 if (!parse_object(sha1))
 340                                         die("error in object: %s", line);
 341                                 if (unregister_shallow(sha1))
 342                                         die("no shallow found: %s", line);
 343                                 continue;
 344                         }
 345                         die("expected shallow/unshallow, got %s", line);
 346                 }
 347         } else if (!args->stateless_rpc)
 348                 send_request(args, fd[1], &req_buf);
 349
 350         if (!args->stateless_rpc) {
 351                 /* If we aren't using the stateless-rpc interface
 352                  * we don't need to retain the headers.
 353                  */
 354                 strbuf_setlen(&req_buf, 0);
 355                 state_len = 0;
 356         }
 357
 358         flushes = 0;
 359         retval = -1;
 360         while ((sha1 = get_rev())) {
 361                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 362                 if (args->verbose)
 363                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 364                 in_vain++;
 365                 if (flush_at <= ++count) {
 366                         int ack;
 367
 368                         packet_buf_flush(&req_buf);
 369                         send_request(args, fd[1], &req_buf);
 370                         strbuf_setlen(&req_buf, state_len);
 371                         flushes++;
 372                         flush_at = next_flush(args, count);
 373
 374                         /*
 375                          * We keep one window "ahead" of the other side, and
 376                          * will wait for an ACK only on the next one
 377                          */
 378                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 379                                 continue;
 380
 381                         consume_shallow_list(args, fd[0]);
 382                         do {
 383                                 ack = get_ack(fd[0], result_sha1);
 384                                 if (args->verbose && ack)
 385                                         fprintf(stderr, "got ack %d %s\n", ack,
 386                                                         sha1_to_hex(result_sha1));
 387                                 switch (ack) {
 388                                 case ACK:
 389                                         flushes = 0;
 390                                         multi_ack = 0;
 391                                         retval = 0;
 392                                         goto done;
 393                                 case ACK_common:
 394                                 case ACK_ready:
 395                                 case ACK_continue: {
 396                                         struct commit *commit =
 397                                                 lookup_commit(result_sha1);
 398                                         if (!commit)
 399                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 400                                         if (args->stateless_rpc
 401                                          && ack == ACK_common
 402                                          && !(commit->object.flags & COMMON)) {
 403                                                 /* We need to replay the have for this object
 404                                                  * on the next RPC request so the peer knows
 405                                                  * it is in common with us.
 406                                                  */
 407                                                 const char *hex = sha1_to_hex(result_sha1);
 408                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 409                                                 state_len = req_buf.len;
 410                                         }
 411                                         mark_common(commit, 0, 1);
 412                                         retval = 0;
 413                                         in_vain = 0;
 414                                         got_continue = 1;
 415                                         if (ack == ACK_ready) {
 416                                                 clear_prio_queue(&rev_list);
 417                                                 got_ready = 1;
 418                                         }
 419                                         break;
 420                                         }
 421                                 }
 422                         } while (ack);
 423                         flushes--;
 424                         if (got_continue && MAX_IN_VAIN < in_vain) {
 425                                 if (args->verbose)
 426                                         fprintf(stderr, "giving up\n");
 427                                 break; /* give up */
 428                         }
 429                 }
 430         }
 431 done:
 432         if (!got_ready || !no_done) {
 433                 packet_buf_write(&req_buf, "done\n");
 434                 send_request(args, fd[1], &req_buf);
 435         }
 436         if (args->verbose)
 437                 fprintf(stderr, "done\n");
 438         if (retval != 0) {
 439                 multi_ack = 0;
 440                 flushes++;
 441         }
 442         strbuf_release(&req_buf);
 443
 444         if (!got_ready || !no_done)
 445                 consume_shallow_list(args, fd[0]);
 446         while (flushes || multi_ack) {
 447                 int ack = get_ack(fd[0], result_sha1);
 448                 if (ack) {
 449                         if (args->verbose)
 450                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 451                                         sha1_to_hex(result_sha1));
 452                         if (ack == ACK)
 453                                 return 0;
 454                         multi_ack = 1;
 455                         continue;
 456                 }
 457                 flushes--;
 458         }
 459         /* it is no error to fetch into a completely empty repo */
 460         return count ? retval : 0;
 461 }
 462
 463 static struct commit_list *complete;
 464
 465 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 466 {
 467         struct object *o = parse_object(sha1);
 468
 469         while (o && o->type == OBJ_TAG) {
 470                 struct tag *t = (struct tag *) o;
 471                 if (!t->tagged)
 472                         break; /* broken repository */
 473                 o->flags |= COMPLETE;
 474                 o = parse_object(t->tagged->sha1);
 475         }
 476         if (o && o->type == OBJ_COMMIT) {
 477                 struct commit *commit = (struct commit *)o;
 478                 if (!(commit->object.flags & COMPLETE)) {
 479                         commit->object.flags |= COMPLETE;
 480                         commit_list_insert(commit, &complete);
 481                 }
 482         }
 483         return 0;
 484 }
 485
 486 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 487                                          unsigned long cutoff)
 488 {
 489         while (complete && cutoff <= complete->item->date) {
 490                 if (args->verbose)
 491                         fprintf(stderr, "Marking %s as complete\n",
 492                                 sha1_to_hex(complete->item->object.sha1));
 493                 pop_most_recent_commit(&complete, COMPLETE);
 494         }
 495 }
 496
 497 static void filter_refs(struct fetch_pack_args *args,
 498                         struct ref **refs,
 499                         struct ref **sought, int nr_sought)
 500 {
 501         struct ref *newlist = NULL;
 502         struct ref **newtail = &newlist;
 503         struct ref *ref, *next;
 504         int i;
 505
 506         i = 0;
 507         for (ref = *refs; ref; ref = next) {
 508                 int keep = 0;
 509                 next = ref->next;
 510
 511                 if (!memcmp(ref->name, "refs/", 5) &&
 512                     check_refname_format(ref->name, 0))
 513                         ; /* trash */
 514                 else {
 515                         while (i < nr_sought) {
 516                                 int cmp = strcmp(ref->name, sought[i]->name);
 517                                 if (cmp < 0)
 518                                         break; /* definitely do not have it */
 519                                 else if (cmp == 0) {
 520                                         keep = 1; /* definitely have it */
 521                                         sought[i]->matched = 1;
 522                                 }
 523                                 i++;
 524                         }
 525                 }
 526
 527                 if (!keep && args->fetch_all &&
 528                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 529                         keep = 1;
 530
 531                 if (keep) {
 532                         *newtail = ref;
 533                         ref->next = NULL;
 534                         newtail = &ref->next;
 535                 } else {
 536                         free(ref);
 537                 }
 538         }
 539
 540         /* Append unmatched requests to the list */
 541         if (allow_tip_sha1_in_want) {
 542                 for (i = 0; i < nr_sought; i++) {
 543                         ref = sought[i];
 544                         if (ref->matched)
 545                                 continue;
 546                         if (get_sha1_hex(ref->name, ref->old_sha1))
 547                                 continue;
 548
 549                         ref->matched = 1;
 550                         *newtail = ref;
 551                         ref->next = NULL;
 552                         newtail = &ref->next;
 553                 }
 554         }
 555         *refs = newlist;
 556 }
 557
 558 static void mark_alternate_complete(const struct ref *ref, void *unused)
 559 {
 560         mark_complete(NULL, ref->old_sha1, 0, NULL);
 561 }
 562
 563 static int everything_local(struct fetch_pack_args *args,
 564                             struct ref **refs,
 565                             struct ref **sought, int nr_sought)
 566 {
 567         struct ref *ref;
 568         int retval;
 569         unsigned long cutoff = 0;
 570
 571         save_commit_buffer = 0;
 572
 573         for (ref = *refs; ref; ref = ref->next) {
 574                 struct object *o;
 575
 576                 if (!has_sha1_file(ref->old_sha1))
 577                         continue;
 578
 579                 o = parse_object(ref->old_sha1);
 580                 if (!o)
 581                         continue;
 582
 583                 /* We already have it -- which may mean that we were
 584                  * in sync with the other side at some time after
 585                  * that (it is OK if we guess wrong here).
 586                  */
 587                 if (o->type == OBJ_COMMIT) {
 588                         struct commit *commit = (struct commit *)o;
 589                         if (!cutoff || cutoff < commit->date)
 590                                 cutoff = commit->date;
 591                 }
 592         }
 593
 594         if (!args->depth) {
 595                 for_each_ref(mark_complete, NULL);
 596                 for_each_alternate_ref(mark_alternate_complete, NULL);
 597                 commit_list_sort_by_date(&complete);
 598                 if (cutoff)
 599                         mark_recent_complete_commits(args, cutoff);
 600         }
 601
 602         /*
 603          * Mark all complete remote refs as common refs.
 604          * Don't mark them common yet; the server has to be told so first.
 605          */
 606         for (ref = *refs; ref; ref = ref->next) {
 607                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 608                                              NULL, 0);
 609
 610                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 611                         continue;
 612
 613                 if (!(o->flags & SEEN)) {
 614                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 615
 616                         mark_common((struct commit *)o, 1, 1);
 617                 }
 618         }
 619
 620         filter_refs(args, refs, sought, nr_sought);
 621
 622         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 623                 const unsigned char *remote = ref->old_sha1;
 624                 unsigned char local[20];
 625                 struct object *o;
 626
 627                 o = lookup_object(remote);
 628                 if (!o || !(o->flags & COMPLETE)) {
 629                         retval = 0;
 630                         if (!args->verbose)
 631                                 continue;
 632                         fprintf(stderr,
 633                                 "want %s (%s)\n", sha1_to_hex(remote),
 634                                 ref->name);
 635                         continue;
 636                 }
 637
 638                 hashcpy(ref->new_sha1, local);
 639                 if (!args->verbose)
 640                         continue;
 641                 fprintf(stderr,
 642                         "already have %s (%s)\n", sha1_to_hex(remote),
 643                         ref->name);
 644         }
 645         return retval;
 646 }
 647
 648 static int sideband_demux(int in, int out, void *data)
 649 {
 650         int *xd = data;
 651
 652         int ret = recv_sideband("fetch-pack", xd[0], out);
 653         close(out);
 654         return ret;
 655 }
 656
 657 static int get_pack(struct fetch_pack_args *args,
 658                     int xd[2], char **pack_lockfile)
 659 {
 660         struct async demux;
 661         const char *argv[22];
 662         char keep_arg[256];
 663         char hdr_arg[256];
 664         const char **av, *cmd_name;
 665         int do_keep = args->keep_pack;
 666         struct child_process cmd;
 667         int ret;
 668
 669         memset(&demux, 0, sizeof(demux));
 670         if (use_sideband) {
 671                 /* xd[] is talking with upload-pack; subprocess reads from
 672                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 673                  * through demux->out.
 674                  */
 675                 demux.proc = sideband_demux;
 676                 demux.data = xd;
 677                 demux.out = -1;
 678                 if (start_async(&demux))
 679                         die("fetch-pack: unable to fork off sideband"
 680                             " demultiplexer");
 681         }
 682         else
 683                 demux.out = xd[0];
 684
 685         memset(&cmd, 0, sizeof(cmd));
 686         cmd.argv = argv;
 687         av = argv;
 688         *hdr_arg = 0;
 689         if (!args->keep_pack && unpack_limit) {
 690                 struct pack_header header;
 691
 692                 if (read_pack_header(demux.out, &header))
 693                         die("protocol error: bad pack header");
 694                 snprintf(hdr_arg, sizeof(hdr_arg),
 695                          "--pack_header=%"PRIu32",%"PRIu32,
 696                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 697                 if (ntohl(header.hdr_entries) < unpack_limit)
 698                         do_keep = 0;
 699                 else
 700                         do_keep = 1;
 701         }
 702
 703         if (alternate_shallow_file) {
 704                 *av++ = "--shallow-file";
 705                 *av++ = alternate_shallow_file;
 706         }
 707
 708         if (do_keep) {
 709                 if (pack_lockfile)
 710                         cmd.out = -1;
 711                 *av++ = cmd_name = "index-pack";
 712                 *av++ = "--stdin";
 713                 if (!args->quiet && !args->no_progress)
 714                         *av++ = "-v";
 715                 if (args->use_thin_pack)
 716                         *av++ = "--fix-thin";
 717                 if (args->lock_pack || unpack_limit) {
 718                         int s = sprintf(keep_arg,
 719                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 720                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 721                                 strcpy(keep_arg + s, "localhost");
 722                         *av++ = keep_arg;
 723                 }
 724                 if (args->check_self_contained_and_connected)
 725                         *av++ = "--check-self-contained-and-connected";
 726         }
 727         else {
 728                 *av++ = cmd_name = "unpack-objects";
 729                 if (args->quiet || args->no_progress)
 730                         *av++ = "-q";
 731                 args->check_self_contained_and_connected = 0;
 732         }
 733         if (*hdr_arg)
 734                 *av++ = hdr_arg;
 735         if (fetch_fsck_objects >= 0
 736             ? fetch_fsck_objects
 737             : transfer_fsck_objects >= 0
 738             ? transfer_fsck_objects
 739             : 0)
 740                 *av++ = "--strict";
 741         *av++ = NULL;
 742
 743         cmd.in = demux.out;
 744         cmd.git_cmd = 1;
 745         if (start_command(&cmd))
 746                 die("fetch-pack: unable to fork off %s", cmd_name);
 747         if (do_keep && pack_lockfile) {
 748                 *pack_lockfile = index_pack_lockfile(cmd.out);
 749                 close(cmd.out);
 750         }
 751
 752         if (!use_sideband)
 753                 /* Closed by start_command() */
 754                 xd[0] = -1;
 755
 756         ret = finish_command(&cmd);
 757         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 758                 args->self_contained_and_connected =
 759                         args->check_self_contained_and_connected &&
 760                         ret == 0;
 761         else
 762                 die("%s failed", cmd_name);
 763         if (use_sideband && finish_async(&demux))
 764                 die("error in sideband demultiplexer");
 765         return 0;
 766 }
 767
 768 static int cmp_ref_by_name(const void *a_, const void *b_)
 769 {
 770         const struct ref *a = *((const struct ref **)a_);
 771         const struct ref *b = *((const struct ref **)b_);
 772         return strcmp(a->name, b->name);
 773 }
 774
 775 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 776                                  int fd[2],
 777                                  const struct ref *orig_ref,
 778                                  struct ref **sought, int nr_sought,
 779                                  struct shallow_info *si,
 780                                  char **pack_lockfile)
 781 {
 782         struct ref *ref = copy_ref_list(orig_ref);
 783         unsigned char sha1[20];
 784         const char *agent_feature;
 785         int agent_len;
 786
 787         sort_ref_list(&ref, ref_compare_name);
 788         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 789
 790         if (is_repository_shallow() && !server_supports("shallow"))
 791                 die("Server does not support shallow clients");
 792         if (server_supports("multi_ack_detailed")) {
 793                 if (args->verbose)
 794                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 795                 multi_ack = 2;
 796                 if (server_supports("no-done")) {
 797                         if (args->verbose)
 798                                 fprintf(stderr, "Server supports no-done\n");
 799                         if (args->stateless_rpc)
 800                                 no_done = 1;
 801                 }
 802         }
 803         else if (server_supports("multi_ack")) {
 804                 if (args->verbose)
 805                         fprintf(stderr, "Server supports multi_ack\n");
 806                 multi_ack = 1;
 807         }
 808         if (server_supports("side-band-64k")) {
 809                 if (args->verbose)
 810                         fprintf(stderr, "Server supports side-band-64k\n");
 811                 use_sideband = 2;
 812         }
 813         else if (server_supports("side-band")) {
 814                 if (args->verbose)
 815                         fprintf(stderr, "Server supports side-band\n");
 816                 use_sideband = 1;
 817         }
 818         if (server_supports("allow-tip-sha1-in-want")) {
 819                 if (args->verbose)
 820                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 821                 allow_tip_sha1_in_want = 1;
 822         }
 823         if (!server_supports("thin-pack"))
 824                 args->use_thin_pack = 0;
 825         if (!server_supports("no-progress"))
 826                 args->no_progress = 0;
 827         if (!server_supports("include-tag"))
 828                 args->include_tag = 0;
 829         if (server_supports("ofs-delta")) {
 830                 if (args->verbose)
 831                         fprintf(stderr, "Server supports ofs-delta\n");
 832         } else
 833                 prefer_ofs_delta = 0;
 834
 835         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 836                 agent_supported = 1;
 837                 if (args->verbose && agent_len)
 838                         fprintf(stderr, "Server version is %.*s\n",
 839                                 agent_len, agent_feature);
 840         }
 841
 842         if (everything_local(args, &ref, sought, nr_sought)) {
 843                 packet_flush(fd[1]);
 844                 goto all_done;
 845         }
 846         if (find_common(args, fd, sha1, ref) < 0)
 847                 if (!args->keep_pack)
 848                         /* When cloning, it is not unusual to have
 849                          * no common commit.
 850                          */
 851                         warning("no common commits");
 852
 853         if (args->stateless_rpc)
 854                 packet_flush(fd[1]);
 855         if (args->depth > 0)
 856                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 857                                         NULL);
 858         else if (si->nr_ours || si->nr_theirs)
 859                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 860         else
 861                 alternate_shallow_file = NULL;
 862         if (get_pack(args, fd, pack_lockfile))
 863                 die("git fetch-pack: fetch failed.");
 864
 865  all_done:
 866         return ref;
 867 }
 868
 869 static int fetch_pack_config(const char *var, const char *value, void *cb)
 870 {
 871         if (strcmp(var, "fetch.unpacklimit") == 0) {
 872                 fetch_unpack_limit = git_config_int(var, value);
 873                 return 0;
 874         }
 875
 876         if (strcmp(var, "transfer.unpacklimit") == 0) {
 877                 transfer_unpack_limit = git_config_int(var, value);
 878                 return 0;
 879         }
 880
 881         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 882                 prefer_ofs_delta = git_config_bool(var, value);
 883                 return 0;
 884         }
 885
 886         if (!strcmp(var, "fetch.fsckobjects")) {
 887                 fetch_fsck_objects = git_config_bool(var, value);
 888                 return 0;
 889         }
 890
 891         if (!strcmp(var, "transfer.fsckobjects")) {
 892                 transfer_fsck_objects = git_config_bool(var, value);
 893                 return 0;
 894         }
 895
 896         return git_default_config(var, value, cb);
 897 }
 898
 899 static void fetch_pack_setup(void)
 900 {
 901         static int did_setup;
 902         if (did_setup)
 903                 return;
 904         git_config(fetch_pack_config, NULL);
 905         if (0 <= transfer_unpack_limit)
 906                 unpack_limit = transfer_unpack_limit;
 907         else if (0 <= fetch_unpack_limit)
 908                 unpack_limit = fetch_unpack_limit;
 909         did_setup = 1;
 910 }
 911
 912 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 913 {
 914         struct string_list names = STRING_LIST_INIT_NODUP;
 915         int src, dst;
 916
 917         for (src = dst = 0; src < nr; src++) {
 918                 struct string_list_item *item;
 919                 item = string_list_insert(&names, ref[src]->name);
 920                 if (item->util)
 921                         continue; /* already have it */
 922                 item->util = ref[src];
 923                 if (src != dst)
 924                         ref[dst] = ref[src];
 925                 dst++;
 926         }
 927         for (src = dst; src < nr; src++)
 928                 ref[src] = NULL;
 929         string_list_clear(&names, 0);
 930         return dst;
 931 }
 932
 933 static void update_shallow(struct fetch_pack_args *args,
 934                            struct ref **sought, int nr_sought,
 935                            struct shallow_info *si)
 936 {
 937         struct sha1_array ref = SHA1_ARRAY_INIT;
 938         int *status;
 939         int i;
 940
 941         if (args->depth > 0 && alternate_shallow_file) {
 942                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 943                         unlink_or_warn(git_path("shallow"));
 944                         rollback_lock_file(&shallow_lock);
 945                 } else
 946                         commit_lock_file(&shallow_lock);
 947                 return;
 948         }
 949
 950         if (!si->shallow || !si->shallow->nr)
 951                 return;
 952
 953         if (args->cloning) {
 954                 /*
 955                  * remote is shallow, but this is a clone, there are
 956                  * no objects in repo to worry about. Accept any
 957                  * shallow points that exist in the pack (iow in repo
 958                  * after get_pack() and reprepare_packed_git())
 959                  */
 960                 struct sha1_array extra = SHA1_ARRAY_INIT;
 961                 unsigned char (*sha1)[20] = si->shallow->sha1;
 962                 for (i = 0; i < si->shallow->nr; i++)
 963                         if (has_sha1_file(sha1[i]))
 964                                 sha1_array_append(&extra, sha1[i]);
 965                 if (extra.nr) {
 966                         setup_alternate_shallow(&shallow_lock,
 967                                                 &alternate_shallow_file,
 968                                                 &extra);
 969                         commit_lock_file(&shallow_lock);
 970                 }
 971                 sha1_array_clear(&extra);
 972                 return;
 973         }
 974
 975         if (!si->nr_ours && !si->nr_theirs)
 976                 return;
 977
 978         remove_nonexistent_theirs_shallow(si);
 979         if (!si->nr_ours && !si->nr_theirs)
 980                 return;
 981         for (i = 0; i < nr_sought; i++)
 982                 sha1_array_append(&ref, sought[i]->old_sha1);
 983         si->ref = &ref;
 984
 985         if (args->update_shallow) {
 986                 /*
 987                  * remote is also shallow, .git/shallow may be updated
 988                  * so all refs can be accepted. Make sure we only add
 989                  * shallow roots that are actually reachable from new
 990                  * refs.
 991                  */
 992                 struct sha1_array extra = SHA1_ARRAY_INIT;
 993                 unsigned char (*sha1)[20] = si->shallow->sha1;
 994                 assign_shallow_commits_to_refs(si, NULL, NULL);
 995                 if (!si->nr_ours && !si->nr_theirs) {
 996                         sha1_array_clear(&ref);
 997                         return;
 998                 }
 999                 for (i = 0; i < si->nr_ours; i++)
1000                         sha1_array_append(&extra, sha1[si->ours[i]]);
1001                 for (i = 0; i < si->nr_theirs; i++)
1002                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1003                 setup_alternate_shallow(&shallow_lock,
1004                                         &alternate_shallow_file,
1005                                         &extra);
1006                 commit_lock_file(&shallow_lock);
1007                 sha1_array_clear(&extra);
1008                 sha1_array_clear(&ref);
1009                 return;
1010         }
1011
1012         /*
1013          * remote is also shallow, check what ref is safe to update
1014          * without updating .git/shallow
1015          */
1016         status = xcalloc(nr_sought, sizeof(*status));
1017         assign_shallow_commits_to_refs(si, NULL, status);
1018         if (si->nr_ours || si->nr_theirs) {
1019                 for (i = 0; i < nr_sought; i++)
1020                         if (status[i])
1021                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1022         }
1023         free(status);
1024         sha1_array_clear(&ref);
1025 }
1026
1027 struct ref *fetch_pack(struct fetch_pack_args *args,
1028                        int fd[], struct child_process *conn,
1029                        const struct ref *ref,
1030                        const char *dest,
1031                        struct ref **sought, int nr_sought,
1032                        struct sha1_array *shallow,
1033                        char **pack_lockfile)
1034 {
1035         struct ref *ref_cpy;
1036         struct shallow_info si;
1037
1038         fetch_pack_setup();
1039         if (nr_sought)
1040                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1041
1042         if (!ref) {
1043                 packet_flush(fd[1]);
1044                 die("no matching remote head");
1045         }
1046         prepare_shallow_info(&si, shallow);
1047         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1048                                 &si, pack_lockfile);
1049         reprepare_packed_git();
1050         update_shallow(args, sought, nr_sought, &si);
1051         clear_shallow_info(&si);
1052         return ref_cpy;
1053 }