git-p4.py

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10 import sys
  11 if sys.hexversion < 0x02040000:
  12     # The limiter is the subprocess module
  13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
  14     sys.exit(1)
  15 import os
  16 import optparse
  17 import marshal
  18 import subprocess
  19 import tempfile
  20 import time
  21 import platform
  22 import re
  23 import shutil
  24 import stat
  25 import zipfile
  26 import zlib
  27 import ctypes
  28 import errno
  29
  30 try:
  31     from subprocess import CalledProcessError
  32 except ImportError:
  33     # from python2.7:subprocess.py
  34     # Exception classes used by this module.
  35     class CalledProcessError(Exception):
  36         """This exception is raised when a process run by check_call() returns
  37         a non-zero exit status.  The exit status will be stored in the
  38         returncode attribute."""
  39         def __init__(self, returncode, cmd):
  40             self.returncode = returncode
  41             self.cmd = cmd
  42         def __str__(self):
  43             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
  44
  45 verbose = False
  46
  47 # Only labels/tags matching this will be imported/exported
  48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  49
  50 # The block size is reduced automatically if required
  51 defaultBlockSize = 1<<20
  52
  53 p4_access_checked = False
  54
  55 def p4_build_cmd(cmd):
  56     """Build a suitable p4 command line.
  57
  58     This consolidates building and returning a p4 command line into one
  59     location. It means that hooking into the environment, or other configuration
  60     can be done more easily.
  61     """
  62     real_cmd = ["p4"]
  63
  64     user = gitConfig("git-p4.user")
  65     if len(user) > 0:
  66         real_cmd += ["-u",user]
  67
  68     password = gitConfig("git-p4.password")
  69     if len(password) > 0:
  70         real_cmd += ["-P", password]
  71
  72     port = gitConfig("git-p4.port")
  73     if len(port) > 0:
  74         real_cmd += ["-p", port]
  75
  76     host = gitConfig("git-p4.host")
  77     if len(host) > 0:
  78         real_cmd += ["-H", host]
  79
  80     client = gitConfig("git-p4.client")
  81     if len(client) > 0:
  82         real_cmd += ["-c", client]
  83
  84     retries = gitConfigInt("git-p4.retries")
  85     if retries is None:
  86         # Perform 3 retries by default
  87         retries = 3
  88     if retries > 0:
  89         # Provide a way to not pass this option by setting git-p4.retries to 0
  90         real_cmd += ["-r", str(retries)]
  91
  92     if isinstance(cmd,basestring):
  93         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  94     else:
  95         real_cmd += cmd
  96
  97     # now check that we can actually talk to the server
  98     global p4_access_checked
  99     if not p4_access_checked:
 100         p4_access_checked = True    # suppress access checks in p4_check_access itself
 101         p4_check_access()
 102
 103     return real_cmd
 104
 105 def git_dir(path):
 106     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
 107         This won't automatically add ".git" to a directory.
 108     """
 109     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
 110     if not d or len(d) == 0:
 111         return None
 112     else:
 113         return d
 114
 115 def chdir(path, is_client_path=False):
 116     """Do chdir to the given path, and set the PWD environment
 117        variable for use by P4.  It does not look at getcwd() output.
 118        Since we're not using the shell, it is necessary to set the
 119        PWD environment variable explicitly.
 120
 121        Normally, expand the path to force it to be absolute.  This
 122        addresses the use of relative path names inside P4 settings,
 123        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
 124        as given; it looks for .p4config using PWD.
 125
 126        If is_client_path, the path was handed to us directly by p4,
 127        and may be a symbolic link.  Do not call os.getcwd() in this
 128        case, because it will cause p4 to think that PWD is not inside
 129        the client path.
 130        """
 131
 132     os.chdir(path)
 133     if not is_client_path:
 134         path = os.getcwd()
 135     os.environ['PWD'] = path
 136
 137 def calcDiskFree():
 138     """Return free space in bytes on the disk of the given dirname."""
 139     if platform.system() == 'Windows':
 140         free_bytes = ctypes.c_ulonglong(0)
 141         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
 142         return free_bytes.value
 143     else:
 144         st = os.statvfs(os.getcwd())
 145         return st.f_bavail * st.f_frsize
 146
 147 def die(msg):
 148     if verbose:
 149         raise Exception(msg)
 150     else:
 151         sys.stderr.write(msg + "\n")
 152         sys.exit(1)
 153
 154 def write_pipe(c, stdin):
 155     if verbose:
 156         sys.stderr.write('Writing pipe: %s\n' % str(c))
 157
 158     expand = isinstance(c,basestring)
 159     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
 160     pipe = p.stdin
 161     val = pipe.write(stdin)
 162     pipe.close()
 163     if p.wait():
 164         die('Command failed: %s' % str(c))
 165
 166     return val
 167
 168 def p4_write_pipe(c, stdin):
 169     real_cmd = p4_build_cmd(c)
 170     return write_pipe(real_cmd, stdin)
 171
 172 def read_pipe_full(c):
 173     """ Read output from  command. Returns a tuple
 174         of the return status, stdout text and stderr
 175         text.
 176     """
 177     if verbose:
 178         sys.stderr.write('Reading pipe: %s\n' % str(c))
 179
 180     expand = isinstance(c,basestring)
 181     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
 182     (out, err) = p.communicate()
 183     return (p.returncode, out, err)
 184
 185 def read_pipe(c, ignore_error=False):
 186     """ Read output from  command. Returns the output text on
 187         success. On failure, terminates execution, unless
 188         ignore_error is True, when it returns an empty string.
 189     """
 190     (retcode, out, err) = read_pipe_full(c)
 191     if retcode != 0:
 192         if ignore_error:
 193             out = ""
 194         else:
 195             die('Command failed: %s\nError: %s' % (str(c), err))
 196     return out
 197
 198 def read_pipe_text(c):
 199     """ Read output from a command with trailing whitespace stripped.
 200         On error, returns None.
 201     """
 202     (retcode, out, err) = read_pipe_full(c)
 203     if retcode != 0:
 204         return None
 205     else:
 206         return out.rstrip()
 207
 208 def p4_read_pipe(c, ignore_error=False):
 209     real_cmd = p4_build_cmd(c)
 210     return read_pipe(real_cmd, ignore_error)
 211
 212 def read_pipe_lines(c):
 213     if verbose:
 214         sys.stderr.write('Reading pipe: %s\n' % str(c))
 215
 216     expand = isinstance(c, basestring)
 217     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 218     pipe = p.stdout
 219     val = pipe.readlines()
 220     if pipe.close() or p.wait():
 221         die('Command failed: %s' % str(c))
 222
 223     return val
 224
 225 def p4_read_pipe_lines(c):
 226     """Specifically invoke p4 on the command supplied. """
 227     real_cmd = p4_build_cmd(c)
 228     return read_pipe_lines(real_cmd)
 229
 230 def p4_has_command(cmd):
 231     """Ask p4 for help on this command.  If it returns an error, the
 232        command does not exist in this version of p4."""
 233     real_cmd = p4_build_cmd(["help", cmd])
 234     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
 235                                    stderr=subprocess.PIPE)
 236     p.communicate()
 237     return p.returncode == 0
 238
 239 def p4_has_move_command():
 240     """See if the move command exists, that it supports -k, and that
 241        it has not been administratively disabled.  The arguments
 242        must be correct, but the filenames do not have to exist.  Use
 243        ones with wildcards so even if they exist, it will fail."""
 244
 245     if not p4_has_command("move"):
 246         return False
 247     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
 248     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 249     (out, err) = p.communicate()
 250     # return code will be 1 in either case
 251     if err.find("Invalid option") >= 0:
 252         return False
 253     if err.find("disabled") >= 0:
 254         return False
 255     # assume it failed because @... was invalid changelist
 256     return True
 257
 258 def system(cmd, ignore_error=False):
 259     expand = isinstance(cmd,basestring)
 260     if verbose:
 261         sys.stderr.write("executing %s\n" % str(cmd))
 262     retcode = subprocess.call(cmd, shell=expand)
 263     if retcode and not ignore_error:
 264         raise CalledProcessError(retcode, cmd)
 265
 266     return retcode
 267
 268 def p4_system(cmd):
 269     """Specifically invoke p4 as the system command. """
 270     real_cmd = p4_build_cmd(cmd)
 271     expand = isinstance(real_cmd, basestring)
 272     retcode = subprocess.call(real_cmd, shell=expand)
 273     if retcode:
 274         raise CalledProcessError(retcode, real_cmd)
 275
 276 def die_bad_access(s):
 277     die("failure accessing depot: {0}".format(s.rstrip()))
 278
 279 def p4_check_access(min_expiration=1):
 280     """ Check if we can access Perforce - account still logged in
 281     """
 282     results = p4CmdList(["login", "-s"])
 283
 284     if len(results) == 0:
 285         # should never get here: always get either some results, or a p4ExitCode
 286         assert("could not parse response from perforce")
 287
 288     result = results[0]
 289
 290     if 'p4ExitCode' in result:
 291         # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
 292         die_bad_access("could not run p4")
 293
 294     code = result.get("code")
 295     if not code:
 296         # we get here if we couldn't connect and there was nothing to unmarshal
 297         die_bad_access("could not connect")
 298
 299     elif code == "stat":
 300         expiry = result.get("TicketExpiration")
 301         if expiry:
 302             expiry = int(expiry)
 303             if expiry > min_expiration:
 304                 # ok to carry on
 305                 return
 306             else:
 307                 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
 308
 309         else:
 310             # account without a timeout - all ok
 311             return
 312
 313     elif code == "error":
 314         data = result.get("data")
 315         if data:
 316             die_bad_access("p4 error: {0}".format(data))
 317         else:
 318             die_bad_access("unknown error")
 319     else:
 320         die_bad_access("unknown error code {0}".format(code))
 321
 322 _p4_version_string = None
 323 def p4_version_string():
 324     """Read the version string, showing just the last line, which
 325        hopefully is the interesting version bit.
 326
 327        $ p4 -V
 328        Perforce - The Fast Software Configuration Management System.
 329        Copyright 1995-2011 Perforce Software.  All rights reserved.
 330        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
 331     """
 332     global _p4_version_string
 333     if not _p4_version_string:
 334         a = p4_read_pipe_lines(["-V"])
 335         _p4_version_string = a[-1].rstrip()
 336     return _p4_version_string
 337
 338 def p4_integrate(src, dest):
 339     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
 340
 341 def p4_sync(f, *options):
 342     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
 343
 344 def p4_add(f):
 345     # forcibly add file names with wildcards
 346     if wildcard_present(f):
 347         p4_system(["add", "-f", f])
 348     else:
 349         p4_system(["add", f])
 350
 351 def p4_delete(f):
 352     p4_system(["delete", wildcard_encode(f)])
 353
 354 def p4_edit(f, *options):
 355     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
 356
 357 def p4_revert(f):
 358     p4_system(["revert", wildcard_encode(f)])
 359
 360 def p4_reopen(type, f):
 361     p4_system(["reopen", "-t", type, wildcard_encode(f)])
 362
 363 def p4_reopen_in_change(changelist, files):
 364     cmd = ["reopen", "-c", str(changelist)] + files
 365     p4_system(cmd)
 366
 367 def p4_move(src, dest):
 368     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
 369
 370 def p4_last_change():
 371     results = p4CmdList(["changes", "-m", "1"], skip_info=True)
 372     return int(results[0]['change'])
 373
 374 def p4_describe(change, shelved=False):
 375     """Make sure it returns a valid result by checking for
 376        the presence of field "time".  Return a dict of the
 377        results."""
 378
 379     cmd = ["describe", "-s"]
 380     if shelved:
 381         cmd += ["-S"]
 382     cmd += [str(change)]
 383
 384     ds = p4CmdList(cmd, skip_info=True)
 385     if len(ds) != 1:
 386         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
 387
 388     d = ds[0]
 389
 390     if "p4ExitCode" in d:
 391         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
 392                                                       str(d)))
 393     if "code" in d:
 394         if d["code"] == "error":
 395             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
 396
 397     if "time" not in d:
 398         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
 399
 400     return d
 401
 402 #
 403 # Canonicalize the p4 type and return a tuple of the
 404 # base type, plus any modifiers.  See "p4 help filetypes"
 405 # for a list and explanation.
 406 #
 407 def split_p4_type(p4type):
 408
 409     p4_filetypes_historical = {
 410         "ctempobj": "binary+Sw",
 411         "ctext": "text+C",
 412         "cxtext": "text+Cx",
 413         "ktext": "text+k",
 414         "kxtext": "text+kx",
 415         "ltext": "text+F",
 416         "tempobj": "binary+FSw",
 417         "ubinary": "binary+F",
 418         "uresource": "resource+F",
 419         "uxbinary": "binary+Fx",
 420         "xbinary": "binary+x",
 421         "xltext": "text+Fx",
 422         "xtempobj": "binary+Swx",
 423         "xtext": "text+x",
 424         "xunicode": "unicode+x",
 425         "xutf16": "utf16+x",
 426     }
 427     if p4type in p4_filetypes_historical:
 428         p4type = p4_filetypes_historical[p4type]
 429     mods = ""
 430     s = p4type.split("+")
 431     base = s[0]
 432     mods = ""
 433     if len(s) > 1:
 434         mods = s[1]
 435     return (base, mods)
 436
 437 #
 438 # return the raw p4 type of a file (text, text+ko, etc)
 439 #
 440 def p4_type(f):
 441     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
 442     return results[0]['headType']
 443
 444 #
 445 # Given a type base and modifier, return a regexp matching
 446 # the keywords that can be expanded in the file
 447 #
 448 def p4_keywords_regexp_for_type(base, type_mods):
 449     if base in ("text", "unicode", "binary"):
 450         kwords = None
 451         if "ko" in type_mods:
 452             kwords = 'Id|Header'
 453         elif "k" in type_mods:
 454             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 455         else:
 456             return None
 457         pattern = r"""
 458             \$              # Starts with a dollar, followed by...
 459             (%s)            # one of the keywords, followed by...
 460             (:[^$\n]+)?     # possibly an old expansion, followed by...
 461             \$              # another dollar
 462             """ % kwords
 463         return pattern
 464     else:
 465         return None
 466
 467 #
 468 # Given a file, return a regexp matching the possible
 469 # RCS keywords that will be expanded, or None for files
 470 # with kw expansion turned off.
 471 #
 472 def p4_keywords_regexp_for_file(file):
 473     if not os.path.exists(file):
 474         return None
 475     else:
 476         (type_base, type_mods) = split_p4_type(p4_type(file))
 477         return p4_keywords_regexp_for_type(type_base, type_mods)
 478
 479 def setP4ExecBit(file, mode):
 480     # Reopens an already open file and changes the execute bit to match
 481     # the execute bit setting in the passed in mode.
 482
 483     p4Type = "+x"
 484
 485     if not isModeExec(mode):
 486         p4Type = getP4OpenedType(file)
 487         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 488         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 489         if p4Type[-1] == "+":
 490             p4Type = p4Type[0:-1]
 491
 492     p4_reopen(p4Type, file)
 493
 494 def getP4OpenedType(file):
 495     # Returns the perforce file type for the given file.
 496
 497     result = p4_read_pipe(["opened", wildcard_encode(file)])
 498     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
 499     if match:
 500         return match.group(1)
 501     else:
 502         die("Could not determine file type for %s (result: '%s')" % (file, result))
 503
 504 # Return the set of all p4 labels
 505 def getP4Labels(depotPaths):
 506     labels = set()
 507     if isinstance(depotPaths,basestring):
 508         depotPaths = [depotPaths]
 509
 510     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 511         label = l['label']
 512         labels.add(label)
 513
 514     return labels
 515
 516 # Return the set of all git tags
 517 def getGitTags():
 518     gitTags = set()
 519     for line in read_pipe_lines(["git", "tag"]):
 520         tag = line.strip()
 521         gitTags.add(tag)
 522     return gitTags
 523
 524 def diffTreePattern():
 525     # This is a simple generator for the diff tree regex pattern. This could be
 526     # a class variable if this and parseDiffTreeEntry were a part of a class.
 527     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 528     while True:
 529         yield pattern
 530
 531 def parseDiffTreeEntry(entry):
 532     """Parses a single diff tree entry into its component elements.
 533
 534     See git-diff-tree(1) manpage for details about the format of the diff
 535     output. This method returns a dictionary with the following elements:
 536
 537     src_mode - The mode of the source file
 538     dst_mode - The mode of the destination file
 539     src_sha1 - The sha1 for the source file
 540     dst_sha1 - The sha1 fr the destination file
 541     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 542     status_score - The score for the status (applicable for 'C' and 'R'
 543                    statuses). This is None if there is no score.
 544     src - The path for the source file.
 545     dst - The path for the destination file. This is only present for
 546           copy or renames. If it is not present, this is None.
 547
 548     If the pattern is not matched, None is returned."""
 549
 550     match = diffTreePattern().next().match(entry)
 551     if match:
 552         return {
 553             'src_mode': match.group(1),
 554             'dst_mode': match.group(2),
 555             'src_sha1': match.group(3),
 556             'dst_sha1': match.group(4),
 557             'status': match.group(5),
 558             'status_score': match.group(6),
 559             'src': match.group(7),
 560             'dst': match.group(10)
 561         }
 562     return None
 563
 564 def isModeExec(mode):
 565     # Returns True if the given git mode represents an executable file,
 566     # otherwise False.
 567     return mode[-3:] == "755"
 568
 569 class P4Exception(Exception):
 570     """ Base class for exceptions from the p4 client """
 571     def __init__(self, exit_code):
 572         self.p4ExitCode = exit_code
 573
 574 class P4ServerException(P4Exception):
 575     """ Base class for exceptions where we get some kind of marshalled up result from the server """
 576     def __init__(self, exit_code, p4_result):
 577         super(P4ServerException, self).__init__(exit_code)
 578         self.p4_result = p4_result
 579         self.code = p4_result[0]['code']
 580         self.data = p4_result[0]['data']
 581
 582 class P4RequestSizeException(P4ServerException):
 583     """ One of the maxresults or maxscanrows errors """
 584     def __init__(self, exit_code, p4_result, limit):
 585         super(P4RequestSizeException, self).__init__(exit_code, p4_result)
 586         self.limit = limit
 587
 588 def isModeExecChanged(src_mode, dst_mode):
 589     return isModeExec(src_mode) != isModeExec(dst_mode)
 590
 591 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
 592         errors_as_exceptions=False):
 593
 594     if isinstance(cmd,basestring):
 595         cmd = "-G " + cmd
 596         expand = True
 597     else:
 598         cmd = ["-G"] + cmd
 599         expand = False
 600
 601     cmd = p4_build_cmd(cmd)
 602     if verbose:
 603         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 604
 605     # Use a temporary file to avoid deadlocks without
 606     # subprocess.communicate(), which would put another copy
 607     # of stdout into memory.
 608     stdin_file = None
 609     if stdin is not None:
 610         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 611         if isinstance(stdin,basestring):
 612             stdin_file.write(stdin)
 613         else:
 614             for i in stdin:
 615                 stdin_file.write(i + '\n')
 616         stdin_file.flush()
 617         stdin_file.seek(0)
 618
 619     p4 = subprocess.Popen(cmd,
 620                           shell=expand,
 621                           stdin=stdin_file,
 622                           stdout=subprocess.PIPE)
 623
 624     result = []
 625     try:
 626         while True:
 627             entry = marshal.load(p4.stdout)
 628             if skip_info:
 629                 if 'code' in entry and entry['code'] == 'info':
 630                     continue
 631             if cb is not None:
 632                 cb(entry)
 633             else:
 634                 result.append(entry)
 635     except EOFError:
 636         pass
 637     exitCode = p4.wait()
 638     if exitCode != 0:
 639         if errors_as_exceptions:
 640             if len(result) > 0:
 641                 data = result[0].get('data')
 642                 if data:
 643                     m = re.search('Too many rows scanned \(over (\d+)\)', data)
 644                     if not m:
 645                         m = re.search('Request too large \(over (\d+)\)', data)
 646
 647                     if m:
 648                         limit = int(m.group(1))
 649                         raise P4RequestSizeException(exitCode, result, limit)
 650
 651                 raise P4ServerException(exitCode, result)
 652             else:
 653                 raise P4Exception(exitCode)
 654         else:
 655             entry = {}
 656             entry["p4ExitCode"] = exitCode
 657             result.append(entry)
 658
 659     return result
 660
 661 def p4Cmd(cmd):
 662     list = p4CmdList(cmd)
 663     result = {}
 664     for entry in list:
 665         result.update(entry)
 666     return result;
 667
 668 def p4Where(depotPath):
 669     if not depotPath.endswith("/"):
 670         depotPath += "/"
 671     depotPathLong = depotPath + "..."
 672     outputList = p4CmdList(["where", depotPathLong])
 673     output = None
 674     for entry in outputList:
 675         if "depotFile" in entry:
 676             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
 677             # The base path always ends with "/...".
 678             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
 679                 output = entry
 680                 break
 681         elif "data" in entry:
 682             data = entry.get("data")
 683             space = data.find(" ")
 684             if data[:space] == depotPath:
 685                 output = entry
 686                 break
 687     if output == None:
 688         return ""
 689     if output["code"] == "error":
 690         return ""
 691     clientPath = ""
 692     if "path" in output:
 693         clientPath = output.get("path")
 694     elif "data" in output:
 695         data = output.get("data")
 696         lastSpace = data.rfind(" ")
 697         clientPath = data[lastSpace + 1:]
 698
 699     if clientPath.endswith("..."):
 700         clientPath = clientPath[:-3]
 701     return clientPath
 702
 703 def currentGitBranch():
 704     return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
 705
 706 def isValidGitDir(path):
 707     return git_dir(path) != None
 708
 709 def parseRevision(ref):
 710     return read_pipe("git rev-parse %s" % ref).strip()
 711
 712 def branchExists(ref):
 713     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 714                      ignore_error=True)
 715     return len(rev) > 0
 716
 717 def extractLogMessageFromGitCommit(commit):
 718     logMessage = ""
 719
 720     ## fixme: title is first line of commit, not 1st paragraph.
 721     foundTitle = False
 722     for log in read_pipe_lines("git cat-file commit %s" % commit):
 723        if not foundTitle:
 724            if len(log) == 1:
 725                foundTitle = True
 726            continue
 727
 728        logMessage += log
 729     return logMessage
 730
 731 def extractSettingsGitLog(log):
 732     values = {}
 733     for line in log.split("\n"):
 734         line = line.strip()
 735         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 736         if not m:
 737             continue
 738
 739         assignments = m.group(1).split (':')
 740         for a in assignments:
 741             vals = a.split ('=')
 742             key = vals[0].strip()
 743             val = ('='.join (vals[1:])).strip()
 744             if val.endswith ('\"') and val.startswith('"'):
 745                 val = val[1:-1]
 746
 747             values[key] = val
 748
 749     paths = values.get("depot-paths")
 750     if not paths:
 751         paths = values.get("depot-path")
 752     if paths:
 753         values['depot-paths'] = paths.split(',')
 754     return values
 755
 756 def gitBranchExists(branch):
 757     proc = subprocess.Popen(["git", "rev-parse", branch],
 758                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 759     return proc.wait() == 0;
 760
 761 def gitUpdateRef(ref, newvalue):
 762     subprocess.check_call(["git", "update-ref", ref, newvalue])
 763
 764 def gitDeleteRef(ref):
 765     subprocess.check_call(["git", "update-ref", "-d", ref])
 766
 767 _gitConfig = {}
 768
 769 def gitConfig(key, typeSpecifier=None):
 770     if not _gitConfig.has_key(key):
 771         cmd = [ "git", "config" ]
 772         if typeSpecifier:
 773             cmd += [ typeSpecifier ]
 774         cmd += [ key ]
 775         s = read_pipe(cmd, ignore_error=True)
 776         _gitConfig[key] = s.strip()
 777     return _gitConfig[key]
 778
 779 def gitConfigBool(key):
 780     """Return a bool, using git config --bool.  It is True only if the
 781        variable is set to true, and False if set to false or not present
 782        in the config."""
 783
 784     if not _gitConfig.has_key(key):
 785         _gitConfig[key] = gitConfig(key, '--bool') == "true"
 786     return _gitConfig[key]
 787
 788 def gitConfigInt(key):
 789     if not _gitConfig.has_key(key):
 790         cmd = [ "git", "config", "--int", key ]
 791         s = read_pipe(cmd, ignore_error=True)
 792         v = s.strip()
 793         try:
 794             _gitConfig[key] = int(gitConfig(key, '--int'))
 795         except ValueError:
 796             _gitConfig[key] = None
 797     return _gitConfig[key]
 798
 799 def gitConfigList(key):
 800     if not _gitConfig.has_key(key):
 801         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
 802         _gitConfig[key] = s.strip().splitlines()
 803         if _gitConfig[key] == ['']:
 804             _gitConfig[key] = []
 805     return _gitConfig[key]
 806
 807 def p4BranchesInGit(branchesAreInRemotes=True):
 808     """Find all the branches whose names start with "p4/", looking
 809        in remotes or heads as specified by the argument.  Return
 810        a dictionary of { branch: revision } for each one found.
 811        The branch names are the short names, without any
 812        "p4/" prefix."""
 813
 814     branches = {}
 815
 816     cmdline = "git rev-parse --symbolic "
 817     if branchesAreInRemotes:
 818         cmdline += "--remotes"
 819     else:
 820         cmdline += "--branches"
 821
 822     for line in read_pipe_lines(cmdline):
 823         line = line.strip()
 824
 825         # only import to p4/
 826         if not line.startswith('p4/'):
 827             continue
 828         # special symbolic ref to p4/master
 829         if line == "p4/HEAD":
 830             continue
 831
 832         # strip off p4/ prefix
 833         branch = line[len("p4/"):]
 834
 835         branches[branch] = parseRevision(line)
 836
 837     return branches
 838
 839 def branch_exists(branch):
 840     """Make sure that the given ref name really exists."""
 841
 842     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
 843     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 844     out, _ = p.communicate()
 845     if p.returncode:
 846         return False
 847     # expect exactly one line of output: the branch name
 848     return out.rstrip() == branch
 849
 850 def findUpstreamBranchPoint(head = "HEAD"):
 851     branches = p4BranchesInGit()
 852     # map from depot-path to branch name
 853     branchByDepotPath = {}
 854     for branch in branches.keys():
 855         tip = branches[branch]
 856         log = extractLogMessageFromGitCommit(tip)
 857         settings = extractSettingsGitLog(log)
 858         if settings.has_key("depot-paths"):
 859             paths = ",".join(settings["depot-paths"])
 860             branchByDepotPath[paths] = "remotes/p4/" + branch
 861
 862     settings = None
 863     parent = 0
 864     while parent < 65535:
 865         commit = head + "~%s" % parent
 866         log = extractLogMessageFromGitCommit(commit)
 867         settings = extractSettingsGitLog(log)
 868         if settings.has_key("depot-paths"):
 869             paths = ",".join(settings["depot-paths"])
 870             if branchByDepotPath.has_key(paths):
 871                 return [branchByDepotPath[paths], settings]
 872
 873         parent = parent + 1
 874
 875     return ["", settings]
 876
 877 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 878     if not silent:
 879         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 880                % localRefPrefix)
 881
 882     originPrefix = "origin/p4/"
 883
 884     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 885         line = line.strip()
 886         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 887             continue
 888
 889         headName = line[len(originPrefix):]
 890         remoteHead = localRefPrefix + headName
 891         originHead = line
 892
 893         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 894         if (not original.has_key('depot-paths')
 895             or not original.has_key('change')):
 896             continue
 897
 898         update = False
 899         if not gitBranchExists(remoteHead):
 900             if verbose:
 901                 print "creating %s" % remoteHead
 902             update = True
 903         else:
 904             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 905             if settings.has_key('change') > 0:
 906                 if settings['depot-paths'] == original['depot-paths']:
 907                     originP4Change = int(original['change'])
 908                     p4Change = int(settings['change'])
 909                     if originP4Change > p4Change:
 910                         print ("%s (%s) is newer than %s (%s). "
 911                                "Updating p4 branch from origin."
 912                                % (originHead, originP4Change,
 913                                   remoteHead, p4Change))
 914                         update = True
 915                 else:
 916                     print ("Ignoring: %s was imported from %s while "
 917                            "%s was imported from %s"
 918                            % (originHead, ','.join(original['depot-paths']),
 919                               remoteHead, ','.join(settings['depot-paths'])))
 920
 921         if update:
 922             system("git update-ref %s %s" % (remoteHead, originHead))
 923
 924 def originP4BranchesExist():
 925         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 926
 927
 928 def p4ParseNumericChangeRange(parts):
 929     changeStart = int(parts[0][1:])
 930     if parts[1] == '#head':
 931         changeEnd = p4_last_change()
 932     else:
 933         changeEnd = int(parts[1])
 934
 935     return (changeStart, changeEnd)
 936
 937 def chooseBlockSize(blockSize):
 938     if blockSize:
 939         return blockSize
 940     else:
 941         return defaultBlockSize
 942
 943 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
 944     assert depotPaths
 945
 946     # Parse the change range into start and end. Try to find integer
 947     # revision ranges as these can be broken up into blocks to avoid
 948     # hitting server-side limits (maxrows, maxscanresults). But if
 949     # that doesn't work, fall back to using the raw revision specifier
 950     # strings, without using block mode.
 951
 952     if changeRange is None or changeRange == '':
 953         changeStart = 1
 954         changeEnd = p4_last_change()
 955         block_size = chooseBlockSize(requestedBlockSize)
 956     else:
 957         parts = changeRange.split(',')
 958         assert len(parts) == 2
 959         try:
 960             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
 961             block_size = chooseBlockSize(requestedBlockSize)
 962         except ValueError:
 963             changeStart = parts[0][1:]
 964             changeEnd = parts[1]
 965             if requestedBlockSize:
 966                 die("cannot use --changes-block-size with non-numeric revisions")
 967             block_size = None
 968
 969     changes = set()
 970
 971     # Retrieve changes a block at a time, to prevent running
 972     # into a MaxResults/MaxScanRows error from the server. If
 973     # we _do_ hit one of those errors, turn down the block size
 974
 975     while True:
 976         cmd = ['changes']
 977
 978         if block_size:
 979             end = min(changeEnd, changeStart + block_size)
 980             revisionRange = "%d,%d" % (changeStart, end)
 981         else:
 982             revisionRange = "%s,%s" % (changeStart, changeEnd)
 983
 984         for p in depotPaths:
 985             cmd += ["%s...@%s" % (p, revisionRange)]
 986
 987         # fetch the changes
 988         try:
 989             result = p4CmdList(cmd, errors_as_exceptions=True)
 990         except P4RequestSizeException as e:
 991             if not block_size:
 992                 block_size = e.limit
 993             elif block_size > e.limit:
 994                 block_size = e.limit
 995             else:
 996                 block_size = max(2, block_size // 2)
 997
 998             if verbose: print("block size error, retrying with block size {0}".format(block_size))
 999             continue
1000         except P4Exception as e:
1001             die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1002
1003         # Insert changes in chronological order
1004         for entry in reversed(result):
1005             if not entry.has_key('change'):
1006                 continue
1007             changes.add(int(entry['change']))
1008
1009         if not block_size:
1010             break
1011
1012         if end >= changeEnd:
1013             break
1014
1015         changeStart = end + 1
1016
1017     changes = sorted(changes)
1018     return changes
1019
1020 def p4PathStartsWith(path, prefix):
1021     # This method tries to remedy a potential mixed-case issue:
1022     #
1023     # If UserA adds  //depot/DirA/file1
1024     # and UserB adds //depot/dira/file2
1025     #
1026     # we may or may not have a problem. If you have core.ignorecase=true,
1027     # we treat DirA and dira as the same directory
1028     if gitConfigBool("core.ignorecase"):
1029         return path.lower().startswith(prefix.lower())
1030     return path.startswith(prefix)
1031
1032 def getClientSpec():
1033     """Look at the p4 client spec, create a View() object that contains
1034        all the mappings, and return it."""
1035
1036     specList = p4CmdList("client -o")
1037     if len(specList) != 1:
1038         die('Output from "client -o" is %d lines, expecting 1' %
1039             len(specList))
1040
1041     # dictionary of all client parameters
1042     entry = specList[0]
1043
1044     # the //client/ name
1045     client_name = entry["Client"]
1046
1047     # just the keys that start with "View"
1048     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1049
1050     # hold this new View
1051     view = View(client_name)
1052
1053     # append the lines, in order, to the view
1054     for view_num in range(len(view_keys)):
1055         k = "View%d" % view_num
1056         if k not in view_keys:
1057             die("Expected view key %s missing" % k)
1058         view.append(entry[k])
1059
1060     return view
1061
1062 def getClientRoot():
1063     """Grab the client directory."""
1064
1065     output = p4CmdList("client -o")
1066     if len(output) != 1:
1067         die('Output from "client -o" is %d lines, expecting 1' % len(output))
1068
1069     entry = output[0]
1070     if "Root" not in entry:
1071         die('Client has no "Root"')
1072
1073     return entry["Root"]
1074
1075 #
1076 # P4 wildcards are not allowed in filenames.  P4 complains
1077 # if you simply add them, but you can force it with "-f", in
1078 # which case it translates them into %xx encoding internally.
1079 #
1080 def wildcard_decode(path):
1081     # Search for and fix just these four characters.  Do % last so
1082     # that fixing it does not inadvertently create new %-escapes.
1083     # Cannot have * in a filename in windows; untested as to
1084     # what p4 would do in such a case.
1085     if not platform.system() == "Windows":
1086         path = path.replace("%2A", "*")
1087     path = path.replace("%23", "#") \
1088                .replace("%40", "@") \
1089                .replace("%25", "%")
1090     return path
1091
1092 def wildcard_encode(path):
1093     # do % first to avoid double-encoding the %s introduced here
1094     path = path.replace("%", "%25") \
1095                .replace("*", "%2A") \
1096                .replace("#", "%23") \
1097                .replace("@", "%40")
1098     return path
1099
1100 def wildcard_present(path):
1101     m = re.search("[*#@%]", path)
1102     return m is not None
1103
1104 class LargeFileSystem(object):
1105     """Base class for large file system support."""
1106
1107     def __init__(self, writeToGitStream):
1108         self.largeFiles = set()
1109         self.writeToGitStream = writeToGitStream
1110
1111     def generatePointer(self, cloneDestination, contentFile):
1112         """Return the content of a pointer file that is stored in Git instead of
1113            the actual content."""
1114         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1115
1116     def pushFile(self, localLargeFile):
1117         """Push the actual content which is not stored in the Git repository to
1118            a server."""
1119         assert False, "Method 'pushFile' required in " + self.__class__.__name__
1120
1121     def hasLargeFileExtension(self, relPath):
1122         return reduce(
1123             lambda a, b: a or b,
1124             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1125             False
1126         )
1127
1128     def generateTempFile(self, contents):
1129         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1130         for d in contents:
1131             contentFile.write(d)
1132         contentFile.close()
1133         return contentFile.name
1134
1135     def exceedsLargeFileThreshold(self, relPath, contents):
1136         if gitConfigInt('git-p4.largeFileThreshold'):
1137             contentsSize = sum(len(d) for d in contents)
1138             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1139                 return True
1140         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1141             contentsSize = sum(len(d) for d in contents)
1142             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1143                 return False
1144             contentTempFile = self.generateTempFile(contents)
1145             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1146             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1147             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1148             zf.close()
1149             compressedContentsSize = zf.infolist()[0].compress_size
1150             os.remove(contentTempFile)
1151             os.remove(compressedContentFile.name)
1152             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1153                 return True
1154         return False
1155
1156     def addLargeFile(self, relPath):
1157         self.largeFiles.add(relPath)
1158
1159     def removeLargeFile(self, relPath):
1160         self.largeFiles.remove(relPath)
1161
1162     def isLargeFile(self, relPath):
1163         return relPath in self.largeFiles
1164
1165     def processContent(self, git_mode, relPath, contents):
1166         """Processes the content of git fast import. This method decides if a
1167            file is stored in the large file system and handles all necessary
1168            steps."""
1169         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1170             contentTempFile = self.generateTempFile(contents)
1171             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1172             if pointer_git_mode:
1173                 git_mode = pointer_git_mode
1174             if localLargeFile:
1175                 # Move temp file to final location in large file system
1176                 largeFileDir = os.path.dirname(localLargeFile)
1177                 if not os.path.isdir(largeFileDir):
1178                     os.makedirs(largeFileDir)
1179                 shutil.move(contentTempFile, localLargeFile)
1180                 self.addLargeFile(relPath)
1181                 if gitConfigBool('git-p4.largeFilePush'):
1182                     self.pushFile(localLargeFile)
1183                 if verbose:
1184                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1185         return (git_mode, contents)
1186
1187 class MockLFS(LargeFileSystem):
1188     """Mock large file system for testing."""
1189
1190     def generatePointer(self, contentFile):
1191         """The pointer content is the original content prefixed with "pointer-".
1192            The local filename of the large file storage is derived from the file content.
1193            """
1194         with open(contentFile, 'r') as f:
1195             content = next(f)
1196             gitMode = '100644'
1197             pointerContents = 'pointer-' + content
1198             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1199             return (gitMode, pointerContents, localLargeFile)
1200
1201     def pushFile(self, localLargeFile):
1202         """The remote filename of the large file storage is the same as the local
1203            one but in a different directory.
1204            """
1205         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1206         if not os.path.exists(remotePath):
1207             os.makedirs(remotePath)
1208         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1209
1210 class GitLFS(LargeFileSystem):
1211     """Git LFS as backend for the git-p4 large file system.
1212        See https://git-lfs.github.com/ for details."""
1213
1214     def __init__(self, *args):
1215         LargeFileSystem.__init__(self, *args)
1216         self.baseGitAttributes = []
1217
1218     def generatePointer(self, contentFile):
1219         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1220            mode and content which is stored in the Git repository instead of
1221            the actual content. Return also the new location of the actual
1222            content.
1223            """
1224         if os.path.getsize(contentFile) == 0:
1225             return (None, '', None)
1226
1227         pointerProcess = subprocess.Popen(
1228             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1229             stdout=subprocess.PIPE
1230         )
1231         pointerFile = pointerProcess.stdout.read()
1232         if pointerProcess.wait():
1233             os.remove(contentFile)
1234             die('git-lfs pointer command failed. Did you install the extension?')
1235
1236         # Git LFS removed the preamble in the output of the 'pointer' command
1237         # starting from version 1.2.0. Check for the preamble here to support
1238         # earlier versions.
1239         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1240         if pointerFile.startswith('Git LFS pointer for'):
1241             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1242
1243         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1244         localLargeFile = os.path.join(
1245             os.getcwd(),
1246             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1247             oid,
1248         )
1249         # LFS Spec states that pointer files should not have the executable bit set.
1250         gitMode = '100644'
1251         return (gitMode, pointerFile, localLargeFile)
1252
1253     def pushFile(self, localLargeFile):
1254         uploadProcess = subprocess.Popen(
1255             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1256         )
1257         if uploadProcess.wait():
1258             die('git-lfs push command failed. Did you define a remote?')
1259
1260     def generateGitAttributes(self):
1261         return (
1262             self.baseGitAttributes +
1263             [
1264                 '\n',
1265                 '#\n',
1266                 '# Git LFS (see https://git-lfs.github.com/)\n',
1267                 '#\n',
1268             ] +
1269             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1270                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1271             ] +
1272             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1273                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1274             ]
1275         )
1276
1277     def addLargeFile(self, relPath):
1278         LargeFileSystem.addLargeFile(self, relPath)
1279         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1280
1281     def removeLargeFile(self, relPath):
1282         LargeFileSystem.removeLargeFile(self, relPath)
1283         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1284
1285     def processContent(self, git_mode, relPath, contents):
1286         if relPath == '.gitattributes':
1287             self.baseGitAttributes = contents
1288             return (git_mode, self.generateGitAttributes())
1289         else:
1290             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1291
1292 class Command:
1293     def __init__(self):
1294         self.usage = "usage: %prog [options]"
1295         self.needsGit = True
1296         self.verbose = False
1297
1298     # This is required for the "append" cloneExclude action
1299     def ensure_value(self, attr, value):
1300         if not hasattr(self, attr) or getattr(self, attr) is None:
1301             setattr(self, attr, value)
1302         return getattr(self, attr)
1303
1304 class P4UserMap:
1305     def __init__(self):
1306         self.userMapFromPerforceServer = False
1307         self.myP4UserId = None
1308
1309     def p4UserId(self):
1310         if self.myP4UserId:
1311             return self.myP4UserId
1312
1313         results = p4CmdList("user -o")
1314         for r in results:
1315             if r.has_key('User'):
1316                 self.myP4UserId = r['User']
1317                 return r['User']
1318         die("Could not find your p4 user id")
1319
1320     def p4UserIsMe(self, p4User):
1321         # return True if the given p4 user is actually me
1322         me = self.p4UserId()
1323         if not p4User or p4User != me:
1324             return False
1325         else:
1326             return True
1327
1328     def getUserCacheFilename(self):
1329         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1330         return home + "/.gitp4-usercache.txt"
1331
1332     def getUserMapFromPerforceServer(self):
1333         if self.userMapFromPerforceServer:
1334             return
1335         self.users = {}
1336         self.emails = {}
1337
1338         for output in p4CmdList("users"):
1339             if not output.has_key("User"):
1340                 continue
1341             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1342             self.emails[output["Email"]] = output["User"]
1343
1344         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1345         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1346             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1347             if mapUser and len(mapUser[0]) == 3:
1348                 user = mapUser[0][0]
1349                 fullname = mapUser[0][1]
1350                 email = mapUser[0][2]
1351                 self.users[user] = fullname + " <" + email + ">"
1352                 self.emails[email] = user
1353
1354         s = ''
1355         for (key, val) in self.users.items():
1356             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1357
1358         open(self.getUserCacheFilename(), "wb").write(s)
1359         self.userMapFromPerforceServer = True
1360
1361     def loadUserMapFromCache(self):
1362         self.users = {}
1363         self.userMapFromPerforceServer = False
1364         try:
1365             cache = open(self.getUserCacheFilename(), "rb")
1366             lines = cache.readlines()
1367             cache.close()
1368             for line in lines:
1369                 entry = line.strip().split("\t")
1370                 self.users[entry[0]] = entry[1]
1371         except IOError:
1372             self.getUserMapFromPerforceServer()
1373
1374 class P4Debug(Command):
1375     def __init__(self):
1376         Command.__init__(self)
1377         self.options = []
1378         self.description = "A tool to debug the output of p4 -G."
1379         self.needsGit = False
1380
1381     def run(self, args):
1382         j = 0
1383         for output in p4CmdList(args):
1384             print 'Element: %d' % j
1385             j += 1
1386             print output
1387         return True
1388
1389 class P4RollBack(Command):
1390     def __init__(self):
1391         Command.__init__(self)
1392         self.options = [
1393             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1394         ]
1395         self.description = "A tool to debug the multi-branch import. Don't use :)"
1396         self.rollbackLocalBranches = False
1397
1398     def run(self, args):
1399         if len(args) != 1:
1400             return False
1401         maxChange = int(args[0])
1402
1403         if "p4ExitCode" in p4Cmd("changes -m 1"):
1404             die("Problems executing p4");
1405
1406         if self.rollbackLocalBranches:
1407             refPrefix = "refs/heads/"
1408             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1409         else:
1410             refPrefix = "refs/remotes/"
1411             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1412
1413         for line in lines:
1414             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1415                 line = line.strip()
1416                 ref = refPrefix + line
1417                 log = extractLogMessageFromGitCommit(ref)
1418                 settings = extractSettingsGitLog(log)
1419
1420                 depotPaths = settings['depot-paths']
1421                 change = settings['change']
1422
1423                 changed = False
1424
1425                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1426                                                            for p in depotPaths]))) == 0:
1427                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1428                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1429                     continue
1430
1431                 while change and int(change) > maxChange:
1432                     changed = True
1433                     if self.verbose:
1434                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1435                     system("git update-ref %s \"%s^\"" % (ref, ref))
1436                     log = extractLogMessageFromGitCommit(ref)
1437                     settings =  extractSettingsGitLog(log)
1438
1439
1440                     depotPaths = settings['depot-paths']
1441                     change = settings['change']
1442
1443                 if changed:
1444                     print "%s rewound to %s" % (ref, change)
1445
1446         return True
1447
1448 class P4Submit(Command, P4UserMap):
1449
1450     conflict_behavior_choices = ("ask", "skip", "quit")
1451
1452     def __init__(self):
1453         Command.__init__(self)
1454         P4UserMap.__init__(self)
1455         self.options = [
1456                 optparse.make_option("--origin", dest="origin"),
1457                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1458                 # preserve the user, requires relevant p4 permissions
1459                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1460                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1461                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1462                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1463                 optparse.make_option("--conflict", dest="conflict_behavior",
1464                                      choices=self.conflict_behavior_choices),
1465                 optparse.make_option("--branch", dest="branch"),
1466                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1467                                      help="Shelve instead of submit. Shelved files are reverted, "
1468                                      "restoring the workspace to the state before the shelve"),
1469                 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1470                                      metavar="CHANGELIST",
1471                                      help="update an existing shelved changelist, implies --shelve, "
1472                                            "repeat in-order for multiple shelved changelists"),
1473                 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1474                                      help="submit only the specified commit(s), one commit or xxx..xxx"),
1475                 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1476                                      help="Disable rebase after submit is completed. Can be useful if you "
1477                                      "work from a local git branch that is not master"),
1478                 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1479                                      help="Skip Perforce sync of p4/master after submit or shelve"),
1480         ]
1481         self.description = "Submit changes from git to the perforce depot."
1482         self.usage += " [name of git branch to submit into perforce depot]"
1483         self.origin = ""
1484         self.detectRenames = False
1485         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1486         self.dry_run = False
1487         self.shelve = False
1488         self.update_shelve = list()
1489         self.commit = ""
1490         self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1491         self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1492         self.prepare_p4_only = False
1493         self.conflict_behavior = None
1494         self.isWindows = (platform.system() == "Windows")
1495         self.exportLabels = False
1496         self.p4HasMoveCommand = p4_has_move_command()
1497         self.branch = None
1498
1499         if gitConfig('git-p4.largeFileSystem'):
1500             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1501
1502     def check(self):
1503         if len(p4CmdList("opened ...")) > 0:
1504             die("You have files opened with perforce! Close them before starting the sync.")
1505
1506     def separate_jobs_from_description(self, message):
1507         """Extract and return a possible Jobs field in the commit
1508            message.  It goes into a separate section in the p4 change
1509            specification.
1510
1511            A jobs line starts with "Jobs:" and looks like a new field
1512            in a form.  Values are white-space separated on the same
1513            line or on following lines that start with a tab.
1514
1515            This does not parse and extract the full git commit message
1516            like a p4 form.  It just sees the Jobs: line as a marker
1517            to pass everything from then on directly into the p4 form,
1518            but outside the description section.
1519
1520            Return a tuple (stripped log message, jobs string)."""
1521
1522         m = re.search(r'^Jobs:', message, re.MULTILINE)
1523         if m is None:
1524             return (message, None)
1525
1526         jobtext = message[m.start():]
1527         stripped_message = message[:m.start()].rstrip()
1528         return (stripped_message, jobtext)
1529
1530     def prepareLogMessage(self, template, message, jobs):
1531         """Edits the template returned from "p4 change -o" to insert
1532            the message in the Description field, and the jobs text in
1533            the Jobs field."""
1534         result = ""
1535
1536         inDescriptionSection = False
1537
1538         for line in template.split("\n"):
1539             if line.startswith("#"):
1540                 result += line + "\n"
1541                 continue
1542
1543             if inDescriptionSection:
1544                 if line.startswith("Files:") or line.startswith("Jobs:"):
1545                     inDescriptionSection = False
1546                     # insert Jobs section
1547                     if jobs:
1548                         result += jobs + "\n"
1549                 else:
1550                     continue
1551             else:
1552                 if line.startswith("Description:"):
1553                     inDescriptionSection = True
1554                     line += "\n"
1555                     for messageLine in message.split("\n"):
1556                         line += "\t" + messageLine + "\n"
1557
1558             result += line + "\n"
1559
1560         return result
1561
1562     def patchRCSKeywords(self, file, pattern):
1563         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1564         (handle, outFileName) = tempfile.mkstemp(dir='.')
1565         try:
1566             outFile = os.fdopen(handle, "w+")
1567             inFile = open(file, "r")
1568             regexp = re.compile(pattern, re.VERBOSE)
1569             for line in inFile.readlines():
1570                 line = regexp.sub(r'$\1$', line)
1571                 outFile.write(line)
1572             inFile.close()
1573             outFile.close()
1574             # Forcibly overwrite the original file
1575             os.unlink(file)
1576             shutil.move(outFileName, file)
1577         except:
1578             # cleanup our temporary file
1579             os.unlink(outFileName)
1580             print "Failed to strip RCS keywords in %s" % file
1581             raise
1582
1583         print "Patched up RCS keywords in %s" % file
1584
1585     def p4UserForCommit(self,id):
1586         # Return the tuple (perforce user,git email) for a given git commit id
1587         self.getUserMapFromPerforceServer()
1588         gitEmail = read_pipe(["git", "log", "--max-count=1",
1589                               "--format=%ae", id])
1590         gitEmail = gitEmail.strip()
1591         if not self.emails.has_key(gitEmail):
1592             return (None,gitEmail)
1593         else:
1594             return (self.emails[gitEmail],gitEmail)
1595
1596     def checkValidP4Users(self,commits):
1597         # check if any git authors cannot be mapped to p4 users
1598         for id in commits:
1599             (user,email) = self.p4UserForCommit(id)
1600             if not user:
1601                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1602                 if gitConfigBool("git-p4.allowMissingP4Users"):
1603                     print "%s" % msg
1604                 else:
1605                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1606
1607     def lastP4Changelist(self):
1608         # Get back the last changelist number submitted in this client spec. This
1609         # then gets used to patch up the username in the change. If the same
1610         # client spec is being used by multiple processes then this might go
1611         # wrong.
1612         results = p4CmdList("client -o")        # find the current client
1613         client = None
1614         for r in results:
1615             if r.has_key('Client'):
1616                 client = r['Client']
1617                 break
1618         if not client:
1619             die("could not get client spec")
1620         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1621         for r in results:
1622             if r.has_key('change'):
1623                 return r['change']
1624         die("Could not get changelist number for last submit - cannot patch up user details")
1625
1626     def modifyChangelistUser(self, changelist, newUser):
1627         # fixup the user field of a changelist after it has been submitted.
1628         changes = p4CmdList("change -o %s" % changelist)
1629         if len(changes) != 1:
1630             die("Bad output from p4 change modifying %s to user %s" %
1631                 (changelist, newUser))
1632
1633         c = changes[0]
1634         if c['User'] == newUser: return   # nothing to do
1635         c['User'] = newUser
1636         input = marshal.dumps(c)
1637
1638         result = p4CmdList("change -f -i", stdin=input)
1639         for r in result:
1640             if r.has_key('code'):
1641                 if r['code'] == 'error':
1642                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1643             if r.has_key('data'):
1644                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1645                 return
1646         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1647
1648     def canChangeChangelists(self):
1649         # check to see if we have p4 admin or super-user permissions, either of
1650         # which are required to modify changelists.
1651         results = p4CmdList(["protects", self.depotPath])
1652         for r in results:
1653             if r.has_key('perm'):
1654                 if r['perm'] == 'admin':
1655                     return 1
1656                 if r['perm'] == 'super':
1657                     return 1
1658         return 0
1659
1660     def prepareSubmitTemplate(self, changelist=None):
1661         """Run "p4 change -o" to grab a change specification template.
1662            This does not use "p4 -G", as it is nice to keep the submission
1663            template in original order, since a human might edit it.
1664
1665            Remove lines in the Files section that show changes to files
1666            outside the depot path we're committing into."""
1667
1668         [upstream, settings] = findUpstreamBranchPoint()
1669
1670         template = """\
1671 # A Perforce Change Specification.
1672 #
1673 #  Change:      The change number. 'new' on a new changelist.
1674 #  Date:        The date this specification was last modified.
1675 #  Client:      The client on which the changelist was created.  Read-only.
1676 #  User:        The user who created the changelist.
1677 #  Status:      Either 'pending' or 'submitted'. Read-only.
1678 #  Type:        Either 'public' or 'restricted'. Default is 'public'.
1679 #  Description: Comments about the changelist.  Required.
1680 #  Jobs:        What opened jobs are to be closed by this changelist.
1681 #               You may delete jobs from this list.  (New changelists only.)
1682 #  Files:       What opened files from the default changelist are to be added
1683 #               to this changelist.  You may delete files from this list.
1684 #               (New changelists only.)
1685 """
1686         files_list = []
1687         inFilesSection = False
1688         change_entry = None
1689         args = ['change', '-o']
1690         if changelist:
1691             args.append(str(changelist))
1692         for entry in p4CmdList(args):
1693             if not entry.has_key('code'):
1694                 continue
1695             if entry['code'] == 'stat':
1696                 change_entry = entry
1697                 break
1698         if not change_entry:
1699             die('Failed to decode output of p4 change -o')
1700         for key, value in change_entry.iteritems():
1701             if key.startswith('File'):
1702                 if settings.has_key('depot-paths'):
1703                     if not [p for p in settings['depot-paths']
1704                             if p4PathStartsWith(value, p)]:
1705                         continue
1706                 else:
1707                     if not p4PathStartsWith(value, self.depotPath):
1708                         continue
1709                 files_list.append(value)
1710                 continue
1711         # Output in the order expected by prepareLogMessage
1712         for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1713             if not change_entry.has_key(key):
1714                 continue
1715             template += '\n'
1716             template += key + ':'
1717             if key == 'Description':
1718                 template += '\n'
1719             for field_line in change_entry[key].splitlines():
1720                 template += '\t'+field_line+'\n'
1721         if len(files_list) > 0:
1722             template += '\n'
1723             template += 'Files:\n'
1724         for path in files_list:
1725             template += '\t'+path+'\n'
1726         return template
1727
1728     def edit_template(self, template_file):
1729         """Invoke the editor to let the user change the submission
1730            message.  Return true if okay to continue with the submit."""
1731
1732         # if configured to skip the editing part, just submit
1733         if gitConfigBool("git-p4.skipSubmitEdit"):
1734             return True
1735
1736         # look at the modification time, to check later if the user saved
1737         # the file
1738         mtime = os.stat(template_file).st_mtime
1739
1740         # invoke the editor
1741         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1742             editor = os.environ.get("P4EDITOR")
1743         else:
1744             editor = read_pipe("git var GIT_EDITOR").strip()
1745         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1746
1747         # If the file was not saved, prompt to see if this patch should
1748         # be skipped.  But skip this verification step if configured so.
1749         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1750             return True
1751
1752         # modification time updated means user saved the file
1753         if os.stat(template_file).st_mtime > mtime:
1754             return True
1755
1756         while True:
1757             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1758             if response == 'y':
1759                 return True
1760             if response == 'n':
1761                 return False
1762
1763     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1764         # diff
1765         if os.environ.has_key("P4DIFF"):
1766             del(os.environ["P4DIFF"])
1767         diff = ""
1768         for editedFile in editedFiles:
1769             diff += p4_read_pipe(['diff', '-du',
1770                                   wildcard_encode(editedFile)])
1771
1772         # new file diff
1773         newdiff = ""
1774         for newFile in filesToAdd:
1775             newdiff += "==== new file ====\n"
1776             newdiff += "--- /dev/null\n"
1777             newdiff += "+++ %s\n" % newFile
1778
1779             is_link = os.path.islink(newFile)
1780             expect_link = newFile in symlinks
1781
1782             if is_link and expect_link:
1783                 newdiff += "+%s\n" % os.readlink(newFile)
1784             else:
1785                 f = open(newFile, "r")
1786                 for line in f.readlines():
1787                     newdiff += "+" + line
1788                 f.close()
1789
1790         return (diff + newdiff).replace('\r\n', '\n')
1791
1792     def applyCommit(self, id):
1793         """Apply one commit, return True if it succeeded."""
1794
1795         print "Applying", read_pipe(["git", "show", "-s",
1796                                      "--format=format:%h %s", id])
1797
1798         (p4User, gitEmail) = self.p4UserForCommit(id)
1799
1800         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1801         filesToAdd = set()
1802         filesToChangeType = set()
1803         filesToDelete = set()
1804         editedFiles = set()
1805         pureRenameCopy = set()
1806         symlinks = set()
1807         filesToChangeExecBit = {}
1808         all_files = list()
1809
1810         for line in diff:
1811             diff = parseDiffTreeEntry(line)
1812             modifier = diff['status']
1813             path = diff['src']
1814             all_files.append(path)
1815
1816             if modifier == "M":
1817                 p4_edit(path)
1818                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1819                     filesToChangeExecBit[path] = diff['dst_mode']
1820                 editedFiles.add(path)
1821             elif modifier == "A":
1822                 filesToAdd.add(path)
1823                 filesToChangeExecBit[path] = diff['dst_mode']
1824                 if path in filesToDelete:
1825                     filesToDelete.remove(path)
1826
1827                 dst_mode = int(diff['dst_mode'], 8)
1828                 if dst_mode == 0120000:
1829                     symlinks.add(path)
1830
1831             elif modifier == "D":
1832                 filesToDelete.add(path)
1833                 if path in filesToAdd:
1834                     filesToAdd.remove(path)
1835             elif modifier == "C":
1836                 src, dest = diff['src'], diff['dst']
1837                 p4_integrate(src, dest)
1838                 pureRenameCopy.add(dest)
1839                 if diff['src_sha1'] != diff['dst_sha1']:
1840                     p4_edit(dest)
1841                     pureRenameCopy.discard(dest)
1842                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1843                     p4_edit(dest)
1844                     pureRenameCopy.discard(dest)
1845                     filesToChangeExecBit[dest] = diff['dst_mode']
1846                 if self.isWindows:
1847                     # turn off read-only attribute
1848                     os.chmod(dest, stat.S_IWRITE)
1849                 os.unlink(dest)
1850                 editedFiles.add(dest)
1851             elif modifier == "R":
1852                 src, dest = diff['src'], diff['dst']
1853                 if self.p4HasMoveCommand:
1854                     p4_edit(src)        # src must be open before move
1855                     p4_move(src, dest)  # opens for (move/delete, move/add)
1856                 else:
1857                     p4_integrate(src, dest)
1858                     if diff['src_sha1'] != diff['dst_sha1']:
1859                         p4_edit(dest)
1860                     else:
1861                         pureRenameCopy.add(dest)
1862                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1863                     if not self.p4HasMoveCommand:
1864                         p4_edit(dest)   # with move: already open, writable
1865                     filesToChangeExecBit[dest] = diff['dst_mode']
1866                 if not self.p4HasMoveCommand:
1867                     if self.isWindows:
1868                         os.chmod(dest, stat.S_IWRITE)
1869                     os.unlink(dest)
1870                     filesToDelete.add(src)
1871                 editedFiles.add(dest)
1872             elif modifier == "T":
1873                 filesToChangeType.add(path)
1874             else:
1875                 die("unknown modifier %s for %s" % (modifier, path))
1876
1877         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1878         patchcmd = diffcmd + " | git apply "
1879         tryPatchCmd = patchcmd + "--check -"
1880         applyPatchCmd = patchcmd + "--check --apply -"
1881         patch_succeeded = True
1882
1883         if os.system(tryPatchCmd) != 0:
1884             fixed_rcs_keywords = False
1885             patch_succeeded = False
1886             print "Unfortunately applying the change failed!"
1887
1888             # Patch failed, maybe it's just RCS keyword woes. Look through
1889             # the patch to see if that's possible.
1890             if gitConfigBool("git-p4.attemptRCSCleanup"):
1891                 file = None
1892                 pattern = None
1893                 kwfiles = {}
1894                 for file in editedFiles | filesToDelete:
1895                     # did this file's delta contain RCS keywords?
1896                     pattern = p4_keywords_regexp_for_file(file)
1897
1898                     if pattern:
1899                         # this file is a possibility...look for RCS keywords.
1900                         regexp = re.compile(pattern, re.VERBOSE)
1901                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1902                             if regexp.search(line):
1903                                 if verbose:
1904                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1905                                 kwfiles[file] = pattern
1906                                 break
1907
1908                 for file in kwfiles:
1909                     if verbose:
1910                         print "zapping %s with %s" % (line,pattern)
1911                     # File is being deleted, so not open in p4.  Must
1912                     # disable the read-only bit on windows.
1913                     if self.isWindows and file not in editedFiles:
1914                         os.chmod(file, stat.S_IWRITE)
1915                     self.patchRCSKeywords(file, kwfiles[file])
1916                     fixed_rcs_keywords = True
1917
1918             if fixed_rcs_keywords:
1919                 print "Retrying the patch with RCS keywords cleaned up"
1920                 if os.system(tryPatchCmd) == 0:
1921                     patch_succeeded = True
1922
1923         if not patch_succeeded:
1924             for f in editedFiles:
1925                 p4_revert(f)
1926             return False
1927
1928         #
1929         # Apply the patch for real, and do add/delete/+x handling.
1930         #
1931         system(applyPatchCmd)
1932
1933         for f in filesToChangeType:
1934             p4_edit(f, "-t", "auto")
1935         for f in filesToAdd:
1936             p4_add(f)
1937         for f in filesToDelete:
1938             p4_revert(f)
1939             p4_delete(f)
1940
1941         # Set/clear executable bits
1942         for f in filesToChangeExecBit.keys():
1943             mode = filesToChangeExecBit[f]
1944             setP4ExecBit(f, mode)
1945
1946         update_shelve = 0
1947         if len(self.update_shelve) > 0:
1948             update_shelve = self.update_shelve.pop(0)
1949             p4_reopen_in_change(update_shelve, all_files)
1950
1951         #
1952         # Build p4 change description, starting with the contents
1953         # of the git commit message.
1954         #
1955         logMessage = extractLogMessageFromGitCommit(id)
1956         logMessage = logMessage.strip()
1957         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1958
1959         template = self.prepareSubmitTemplate(update_shelve)
1960         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1961
1962         if self.preserveUser:
1963            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1964
1965         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1966             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1967             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1968             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1969
1970         separatorLine = "######## everything below this line is just the diff #######\n"
1971         if not self.prepare_p4_only:
1972             submitTemplate += separatorLine
1973             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1974
1975         (handle, fileName) = tempfile.mkstemp()
1976         tmpFile = os.fdopen(handle, "w+b")
1977         if self.isWindows:
1978             submitTemplate = submitTemplate.replace("\n", "\r\n")
1979         tmpFile.write(submitTemplate)
1980         tmpFile.close()
1981
1982         if self.prepare_p4_only:
1983             #
1984             # Leave the p4 tree prepared, and the submit template around
1985             # and let the user decide what to do next
1986             #
1987             print
1988             print "P4 workspace prepared for submission."
1989             print "To submit or revert, go to client workspace"
1990             print "  " + self.clientPath
1991             print
1992             print "To submit, use \"p4 submit\" to write a new description,"
1993             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1994                   " \"git p4\"." % fileName
1995             print "You can delete the file \"%s\" when finished." % fileName
1996
1997             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1998                 print "To preserve change ownership by user %s, you must\n" \
1999                       "do \"p4 change -f <change>\" after submitting and\n" \
2000                       "edit the User field."
2001             if pureRenameCopy:
2002                 print "After submitting, renamed files must be re-synced."
2003                 print "Invoke \"p4 sync -f\" on each of these files:"
2004                 for f in pureRenameCopy:
2005                     print "  " + f
2006
2007             print
2008             print "To revert the changes, use \"p4 revert ...\", and delete"
2009             print "the submit template file \"%s\"" % fileName
2010             if filesToAdd:
2011                 print "Since the commit adds new files, they must be deleted:"
2012                 for f in filesToAdd:
2013                     print "  " + f
2014             print
2015             return True
2016
2017         #
2018         # Let the user edit the change description, then submit it.
2019         #
2020         submitted = False
2021
2022         try:
2023             if self.edit_template(fileName):
2024                 # read the edited message and submit
2025                 tmpFile = open(fileName, "rb")
2026                 message = tmpFile.read()
2027                 tmpFile.close()
2028                 if self.isWindows:
2029                     message = message.replace("\r\n", "\n")
2030                 submitTemplate = message[:message.index(separatorLine)]
2031
2032                 if update_shelve:
2033                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2034                 elif self.shelve:
2035                     p4_write_pipe(['shelve', '-i'], submitTemplate)
2036                 else:
2037                     p4_write_pipe(['submit', '-i'], submitTemplate)
2038                     # The rename/copy happened by applying a patch that created a
2039                     # new file.  This leaves it writable, which confuses p4.
2040                     for f in pureRenameCopy:
2041                         p4_sync(f, "-f")
2042
2043                 if self.preserveUser:
2044                     if p4User:
2045                         # Get last changelist number. Cannot easily get it from
2046                         # the submit command output as the output is
2047                         # unmarshalled.
2048                         changelist = self.lastP4Changelist()
2049                         self.modifyChangelistUser(changelist, p4User)
2050
2051                 submitted = True
2052
2053         finally:
2054             # skip this patch
2055             if not submitted or self.shelve:
2056                 if self.shelve:
2057                     print ("Reverting shelved files.")
2058                 else:
2059                     print ("Submission cancelled, undoing p4 changes.")
2060                 for f in editedFiles | filesToDelete:
2061                     p4_revert(f)
2062                 for f in filesToAdd:
2063                     p4_revert(f)
2064                     os.remove(f)
2065
2066         os.remove(fileName)
2067         return submitted
2068
2069     # Export git tags as p4 labels. Create a p4 label and then tag
2070     # with that.
2071     def exportGitTags(self, gitTags):
2072         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2073         if len(validLabelRegexp) == 0:
2074             validLabelRegexp = defaultLabelRegexp
2075         m = re.compile(validLabelRegexp)
2076
2077         for name in gitTags:
2078
2079             if not m.match(name):
2080                 if verbose:
2081                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
2082                 continue
2083
2084             # Get the p4 commit this corresponds to
2085             logMessage = extractLogMessageFromGitCommit(name)
2086             values = extractSettingsGitLog(logMessage)
2087
2088             if not values.has_key('change'):
2089                 # a tag pointing to something not sent to p4; ignore
2090                 if verbose:
2091                     print "git tag %s does not give a p4 commit" % name
2092                 continue
2093             else:
2094                 changelist = values['change']
2095
2096             # Get the tag details.
2097             inHeader = True
2098             isAnnotated = False
2099             body = []
2100             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2101                 l = l.strip()
2102                 if inHeader:
2103                     if re.match(r'tag\s+', l):
2104                         isAnnotated = True
2105                     elif re.match(r'\s*$', l):
2106                         inHeader = False
2107                         continue
2108                 else:
2109                     body.append(l)
2110
2111             if not isAnnotated:
2112                 body = ["lightweight tag imported by git p4\n"]
2113
2114             # Create the label - use the same view as the client spec we are using
2115             clientSpec = getClientSpec()
2116
2117             labelTemplate  = "Label: %s\n" % name
2118             labelTemplate += "Description:\n"
2119             for b in body:
2120                 labelTemplate += "\t" + b + "\n"
2121             labelTemplate += "View:\n"
2122             for depot_side in clientSpec.mappings:
2123                 labelTemplate += "\t%s\n" % depot_side
2124
2125             if self.dry_run:
2126                 print "Would create p4 label %s for tag" % name
2127             elif self.prepare_p4_only:
2128                 print "Not creating p4 label %s for tag due to option" \
2129                       " --prepare-p4-only" % name
2130             else:
2131                 p4_write_pipe(["label", "-i"], labelTemplate)
2132
2133                 # Use the label
2134                 p4_system(["tag", "-l", name] +
2135                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2136
2137                 if verbose:
2138                     print "created p4 label for tag %s" % name
2139
2140     def run(self, args):
2141         if len(args) == 0:
2142             self.master = currentGitBranch()
2143         elif len(args) == 1:
2144             self.master = args[0]
2145             if not branchExists(self.master):
2146                 die("Branch %s does not exist" % self.master)
2147         else:
2148             return False
2149
2150         for i in self.update_shelve:
2151             if i <= 0:
2152                 sys.exit("invalid changelist %d" % i)
2153
2154         if self.master:
2155             allowSubmit = gitConfig("git-p4.allowSubmit")
2156             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2157                 die("%s is not in git-p4.allowSubmit" % self.master)
2158
2159         [upstream, settings] = findUpstreamBranchPoint()
2160         self.depotPath = settings['depot-paths'][0]
2161         if len(self.origin) == 0:
2162             self.origin = upstream
2163
2164         if len(self.update_shelve) > 0:
2165             self.shelve = True
2166
2167         if self.preserveUser:
2168             if not self.canChangeChangelists():
2169                 die("Cannot preserve user names without p4 super-user or admin permissions")
2170
2171         # if not set from the command line, try the config file
2172         if self.conflict_behavior is None:
2173             val = gitConfig("git-p4.conflict")
2174             if val:
2175                 if val not in self.conflict_behavior_choices:
2176                     die("Invalid value '%s' for config git-p4.conflict" % val)
2177             else:
2178                 val = "ask"
2179             self.conflict_behavior = val
2180
2181         if self.verbose:
2182             print "Origin branch is " + self.origin
2183
2184         if len(self.depotPath) == 0:
2185             print "Internal error: cannot locate perforce depot path from existing branches"
2186             sys.exit(128)
2187
2188         self.useClientSpec = False
2189         if gitConfigBool("git-p4.useclientspec"):
2190             self.useClientSpec = True
2191         if self.useClientSpec:
2192             self.clientSpecDirs = getClientSpec()
2193
2194         # Check for the existence of P4 branches
2195         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2196
2197         if self.useClientSpec and not branchesDetected:
2198             # all files are relative to the client spec
2199             self.clientPath = getClientRoot()
2200         else:
2201             self.clientPath = p4Where(self.depotPath)
2202
2203         if self.clientPath == "":
2204             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2205
2206         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2207         self.oldWorkingDirectory = os.getcwd()
2208
2209         # ensure the clientPath exists
2210         new_client_dir = False
2211         if not os.path.exists(self.clientPath):
2212             new_client_dir = True
2213             os.makedirs(self.clientPath)
2214
2215         chdir(self.clientPath, is_client_path=True)
2216         if self.dry_run:
2217             print "Would synchronize p4 checkout in %s" % self.clientPath
2218         else:
2219             print "Synchronizing p4 checkout..."
2220             if new_client_dir:
2221                 # old one was destroyed, and maybe nobody told p4
2222                 p4_sync("...", "-f")
2223             else:
2224                 p4_sync("...")
2225         self.check()
2226
2227         commits = []
2228         if self.master:
2229             committish = self.master
2230         else:
2231             committish = 'HEAD'
2232
2233         if self.commit != "":
2234             if self.commit.find("..") != -1:
2235                 limits_ish = self.commit.split("..")
2236                 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2237                     commits.append(line.strip())
2238                 commits.reverse()
2239             else:
2240                 commits.append(self.commit)
2241         else:
2242             for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2243                 commits.append(line.strip())
2244             commits.reverse()
2245
2246         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2247             self.checkAuthorship = False
2248         else:
2249             self.checkAuthorship = True
2250
2251         if self.preserveUser:
2252             self.checkValidP4Users(commits)
2253
2254         #
2255         # Build up a set of options to be passed to diff when
2256         # submitting each commit to p4.
2257         #
2258         if self.detectRenames:
2259             # command-line -M arg
2260             self.diffOpts = "-M"
2261         else:
2262             # If not explicitly set check the config variable
2263             detectRenames = gitConfig("git-p4.detectRenames")
2264
2265             if detectRenames.lower() == "false" or detectRenames == "":
2266                 self.diffOpts = ""
2267             elif detectRenames.lower() == "true":
2268                 self.diffOpts = "-M"
2269             else:
2270                 self.diffOpts = "-M%s" % detectRenames
2271
2272         # no command-line arg for -C or --find-copies-harder, just
2273         # config variables
2274         detectCopies = gitConfig("git-p4.detectCopies")
2275         if detectCopies.lower() == "false" or detectCopies == "":
2276             pass
2277         elif detectCopies.lower() == "true":
2278             self.diffOpts += " -C"
2279         else:
2280             self.diffOpts += " -C%s" % detectCopies
2281
2282         if gitConfigBool("git-p4.detectCopiesHarder"):
2283             self.diffOpts += " --find-copies-harder"
2284
2285         num_shelves = len(self.update_shelve)
2286         if num_shelves > 0 and num_shelves != len(commits):
2287             sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2288                      (len(commits), num_shelves))
2289
2290         #
2291         # Apply the commits, one at a time.  On failure, ask if should
2292         # continue to try the rest of the patches, or quit.
2293         #
2294         if self.dry_run:
2295             print "Would apply"
2296         applied = []
2297         last = len(commits) - 1
2298         for i, commit in enumerate(commits):
2299             if self.dry_run:
2300                 print " ", read_pipe(["git", "show", "-s",
2301                                       "--format=format:%h %s", commit])
2302                 ok = True
2303             else:
2304                 ok = self.applyCommit(commit)
2305             if ok:
2306                 applied.append(commit)
2307             else:
2308                 if self.prepare_p4_only and i < last:
2309                     print "Processing only the first commit due to option" \
2310                           " --prepare-p4-only"
2311                     break
2312                 if i < last:
2313                     quit = False
2314                     while True:
2315                         # prompt for what to do, or use the option/variable
2316                         if self.conflict_behavior == "ask":
2317                             print "What do you want to do?"
2318                             response = raw_input("[s]kip this commit but apply"
2319                                                  " the rest, or [q]uit? ")
2320                             if not response:
2321                                 continue
2322                         elif self.conflict_behavior == "skip":
2323                             response = "s"
2324                         elif self.conflict_behavior == "quit":
2325                             response = "q"
2326                         else:
2327                             die("Unknown conflict_behavior '%s'" %
2328                                 self.conflict_behavior)
2329
2330                         if response[0] == "s":
2331                             print "Skipping this commit, but applying the rest"
2332                             break
2333                         if response[0] == "q":
2334                             print "Quitting"
2335                             quit = True
2336                             break
2337                     if quit:
2338                         break
2339
2340         chdir(self.oldWorkingDirectory)
2341         shelved_applied = "shelved" if self.shelve else "applied"
2342         if self.dry_run:
2343             pass
2344         elif self.prepare_p4_only:
2345             pass
2346         elif len(commits) == len(applied):
2347             print ("All commits {0}!".format(shelved_applied))
2348
2349             sync = P4Sync()
2350             if self.branch:
2351                 sync.branch = self.branch
2352             if self.disable_p4sync:
2353                 sync.sync_origin_only()
2354             else:
2355                 sync.run([])
2356
2357                 if not self.disable_rebase:
2358                     rebase = P4Rebase()
2359                     rebase.rebase()
2360
2361         else:
2362             if len(applied) == 0:
2363                 print ("No commits {0}.".format(shelved_applied))
2364             else:
2365                 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2366                 for c in commits:
2367                     if c in applied:
2368                         star = "*"
2369                     else:
2370                         star = " "
2371                     print star, read_pipe(["git", "show", "-s",
2372                                            "--format=format:%h %s",  c])
2373                 print "You will have to do 'git p4 sync' and rebase."
2374
2375         if gitConfigBool("git-p4.exportLabels"):
2376             self.exportLabels = True
2377
2378         if self.exportLabels:
2379             p4Labels = getP4Labels(self.depotPath)
2380             gitTags = getGitTags()
2381
2382             missingGitTags = gitTags - p4Labels
2383             self.exportGitTags(missingGitTags)
2384
2385         # exit with error unless everything applied perfectly
2386         if len(commits) != len(applied):
2387                 sys.exit(1)
2388
2389         return True
2390
2391 class View(object):
2392     """Represent a p4 view ("p4 help views"), and map files in a
2393        repo according to the view."""
2394
2395     def __init__(self, client_name):
2396         self.mappings = []
2397         self.client_prefix = "//%s/" % client_name
2398         # cache results of "p4 where" to lookup client file locations
2399         self.client_spec_path_cache = {}
2400
2401     def append(self, view_line):
2402         """Parse a view line, splitting it into depot and client
2403            sides.  Append to self.mappings, preserving order.  This
2404            is only needed for tag creation."""
2405
2406         # Split the view line into exactly two words.  P4 enforces
2407         # structure on these lines that simplifies this quite a bit.
2408         #
2409         # Either or both words may be double-quoted.
2410         # Single quotes do not matter.
2411         # Double-quote marks cannot occur inside the words.
2412         # A + or - prefix is also inside the quotes.
2413         # There are no quotes unless they contain a space.
2414         # The line is already white-space stripped.
2415         # The two words are separated by a single space.
2416         #
2417         if view_line[0] == '"':
2418             # First word is double quoted.  Find its end.
2419             close_quote_index = view_line.find('"', 1)
2420             if close_quote_index <= 0:
2421                 die("No first-word closing quote found: %s" % view_line)
2422             depot_side = view_line[1:close_quote_index]
2423             # skip closing quote and space
2424             rhs_index = close_quote_index + 1 + 1
2425         else:
2426             space_index = view_line.find(" ")
2427             if space_index <= 0:
2428                 die("No word-splitting space found: %s" % view_line)
2429             depot_side = view_line[0:space_index]
2430             rhs_index = space_index + 1
2431
2432         # prefix + means overlay on previous mapping
2433         if depot_side.startswith("+"):
2434             depot_side = depot_side[1:]
2435
2436         # prefix - means exclude this path, leave out of mappings
2437         exclude = False
2438         if depot_side.startswith("-"):
2439             exclude = True
2440             depot_side = depot_side[1:]
2441
2442         if not exclude:
2443             self.mappings.append(depot_side)
2444
2445     def convert_client_path(self, clientFile):
2446         # chop off //client/ part to make it relative
2447         if not clientFile.startswith(self.client_prefix):
2448             die("No prefix '%s' on clientFile '%s'" %
2449                 (self.client_prefix, clientFile))
2450         return clientFile[len(self.client_prefix):]
2451
2452     def update_client_spec_path_cache(self, files):
2453         """ Caching file paths by "p4 where" batch query """
2454
2455         # List depot file paths exclude that already cached
2456         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2457
2458         if len(fileArgs) == 0:
2459             return  # All files in cache
2460
2461         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2462         for res in where_result:
2463             if "code" in res and res["code"] == "error":
2464                 # assume error is "... file(s) not in client view"
2465                 continue
2466             if "clientFile" not in res:
2467                 die("No clientFile in 'p4 where' output")
2468             if "unmap" in res:
2469                 # it will list all of them, but only one not unmap-ped
2470                 continue
2471             if gitConfigBool("core.ignorecase"):
2472                 res['depotFile'] = res['depotFile'].lower()
2473             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2474
2475         # not found files or unmap files set to ""
2476         for depotFile in fileArgs:
2477             if gitConfigBool("core.ignorecase"):
2478                 depotFile = depotFile.lower()
2479             if depotFile not in self.client_spec_path_cache:
2480                 self.client_spec_path_cache[depotFile] = ""
2481
2482     def map_in_client(self, depot_path):
2483         """Return the relative location in the client where this
2484            depot file should live.  Returns "" if the file should
2485            not be mapped in the client."""
2486
2487         if gitConfigBool("core.ignorecase"):
2488             depot_path = depot_path.lower()
2489
2490         if depot_path in self.client_spec_path_cache:
2491             return self.client_spec_path_cache[depot_path]
2492
2493         die( "Error: %s is not found in client spec path" % depot_path )
2494         return ""
2495
2496 class P4Sync(Command, P4UserMap):
2497     delete_actions = ( "delete", "move/delete", "purge" )
2498
2499     def __init__(self):
2500         Command.__init__(self)
2501         P4UserMap.__init__(self)
2502         self.options = [
2503                 optparse.make_option("--branch", dest="branch"),
2504                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2505                 optparse.make_option("--changesfile", dest="changesFile"),
2506                 optparse.make_option("--silent", dest="silent", action="store_true"),
2507                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2508                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2509                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2510                                      help="Import into refs/heads/ , not refs/remotes"),
2511                 optparse.make_option("--max-changes", dest="maxChanges",
2512                                      help="Maximum number of changes to import"),
2513                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2514                                      help="Internal block size to use when iteratively calling p4 changes"),
2515                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2516                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2517                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2518                                      help="Only sync files that are included in the Perforce Client Spec"),
2519                 optparse.make_option("-/", dest="cloneExclude",
2520                                      action="append", type="string",
2521                                      help="exclude depot path"),
2522         ]
2523         self.description = """Imports from Perforce into a git repository.\n
2524     example:
2525     //depot/my/project/ -- to import the current head
2526     //depot/my/project/@all -- to import everything
2527     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2528
2529     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2530
2531         self.usage += " //depot/path[@revRange]"
2532         self.silent = False
2533         self.createdBranches = set()
2534         self.committedChanges = set()
2535         self.branch = ""
2536         self.detectBranches = False
2537         self.detectLabels = False
2538         self.importLabels = False
2539         self.changesFile = ""
2540         self.syncWithOrigin = True
2541         self.importIntoRemotes = True
2542         self.maxChanges = ""
2543         self.changes_block_size = None
2544         self.keepRepoPath = False
2545         self.depotPaths = None
2546         self.p4BranchesInGit = []
2547         self.cloneExclude = []
2548         self.useClientSpec = False
2549         self.useClientSpec_from_options = False
2550         self.clientSpecDirs = None
2551         self.tempBranches = []
2552         self.tempBranchLocation = "refs/git-p4-tmp"
2553         self.largeFileSystem = None
2554         self.suppress_meta_comment = False
2555
2556         if gitConfig('git-p4.largeFileSystem'):
2557             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2558             self.largeFileSystem = largeFileSystemConstructor(
2559                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2560             )
2561
2562         if gitConfig("git-p4.syncFromOrigin") == "false":
2563             self.syncWithOrigin = False
2564
2565         self.depotPaths = []
2566         self.changeRange = ""
2567         self.previousDepotPaths = []
2568         self.hasOrigin = False
2569
2570         # map from branch depot path to parent branch
2571         self.knownBranches = {}
2572         self.initialParents = {}
2573
2574         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2575         self.labels = {}
2576
2577     # Force a checkpoint in fast-import and wait for it to finish
2578     def checkpoint(self):
2579         self.gitStream.write("checkpoint\n\n")
2580         self.gitStream.write("progress checkpoint\n\n")
2581         out = self.gitOutput.readline()
2582         if self.verbose:
2583             print "checkpoint finished: " + out
2584
2585     def cmp_shelved(self, path, filerev, revision):
2586         """ Determine if a path at revision #filerev is the same as the file
2587             at revision @revision for a shelved changelist. If they don't match,
2588             unshelving won't be safe (we will get other changes mixed in).
2589
2590             This is comparing the revision that the shelved changelist is *based* on, not
2591             the shelved changelist itself.
2592         """
2593         ret = p4Cmd(["diff2", "{0}#{1}".format(path, filerev), "{0}@{1}".format(path, revision)])
2594         if verbose:
2595             print("p4 diff2 path %s filerev %s revision %s => %s" % (path, filerev, revision, ret))
2596         return ret["status"] == "identical"
2597
2598     def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0, origin_revision = 0):
2599         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2600                              for path in self.cloneExclude]
2601         files = []
2602         fnum = 0
2603         while commit.has_key("depotFile%s" % fnum):
2604             path =  commit["depotFile%s" % fnum]
2605
2606             if [p for p in self.cloneExclude
2607                 if p4PathStartsWith(path, p)]:
2608                 found = False
2609             else:
2610                 found = [p for p in self.depotPaths
2611                          if p4PathStartsWith(path, p)]
2612             if not found:
2613                 fnum = fnum + 1
2614                 continue
2615
2616             file = {}
2617             file["path"] = path
2618             file["rev"] = commit["rev%s" % fnum]
2619             file["action"] = commit["action%s" % fnum]
2620             file["type"] = commit["type%s" % fnum]
2621             if shelved:
2622                 file["shelved_cl"] = int(shelved_cl)
2623
2624                 # For shelved changelists, check that the revision of each file that the
2625                 # shelve was based on matches the revision that we are using for the
2626                 # starting point for git-fast-import (self.initialParent). Otherwise
2627                 # the resulting diff will contain deltas from multiple commits.
2628
2629                 if file["action"] != "add" and \
2630                     not self.cmp_shelved(path, file["rev"], origin_revision):
2631                     sys.exit("change {0} not based on {1} for {2}, cannot unshelve".format(
2632                         commit["change"], self.initialParent, path))
2633
2634             files.append(file)
2635             fnum = fnum + 1
2636         return files
2637
2638     def extractJobsFromCommit(self, commit):
2639         jobs = []
2640         jnum = 0
2641         while commit.has_key("job%s" % jnum):
2642             job = commit["job%s" % jnum]
2643             jobs.append(job)
2644             jnum = jnum + 1
2645         return jobs
2646
2647     def stripRepoPath(self, path, prefixes):
2648         """When streaming files, this is called to map a p4 depot path
2649            to where it should go in git.  The prefixes are either
2650            self.depotPaths, or self.branchPrefixes in the case of
2651            branch detection."""
2652
2653         if self.useClientSpec:
2654             # branch detection moves files up a level (the branch name)
2655             # from what client spec interpretation gives
2656             path = self.clientSpecDirs.map_in_client(path)
2657             if self.detectBranches:
2658                 for b in self.knownBranches:
2659                     if path.startswith(b + "/"):
2660                         path = path[len(b)+1:]
2661
2662         elif self.keepRepoPath:
2663             # Preserve everything in relative path name except leading
2664             # //depot/; just look at first prefix as they all should
2665             # be in the same depot.
2666             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2667             if p4PathStartsWith(path, depot):
2668                 path = path[len(depot):]
2669
2670         else:
2671             for p in prefixes:
2672                 if p4PathStartsWith(path, p):
2673                     path = path[len(p):]
2674                     break
2675
2676         path = wildcard_decode(path)
2677         return path
2678
2679     def splitFilesIntoBranches(self, commit):
2680         """Look at each depotFile in the commit to figure out to what
2681            branch it belongs."""
2682
2683         if self.clientSpecDirs:
2684             files = self.extractFilesFromCommit(commit)
2685             self.clientSpecDirs.update_client_spec_path_cache(files)
2686
2687         branches = {}
2688         fnum = 0
2689         while commit.has_key("depotFile%s" % fnum):
2690             path =  commit["depotFile%s" % fnum]
2691             found = [p for p in self.depotPaths
2692                      if p4PathStartsWith(path, p)]
2693             if not found:
2694                 fnum = fnum + 1
2695                 continue
2696
2697             file = {}
2698             file["path"] = path
2699             file["rev"] = commit["rev%s" % fnum]
2700             file["action"] = commit["action%s" % fnum]
2701             file["type"] = commit["type%s" % fnum]
2702             fnum = fnum + 1
2703
2704             # start with the full relative path where this file would
2705             # go in a p4 client
2706             if self.useClientSpec:
2707                 relPath = self.clientSpecDirs.map_in_client(path)
2708             else:
2709                 relPath = self.stripRepoPath(path, self.depotPaths)
2710
2711             for branch in self.knownBranches.keys():
2712                 # add a trailing slash so that a commit into qt/4.2foo
2713                 # doesn't end up in qt/4.2, e.g.
2714                 if relPath.startswith(branch + "/"):
2715                     if branch not in branches:
2716                         branches[branch] = []
2717                     branches[branch].append(file)
2718                     break
2719
2720         return branches
2721
2722     def writeToGitStream(self, gitMode, relPath, contents):
2723         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2724         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2725         for d in contents:
2726             self.gitStream.write(d)
2727         self.gitStream.write('\n')
2728
2729     def encodeWithUTF8(self, path):
2730         try:
2731             path.decode('ascii')
2732         except:
2733             encoding = 'utf8'
2734             if gitConfig('git-p4.pathEncoding'):
2735                 encoding = gitConfig('git-p4.pathEncoding')
2736             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2737             if self.verbose:
2738                 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2739         return path
2740
2741     # output one file from the P4 stream
2742     # - helper for streamP4Files
2743
2744     def streamOneP4File(self, file, contents):
2745         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2746         relPath = self.encodeWithUTF8(relPath)
2747         if verbose:
2748             size = int(self.stream_file['fileSize'])
2749             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2750             sys.stdout.flush()
2751
2752         (type_base, type_mods) = split_p4_type(file["type"])
2753
2754         git_mode = "100644"
2755         if "x" in type_mods:
2756             git_mode = "100755"
2757         if type_base == "symlink":
2758             git_mode = "120000"
2759             # p4 print on a symlink sometimes contains "target\n";
2760             # if it does, remove the newline
2761             data = ''.join(contents)
2762             if not data:
2763                 # Some version of p4 allowed creating a symlink that pointed
2764                 # to nothing.  This causes p4 errors when checking out such
2765                 # a change, and errors here too.  Work around it by ignoring
2766                 # the bad symlink; hopefully a future change fixes it.
2767                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2768                 return
2769             elif data[-1] == '\n':
2770                 contents = [data[:-1]]
2771             else:
2772                 contents = [data]
2773
2774         if type_base == "utf16":
2775             # p4 delivers different text in the python output to -G
2776             # than it does when using "print -o", or normal p4 client
2777             # operations.  utf16 is converted to ascii or utf8, perhaps.
2778             # But ascii text saved as -t utf16 is completely mangled.
2779             # Invoke print -o to get the real contents.
2780             #
2781             # On windows, the newlines will always be mangled by print, so put
2782             # them back too.  This is not needed to the cygwin windows version,
2783             # just the native "NT" type.
2784             #
2785             try:
2786                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2787             except Exception as e:
2788                 if 'Translation of file content failed' in str(e):
2789                     type_base = 'binary'
2790                 else:
2791                     raise e
2792             else:
2793                 if p4_version_string().find('/NT') >= 0:
2794                     text = text.replace('\r\n', '\n')
2795                 contents = [ text ]
2796
2797         if type_base == "apple":
2798             # Apple filetype files will be streamed as a concatenation of
2799             # its appledouble header and the contents.  This is useless
2800             # on both macs and non-macs.  If using "print -q -o xx", it
2801             # will create "xx" with the data, and "%xx" with the header.
2802             # This is also not very useful.
2803             #
2804             # Ideally, someday, this script can learn how to generate
2805             # appledouble files directly and import those to git, but
2806             # non-mac machines can never find a use for apple filetype.
2807             print "\nIgnoring apple filetype file %s" % file['depotFile']
2808             return
2809
2810         # Note that we do not try to de-mangle keywords on utf16 files,
2811         # even though in theory somebody may want that.
2812         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2813         if pattern:
2814             regexp = re.compile(pattern, re.VERBOSE)
2815             text = ''.join(contents)
2816             text = regexp.sub(r'$\1$', text)
2817             contents = [ text ]
2818
2819         if self.largeFileSystem:
2820             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2821
2822         self.writeToGitStream(git_mode, relPath, contents)
2823
2824     def streamOneP4Deletion(self, file):
2825         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2826         relPath = self.encodeWithUTF8(relPath)
2827         if verbose:
2828             sys.stdout.write("delete %s\n" % relPath)
2829             sys.stdout.flush()
2830         self.gitStream.write("D %s\n" % relPath)
2831
2832         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2833             self.largeFileSystem.removeLargeFile(relPath)
2834
2835     # handle another chunk of streaming data
2836     def streamP4FilesCb(self, marshalled):
2837
2838         # catch p4 errors and complain
2839         err = None
2840         if "code" in marshalled:
2841             if marshalled["code"] == "error":
2842                 if "data" in marshalled:
2843                     err = marshalled["data"].rstrip()
2844
2845         if not err and 'fileSize' in self.stream_file:
2846             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2847             if required_bytes > 0:
2848                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2849                     os.getcwd(), required_bytes/1024/1024
2850                 )
2851
2852         if err:
2853             f = None
2854             if self.stream_have_file_info:
2855                 if "depotFile" in self.stream_file:
2856                     f = self.stream_file["depotFile"]
2857             # force a failure in fast-import, else an empty
2858             # commit will be made
2859             self.gitStream.write("\n")
2860             self.gitStream.write("die-now\n")
2861             self.gitStream.close()
2862             # ignore errors, but make sure it exits first
2863             self.importProcess.wait()
2864             if f:
2865                 die("Error from p4 print for %s: %s" % (f, err))
2866             else:
2867                 die("Error from p4 print: %s" % err)
2868
2869         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2870             # start of a new file - output the old one first
2871             self.streamOneP4File(self.stream_file, self.stream_contents)
2872             self.stream_file = {}
2873             self.stream_contents = []
2874             self.stream_have_file_info = False
2875
2876         # pick up the new file information... for the
2877         # 'data' field we need to append to our array
2878         for k in marshalled.keys():
2879             if k == 'data':
2880                 if 'streamContentSize' not in self.stream_file:
2881                     self.stream_file['streamContentSize'] = 0
2882                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2883                 self.stream_contents.append(marshalled['data'])
2884             else:
2885                 self.stream_file[k] = marshalled[k]
2886
2887         if (verbose and
2888             'streamContentSize' in self.stream_file and
2889             'fileSize' in self.stream_file and
2890             'depotFile' in self.stream_file):
2891             size = int(self.stream_file["fileSize"])
2892             if size > 0:
2893                 progress = 100*self.stream_file['streamContentSize']/size
2894                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2895                 sys.stdout.flush()
2896
2897         self.stream_have_file_info = True
2898
2899     # Stream directly from "p4 files" into "git fast-import"
2900     def streamP4Files(self, files):
2901         filesForCommit = []
2902         filesToRead = []
2903         filesToDelete = []
2904
2905         for f in files:
2906             filesForCommit.append(f)
2907             if f['action'] in self.delete_actions:
2908                 filesToDelete.append(f)
2909             else:
2910                 filesToRead.append(f)
2911
2912         # deleted files...
2913         for f in filesToDelete:
2914             self.streamOneP4Deletion(f)
2915
2916         if len(filesToRead) > 0:
2917             self.stream_file = {}
2918             self.stream_contents = []
2919             self.stream_have_file_info = False
2920
2921             # curry self argument
2922             def streamP4FilesCbSelf(entry):
2923                 self.streamP4FilesCb(entry)
2924
2925             fileArgs = []
2926             for f in filesToRead:
2927                 if 'shelved_cl' in f:
2928                     # Handle shelved CLs using the "p4 print file@=N" syntax to print
2929                     # the contents
2930                     fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2931                 else:
2932                     fileArg = '%s#%s' % (f['path'], f['rev'])
2933
2934                 fileArgs.append(fileArg)
2935
2936             p4CmdList(["-x", "-", "print"],
2937                       stdin=fileArgs,
2938                       cb=streamP4FilesCbSelf)
2939
2940             # do the last chunk
2941             if self.stream_file.has_key('depotFile'):
2942                 self.streamOneP4File(self.stream_file, self.stream_contents)
2943
2944     def make_email(self, userid):
2945         if userid in self.users:
2946             return self.users[userid]
2947         else:
2948             return "%s <a@b>" % userid
2949
2950     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2951         """ Stream a p4 tag.
2952         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2953         """
2954
2955         if verbose:
2956             print "writing tag %s for commit %s" % (labelName, commit)
2957         gitStream.write("tag %s\n" % labelName)
2958         gitStream.write("from %s\n" % commit)
2959
2960         if labelDetails.has_key('Owner'):
2961             owner = labelDetails["Owner"]
2962         else:
2963             owner = None
2964
2965         # Try to use the owner of the p4 label, or failing that,
2966         # the current p4 user id.
2967         if owner:
2968             email = self.make_email(owner)
2969         else:
2970             email = self.make_email(self.p4UserId())
2971         tagger = "%s %s %s" % (email, epoch, self.tz)
2972
2973         gitStream.write("tagger %s\n" % tagger)
2974
2975         print "labelDetails=",labelDetails
2976         if labelDetails.has_key('Description'):
2977             description = labelDetails['Description']
2978         else:
2979             description = 'Label from git p4'
2980
2981         gitStream.write("data %d\n" % len(description))
2982         gitStream.write(description)
2983         gitStream.write("\n")
2984
2985     def inClientSpec(self, path):
2986         if not self.clientSpecDirs:
2987             return True
2988         inClientSpec = self.clientSpecDirs.map_in_client(path)
2989         if not inClientSpec and self.verbose:
2990             print('Ignoring file outside of client spec: {0}'.format(path))
2991         return inClientSpec
2992
2993     def hasBranchPrefix(self, path):
2994         if not self.branchPrefixes:
2995             return True
2996         hasPrefix = [p for p in self.branchPrefixes
2997                         if p4PathStartsWith(path, p)]
2998         if not hasPrefix and self.verbose:
2999             print('Ignoring file outside of prefix: {0}'.format(path))
3000         return hasPrefix
3001
3002     def commit(self, details, files, branch, parent = ""):
3003         epoch = details["time"]
3004         author = details["user"]
3005         jobs = self.extractJobsFromCommit(details)
3006
3007         if self.verbose:
3008             print('commit into {0}'.format(branch))
3009
3010         if self.clientSpecDirs:
3011             self.clientSpecDirs.update_client_spec_path_cache(files)
3012
3013         files = [f for f in files
3014             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3015
3016         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
3017             print('Ignoring revision {0} as it would produce an empty commit.'
3018                 .format(details['change']))
3019             return
3020
3021         self.gitStream.write("commit %s\n" % branch)
3022         self.gitStream.write("mark :%s\n" % details["change"])
3023         self.committedChanges.add(int(details["change"]))
3024         committer = ""
3025         if author not in self.users:
3026             self.getUserMapFromPerforceServer()
3027         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
3028
3029         self.gitStream.write("committer %s\n" % committer)
3030
3031         self.gitStream.write("data <<EOT\n")
3032         self.gitStream.write(details["desc"])
3033         if len(jobs) > 0:
3034             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
3035
3036         if not self.suppress_meta_comment:
3037             self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3038                                 (','.join(self.branchPrefixes), details["change"]))
3039             if len(details['options']) > 0:
3040                 self.gitStream.write(": options = %s" % details['options'])
3041             self.gitStream.write("]\n")
3042
3043         self.gitStream.write("EOT\n\n")
3044
3045         if len(parent) > 0:
3046             if self.verbose:
3047                 print "parent %s" % parent
3048             self.gitStream.write("from %s\n" % parent)
3049
3050         self.streamP4Files(files)
3051         self.gitStream.write("\n")
3052
3053         change = int(details["change"])
3054
3055         if self.labels.has_key(change):
3056             label = self.labels[change]
3057             labelDetails = label[0]
3058             labelRevisions = label[1]
3059             if self.verbose:
3060                 print "Change %s is labelled %s" % (change, labelDetails)
3061
3062             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3063                                                 for p in self.branchPrefixes])
3064
3065             if len(files) == len(labelRevisions):
3066
3067                 cleanedFiles = {}
3068                 for info in files:
3069                     if info["action"] in self.delete_actions:
3070                         continue
3071                     cleanedFiles[info["depotFile"]] = info["rev"]
3072
3073                 if cleanedFiles == labelRevisions:
3074                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3075
3076                 else:
3077                     if not self.silent:
3078                         print ("Tag %s does not match with change %s: files do not match."
3079                                % (labelDetails["label"], change))
3080
3081             else:
3082                 if not self.silent:
3083                     print ("Tag %s does not match with change %s: file count is different."
3084                            % (labelDetails["label"], change))
3085
3086     # Build a dictionary of changelists and labels, for "detect-labels" option.
3087     def getLabels(self):
3088         self.labels = {}
3089
3090         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3091         if len(l) > 0 and not self.silent:
3092             print "Finding files belonging to labels in %s" % `self.depotPaths`
3093
3094         for output in l:
3095             label = output["label"]
3096             revisions = {}
3097             newestChange = 0
3098             if self.verbose:
3099                 print "Querying files for label %s" % label
3100             for file in p4CmdList(["files"] +
3101                                       ["%s...@%s" % (p, label)
3102                                           for p in self.depotPaths]):
3103                 revisions[file["depotFile"]] = file["rev"]
3104                 change = int(file["change"])
3105                 if change > newestChange:
3106                     newestChange = change
3107
3108             self.labels[newestChange] = [output, revisions]
3109
3110         if self.verbose:
3111             print "Label changes: %s" % self.labels.keys()
3112
3113     # Import p4 labels as git tags. A direct mapping does not
3114     # exist, so assume that if all the files are at the same revision
3115     # then we can use that, or it's something more complicated we should
3116     # just ignore.
3117     def importP4Labels(self, stream, p4Labels):
3118         if verbose:
3119             print "import p4 labels: " + ' '.join(p4Labels)
3120
3121         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3122         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3123         if len(validLabelRegexp) == 0:
3124             validLabelRegexp = defaultLabelRegexp
3125         m = re.compile(validLabelRegexp)
3126
3127         for name in p4Labels:
3128             commitFound = False
3129
3130             if not m.match(name):
3131                 if verbose:
3132                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
3133                 continue
3134
3135             if name in ignoredP4Labels:
3136                 continue
3137
3138             labelDetails = p4CmdList(['label', "-o", name])[0]
3139
3140             # get the most recent changelist for each file in this label
3141             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3142                                 for p in self.depotPaths])
3143
3144             if change.has_key('change'):
3145                 # find the corresponding git commit; take the oldest commit
3146                 changelist = int(change['change'])
3147                 if changelist in self.committedChanges:
3148                     gitCommit = ":%d" % changelist       # use a fast-import mark
3149                     commitFound = True
3150                 else:
3151                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3152                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3153                     if len(gitCommit) == 0:
3154                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
3155                     else:
3156                         commitFound = True
3157                         gitCommit = gitCommit.strip()
3158
3159                 if commitFound:
3160                     # Convert from p4 time format
3161                     try:
3162                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3163                     except ValueError:
3164                         print "Could not convert label time %s" % labelDetails['Update']
3165                         tmwhen = 1
3166
3167                     when = int(time.mktime(tmwhen))
3168                     self.streamTag(stream, name, labelDetails, gitCommit, when)
3169                     if verbose:
3170                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
3171             else:
3172                 if verbose:
3173                     print "Label %s has no changelists - possibly deleted?" % name
3174
3175             if not commitFound:
3176                 # We can't import this label; don't try again as it will get very
3177                 # expensive repeatedly fetching all the files for labels that will
3178                 # never be imported. If the label is moved in the future, the
3179                 # ignore will need to be removed manually.
3180                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3181
3182     def guessProjectName(self):
3183         for p in self.depotPaths:
3184             if p.endswith("/"):
3185                 p = p[:-1]
3186             p = p[p.strip().rfind("/") + 1:]
3187             if not p.endswith("/"):
3188                p += "/"
3189             return p
3190
3191     def getBranchMapping(self):
3192         lostAndFoundBranches = set()
3193
3194         user = gitConfig("git-p4.branchUser")
3195         if len(user) > 0:
3196             command = "branches -u %s" % user
3197         else:
3198             command = "branches"
3199
3200         for info in p4CmdList(command):
3201             details = p4Cmd(["branch", "-o", info["branch"]])
3202             viewIdx = 0
3203             while details.has_key("View%s" % viewIdx):
3204                 paths = details["View%s" % viewIdx].split(" ")
3205                 viewIdx = viewIdx + 1
3206                 # require standard //depot/foo/... //depot/bar/... mapping
3207                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3208                     continue
3209                 source = paths[0]
3210                 destination = paths[1]
3211                 ## HACK
3212                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3213                     source = source[len(self.depotPaths[0]):-4]
3214                     destination = destination[len(self.depotPaths[0]):-4]
3215
3216                     if destination in self.knownBranches:
3217                         if not self.silent:
3218                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3219                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3220                         continue
3221
3222                     self.knownBranches[destination] = source
3223
3224                     lostAndFoundBranches.discard(destination)
3225
3226                     if source not in self.knownBranches:
3227                         lostAndFoundBranches.add(source)
3228
3229         # Perforce does not strictly require branches to be defined, so we also
3230         # check git config for a branch list.
3231         #
3232         # Example of branch definition in git config file:
3233         # [git-p4]
3234         #   branchList=main:branchA
3235         #   branchList=main:branchB
3236         #   branchList=branchA:branchC
3237         configBranches = gitConfigList("git-p4.branchList")
3238         for branch in configBranches:
3239             if branch:
3240                 (source, destination) = branch.split(":")
3241                 self.knownBranches[destination] = source
3242
3243                 lostAndFoundBranches.discard(destination)
3244
3245                 if source not in self.knownBranches:
3246                     lostAndFoundBranches.add(source)
3247
3248
3249         for branch in lostAndFoundBranches:
3250             self.knownBranches[branch] = branch
3251
3252     def getBranchMappingFromGitBranches(self):
3253         branches = p4BranchesInGit(self.importIntoRemotes)
3254         for branch in branches.keys():
3255             if branch == "master":
3256                 branch = "main"
3257             else:
3258                 branch = branch[len(self.projectName):]
3259             self.knownBranches[branch] = branch
3260
3261     def updateOptionDict(self, d):
3262         option_keys = {}
3263         if self.keepRepoPath:
3264             option_keys['keepRepoPath'] = 1
3265
3266         d["options"] = ' '.join(sorted(option_keys.keys()))
3267
3268     def readOptions(self, d):
3269         self.keepRepoPath = (d.has_key('options')
3270                              and ('keepRepoPath' in d['options']))
3271
3272     def gitRefForBranch(self, branch):
3273         if branch == "main":
3274             return self.refPrefix + "master"
3275
3276         if len(branch) <= 0:
3277             return branch
3278
3279         return self.refPrefix + self.projectName + branch
3280
3281     def gitCommitByP4Change(self, ref, change):
3282         if self.verbose:
3283             print "looking in ref " + ref + " for change %s using bisect..." % change
3284
3285         earliestCommit = ""
3286         latestCommit = parseRevision(ref)
3287
3288         while True:
3289             if self.verbose:
3290                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3291             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3292             if len(next) == 0:
3293                 if self.verbose:
3294                     print "argh"
3295                 return ""
3296             log = extractLogMessageFromGitCommit(next)
3297             settings = extractSettingsGitLog(log)
3298             currentChange = int(settings['change'])
3299             if self.verbose:
3300                 print "current change %s" % currentChange
3301
3302             if currentChange == change:
3303                 if self.verbose:
3304                     print "found %s" % next
3305                 return next
3306
3307             if currentChange < change:
3308                 earliestCommit = "^%s" % next
3309             else:
3310                 latestCommit = "%s" % next
3311
3312         return ""
3313
3314     def importNewBranch(self, branch, maxChange):
3315         # make fast-import flush all changes to disk and update the refs using the checkpoint
3316         # command so that we can try to find the branch parent in the git history
3317         self.gitStream.write("checkpoint\n\n");
3318         self.gitStream.flush();
3319         branchPrefix = self.depotPaths[0] + branch + "/"
3320         range = "@1,%s" % maxChange
3321         #print "prefix" + branchPrefix
3322         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3323         if len(changes) <= 0:
3324             return False
3325         firstChange = changes[0]
3326         #print "first change in branch: %s" % firstChange
3327         sourceBranch = self.knownBranches[branch]
3328         sourceDepotPath = self.depotPaths[0] + sourceBranch
3329         sourceRef = self.gitRefForBranch(sourceBranch)
3330         #print "source " + sourceBranch
3331
3332         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3333         #print "branch parent: %s" % branchParentChange
3334         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3335         if len(gitParent) > 0:
3336             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3337             #print "parent git commit: %s" % gitParent
3338
3339         self.importChanges(changes)
3340         return True
3341
3342     def searchParent(self, parent, branch, target):
3343         parentFound = False
3344         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3345                                      "--no-merges", parent]):
3346             blob = blob.strip()
3347             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3348                 parentFound = True
3349                 if self.verbose:
3350                     print "Found parent of %s in commit %s" % (branch, blob)
3351                 break
3352         if parentFound:
3353             return blob
3354         else:
3355             return None
3356
3357     def importChanges(self, changes, shelved=False, origin_revision=0):
3358         cnt = 1
3359         for change in changes:
3360             description = p4_describe(change, shelved)
3361             self.updateOptionDict(description)
3362
3363             if not self.silent:
3364                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3365                 sys.stdout.flush()
3366             cnt = cnt + 1
3367
3368             try:
3369                 if self.detectBranches:
3370                     branches = self.splitFilesIntoBranches(description)
3371                     for branch in branches.keys():
3372                         ## HACK  --hwn
3373                         branchPrefix = self.depotPaths[0] + branch + "/"
3374                         self.branchPrefixes = [ branchPrefix ]
3375
3376                         parent = ""
3377
3378                         filesForCommit = branches[branch]
3379
3380                         if self.verbose:
3381                             print "branch is %s" % branch
3382
3383                         self.updatedBranches.add(branch)
3384
3385                         if branch not in self.createdBranches:
3386                             self.createdBranches.add(branch)
3387                             parent = self.knownBranches[branch]
3388                             if parent == branch:
3389                                 parent = ""
3390                             else:
3391                                 fullBranch = self.projectName + branch
3392                                 if fullBranch not in self.p4BranchesInGit:
3393                                     if not self.silent:
3394                                         print("\n    Importing new branch %s" % fullBranch);
3395                                     if self.importNewBranch(branch, change - 1):
3396                                         parent = ""
3397                                         self.p4BranchesInGit.append(fullBranch)
3398                                     if not self.silent:
3399                                         print("\n    Resuming with change %s" % change);
3400
3401                                 if self.verbose:
3402                                     print "parent determined through known branches: %s" % parent
3403
3404                         branch = self.gitRefForBranch(branch)
3405                         parent = self.gitRefForBranch(parent)
3406
3407                         if self.verbose:
3408                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3409
3410                         if len(parent) == 0 and branch in self.initialParents:
3411                             parent = self.initialParents[branch]
3412                             del self.initialParents[branch]
3413
3414                         blob = None
3415                         if len(parent) > 0:
3416                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3417                             if self.verbose:
3418                                 print "Creating temporary branch: " + tempBranch
3419                             self.commit(description, filesForCommit, tempBranch)
3420                             self.tempBranches.append(tempBranch)
3421                             self.checkpoint()
3422                             blob = self.searchParent(parent, branch, tempBranch)
3423                         if blob:
3424                             self.commit(description, filesForCommit, branch, blob)
3425                         else:
3426                             if self.verbose:
3427                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3428                             self.commit(description, filesForCommit, branch, parent)
3429                 else:
3430                     files = self.extractFilesFromCommit(description, shelved, change, origin_revision)
3431                     self.commit(description, files, self.branch,
3432                                 self.initialParent)
3433                     # only needed once, to connect to the previous commit
3434                     self.initialParent = ""
3435             except IOError:
3436                 print self.gitError.read()
3437                 sys.exit(1)
3438
3439     def sync_origin_only(self):
3440         if self.syncWithOrigin:
3441             self.hasOrigin = originP4BranchesExist()
3442             if self.hasOrigin:
3443                 if not self.silent:
3444                     print 'Syncing with origin first, using "git fetch origin"'
3445                 system("git fetch origin")
3446
3447     def importHeadRevision(self, revision):
3448         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3449
3450         details = {}
3451         details["user"] = "git perforce import user"
3452         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3453                            % (' '.join(self.depotPaths), revision))
3454         details["change"] = revision
3455         newestRevision = 0
3456
3457         fileCnt = 0
3458         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3459
3460         for info in p4CmdList(["files"] + fileArgs):
3461
3462             if 'code' in info and info['code'] == 'error':
3463                 sys.stderr.write("p4 returned an error: %s\n"
3464                                  % info['data'])
3465                 if info['data'].find("must refer to client") >= 0:
3466                     sys.stderr.write("This particular p4 error is misleading.\n")
3467                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3468                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3469                 sys.exit(1)
3470             if 'p4ExitCode' in info:
3471                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3472                 sys.exit(1)
3473
3474
3475             change = int(info["change"])
3476             if change > newestRevision:
3477                 newestRevision = change
3478
3479             if info["action"] in self.delete_actions:
3480                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3481                 #fileCnt = fileCnt + 1
3482                 continue
3483
3484             for prop in ["depotFile", "rev", "action", "type" ]:
3485                 details["%s%s" % (prop, fileCnt)] = info[prop]
3486
3487             fileCnt = fileCnt + 1
3488
3489         details["change"] = newestRevision
3490
3491         # Use time from top-most change so that all git p4 clones of
3492         # the same p4 repo have the same commit SHA1s.
3493         res = p4_describe(newestRevision)
3494         details["time"] = res["time"]
3495
3496         self.updateOptionDict(details)
3497         try:
3498             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3499         except IOError:
3500             print "IO error with git fast-import. Is your git version recent enough?"
3501             print self.gitError.read()
3502
3503     def openStreams(self):
3504         self.importProcess = subprocess.Popen(["git", "fast-import"],
3505                                               stdin=subprocess.PIPE,
3506                                               stdout=subprocess.PIPE,
3507                                               stderr=subprocess.PIPE);
3508         self.gitOutput = self.importProcess.stdout
3509         self.gitStream = self.importProcess.stdin
3510         self.gitError = self.importProcess.stderr
3511
3512     def closeStreams(self):
3513         self.gitStream.close()
3514         if self.importProcess.wait() != 0:
3515             die("fast-import failed: %s" % self.gitError.read())
3516         self.gitOutput.close()
3517         self.gitError.close()
3518
3519     def run(self, args):
3520         if self.importIntoRemotes:
3521             self.refPrefix = "refs/remotes/p4/"
3522         else:
3523             self.refPrefix = "refs/heads/p4/"
3524
3525         self.sync_origin_only()
3526
3527         branch_arg_given = bool(self.branch)
3528         if len(self.branch) == 0:
3529             self.branch = self.refPrefix + "master"
3530             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3531                 system("git update-ref %s refs/heads/p4" % self.branch)
3532                 system("git branch -D p4")
3533
3534         # accept either the command-line option, or the configuration variable
3535         if self.useClientSpec:
3536             # will use this after clone to set the variable
3537             self.useClientSpec_from_options = True
3538         else:
3539             if gitConfigBool("git-p4.useclientspec"):
3540                 self.useClientSpec = True
3541         if self.useClientSpec:
3542             self.clientSpecDirs = getClientSpec()
3543
3544         # TODO: should always look at previous commits,
3545         # merge with previous imports, if possible.
3546         if args == []:
3547             if self.hasOrigin:
3548                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3549
3550             # branches holds mapping from branch name to sha1
3551             branches = p4BranchesInGit(self.importIntoRemotes)
3552
3553             # restrict to just this one, disabling detect-branches
3554             if branch_arg_given:
3555                 short = self.branch.split("/")[-1]
3556                 if short in branches:
3557                     self.p4BranchesInGit = [ short ]
3558             else:
3559                 self.p4BranchesInGit = branches.keys()
3560
3561             if len(self.p4BranchesInGit) > 1:
3562                 if not self.silent:
3563                     print "Importing from/into multiple branches"
3564                 self.detectBranches = True
3565                 for branch in branches.keys():
3566                     self.initialParents[self.refPrefix + branch] = \
3567                         branches[branch]
3568
3569             if self.verbose:
3570                 print "branches: %s" % self.p4BranchesInGit
3571
3572             p4Change = 0
3573             for branch in self.p4BranchesInGit:
3574                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3575
3576                 settings = extractSettingsGitLog(logMsg)
3577
3578                 self.readOptions(settings)
3579                 if (settings.has_key('depot-paths')
3580                     and settings.has_key ('change')):
3581                     change = int(settings['change']) + 1
3582                     p4Change = max(p4Change, change)
3583
3584                     depotPaths = sorted(settings['depot-paths'])
3585                     if self.previousDepotPaths == []:
3586                         self.previousDepotPaths = depotPaths
3587                     else:
3588                         paths = []
3589                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3590                             prev_list = prev.split("/")
3591                             cur_list = cur.split("/")
3592                             for i in range(0, min(len(cur_list), len(prev_list))):
3593                                 if cur_list[i] <> prev_list[i]:
3594                                     i = i - 1
3595                                     break
3596
3597                             paths.append ("/".join(cur_list[:i + 1]))
3598
3599                         self.previousDepotPaths = paths
3600
3601             if p4Change > 0:
3602                 self.depotPaths = sorted(self.previousDepotPaths)
3603                 self.changeRange = "@%s,#head" % p4Change
3604                 if not self.silent and not self.detectBranches:
3605                     print "Performing incremental import into %s git branch" % self.branch
3606
3607         # accept multiple ref name abbreviations:
3608         #    refs/foo/bar/branch -> use it exactly
3609         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3610         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3611         if not self.branch.startswith("refs/"):
3612             if self.importIntoRemotes:
3613                 prepend = "refs/remotes/"
3614             else:
3615                 prepend = "refs/heads/"
3616             if not self.branch.startswith("p4/"):
3617                 prepend += "p4/"
3618             self.branch = prepend + self.branch
3619
3620         if len(args) == 0 and self.depotPaths:
3621             if not self.silent:
3622                 print "Depot paths: %s" % ' '.join(self.depotPaths)
3623         else:
3624             if self.depotPaths and self.depotPaths != args:
3625                 print ("previous import used depot path %s and now %s was specified. "
3626                        "This doesn't work!" % (' '.join (self.depotPaths),
3627                                                ' '.join (args)))
3628                 sys.exit(1)
3629
3630             self.depotPaths = sorted(args)
3631
3632         revision = ""
3633         self.users = {}
3634
3635         # Make sure no revision specifiers are used when --changesfile
3636         # is specified.
3637         bad_changesfile = False
3638         if len(self.changesFile) > 0:
3639             for p in self.depotPaths:
3640                 if p.find("@") >= 0 or p.find("#") >= 0:
3641                     bad_changesfile = True
3642                     break
3643         if bad_changesfile:
3644             die("Option --changesfile is incompatible with revision specifiers")
3645
3646         newPaths = []
3647         for p in self.depotPaths:
3648             if p.find("@") != -1:
3649                 atIdx = p.index("@")
3650                 self.changeRange = p[atIdx:]
3651                 if self.changeRange == "@all":
3652                     self.changeRange = ""
3653                 elif ',' not in self.changeRange:
3654                     revision = self.changeRange
3655                     self.changeRange = ""
3656                 p = p[:atIdx]
3657             elif p.find("#") != -1:
3658                 hashIdx = p.index("#")
3659                 revision = p[hashIdx:]
3660                 p = p[:hashIdx]
3661             elif self.previousDepotPaths == []:
3662                 # pay attention to changesfile, if given, else import
3663                 # the entire p4 tree at the head revision
3664                 if len(self.changesFile) == 0:
3665                     revision = "#head"
3666
3667             p = re.sub ("\.\.\.$", "", p)
3668             if not p.endswith("/"):
3669                 p += "/"
3670
3671             newPaths.append(p)
3672
3673         self.depotPaths = newPaths
3674
3675         # --detect-branches may change this for each branch
3676         self.branchPrefixes = self.depotPaths
3677
3678         self.loadUserMapFromCache()
3679         self.labels = {}
3680         if self.detectLabels:
3681             self.getLabels();
3682
3683         if self.detectBranches:
3684             ## FIXME - what's a P4 projectName ?
3685             self.projectName = self.guessProjectName()
3686
3687             if self.hasOrigin:
3688                 self.getBranchMappingFromGitBranches()
3689             else:
3690                 self.getBranchMapping()
3691             if self.verbose:
3692                 print "p4-git branches: %s" % self.p4BranchesInGit
3693                 print "initial parents: %s" % self.initialParents
3694             for b in self.p4BranchesInGit:
3695                 if b != "master":
3696
3697                     ## FIXME
3698                     b = b[len(self.projectName):]
3699                 self.createdBranches.add(b)
3700
3701         self.openStreams()
3702
3703         if revision:
3704             self.importHeadRevision(revision)
3705         else:
3706             changes = []
3707
3708             if len(self.changesFile) > 0:
3709                 output = open(self.changesFile).readlines()
3710                 changeSet = set()
3711                 for line in output:
3712                     changeSet.add(int(line))
3713
3714                 for change in changeSet:
3715                     changes.append(change)
3716
3717                 changes.sort()
3718             else:
3719                 # catch "git p4 sync" with no new branches, in a repo that
3720                 # does not have any existing p4 branches
3721                 if len(args) == 0:
3722                     if not self.p4BranchesInGit:
3723                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3724
3725                     # The default branch is master, unless --branch is used to
3726                     # specify something else.  Make sure it exists, or complain
3727                     # nicely about how to use --branch.
3728                     if not self.detectBranches:
3729                         if not branch_exists(self.branch):
3730                             if branch_arg_given:
3731                                 die("Error: branch %s does not exist." % self.branch)
3732                             else:
3733                                 die("Error: no branch %s; perhaps specify one with --branch." %
3734                                     self.branch)
3735
3736                 if self.verbose:
3737                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3738                                                               self.changeRange)
3739                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3740
3741                 if len(self.maxChanges) > 0:
3742                     changes = changes[:min(int(self.maxChanges), len(changes))]
3743
3744             if len(changes) == 0:
3745                 if not self.silent:
3746                     print "No changes to import!"
3747             else:
3748                 if not self.silent and not self.detectBranches:
3749                     print "Import destination: %s" % self.branch
3750
3751                 self.updatedBranches = set()
3752
3753                 if not self.detectBranches:
3754                     if args:
3755                         # start a new branch
3756                         self.initialParent = ""
3757                     else:
3758                         # build on a previous revision
3759                         self.initialParent = parseRevision(self.branch)
3760
3761                 self.importChanges(changes)
3762
3763                 if not self.silent:
3764                     print ""
3765                     if len(self.updatedBranches) > 0:
3766                         sys.stdout.write("Updated branches: ")
3767                         for b in self.updatedBranches:
3768                             sys.stdout.write("%s " % b)
3769                         sys.stdout.write("\n")
3770
3771         if gitConfigBool("git-p4.importLabels"):
3772             self.importLabels = True
3773
3774         if self.importLabels:
3775             p4Labels = getP4Labels(self.depotPaths)
3776             gitTags = getGitTags()
3777
3778             missingP4Labels = p4Labels - gitTags
3779             self.importP4Labels(self.gitStream, missingP4Labels)
3780
3781         self.closeStreams()
3782
3783         # Cleanup temporary branches created during import
3784         if self.tempBranches != []:
3785             for branch in self.tempBranches:
3786                 read_pipe("git update-ref -d %s" % branch)
3787             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3788
3789         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3790         # a convenient shortcut refname "p4".
3791         if self.importIntoRemotes:
3792             head_ref = self.refPrefix + "HEAD"
3793             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3794                 system(["git", "symbolic-ref", head_ref, self.branch])
3795
3796         return True
3797
3798 class P4Rebase(Command):
3799     def __init__(self):
3800         Command.__init__(self)
3801         self.options = [
3802                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3803         ]
3804         self.importLabels = False
3805         self.description = ("Fetches the latest revision from perforce and "
3806                             + "rebases the current work (branch) against it")
3807
3808     def run(self, args):
3809         sync = P4Sync()
3810         sync.importLabels = self.importLabels
3811         sync.run([])
3812
3813         return self.rebase()
3814
3815     def rebase(self):
3816         if os.system("git update-index --refresh") != 0:
3817             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");
3818         if len(read_pipe("git diff-index HEAD --")) > 0:
3819             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3820
3821         [upstream, settings] = findUpstreamBranchPoint()
3822         if len(upstream) == 0:
3823             die("Cannot find upstream branchpoint for rebase")
3824
3825         # the branchpoint may be p4/foo~3, so strip off the parent
3826         upstream = re.sub("~[0-9]+$", "", upstream)
3827
3828         print "Rebasing the current branch onto %s" % upstream
3829         oldHead = read_pipe("git rev-parse HEAD").strip()
3830         system("git rebase %s" % upstream)
3831         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3832         return True
3833
3834 class P4Clone(P4Sync):
3835     def __init__(self):
3836         P4Sync.__init__(self)
3837         self.description = "Creates a new git repository and imports from Perforce into it"
3838         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3839         self.options += [
3840             optparse.make_option("--destination", dest="cloneDestination",
3841                                  action='store', default=None,
3842                                  help="where to leave result of the clone"),
3843             optparse.make_option("--bare", dest="cloneBare",
3844                                  action="store_true", default=False),
3845         ]
3846         self.cloneDestination = None
3847         self.needsGit = False
3848         self.cloneBare = False
3849
3850     def defaultDestination(self, args):
3851         ## TODO: use common prefix of args?
3852         depotPath = args[0]
3853         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3854         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3855         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3856         depotDir = re.sub(r"/$", "", depotDir)
3857         return os.path.split(depotDir)[1]
3858
3859     def run(self, args):
3860         if len(args) < 1:
3861             return False
3862
3863         if self.keepRepoPath and not self.cloneDestination:
3864             sys.stderr.write("Must specify destination for --keep-path\n")
3865             sys.exit(1)
3866
3867         depotPaths = args
3868
3869         if not self.cloneDestination and len(depotPaths) > 1:
3870             self.cloneDestination = depotPaths[-1]
3871             depotPaths = depotPaths[:-1]
3872
3873         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3874         for p in depotPaths:
3875             if not p.startswith("//"):
3876                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3877                 return False
3878
3879         if not self.cloneDestination:
3880             self.cloneDestination = self.defaultDestination(args)
3881
3882         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3883
3884         if not os.path.exists(self.cloneDestination):
3885             os.makedirs(self.cloneDestination)
3886         chdir(self.cloneDestination)
3887
3888         init_cmd = [ "git", "init" ]
3889         if self.cloneBare:
3890             init_cmd.append("--bare")
3891         retcode = subprocess.call(init_cmd)
3892         if retcode:
3893             raise CalledProcessError(retcode, init_cmd)
3894
3895         if not P4Sync.run(self, depotPaths):
3896             return False
3897
3898         # create a master branch and check out a work tree
3899         if gitBranchExists(self.branch):
3900             system([ "git", "branch", "master", self.branch ])
3901             if not self.cloneBare:
3902                 system([ "git", "checkout", "-f" ])
3903         else:
3904             print 'Not checking out any branch, use ' \
3905                   '"git checkout -q -b master <branch>"'
3906
3907         # auto-set this variable if invoked with --use-client-spec
3908         if self.useClientSpec_from_options:
3909             system("git config --bool git-p4.useclientspec true")
3910
3911         return True
3912
3913 class P4Unshelve(Command):
3914     def __init__(self):
3915         Command.__init__(self)
3916         self.options = []
3917         self.origin = "HEAD"
3918         self.description = "Unshelve a P4 changelist into a git commit"
3919         self.usage = "usage: %prog [options] changelist"
3920         self.options += [
3921                 optparse.make_option("--origin", dest="origin",
3922                     help="Use this base revision instead of the default (%s)" % self.origin),
3923         ]
3924         self.verbose = False
3925         self.noCommit = False
3926         self.destbranch = "refs/remotes/p4/unshelved"
3927
3928     def renameBranch(self, branch_name):
3929         """ Rename the existing branch to branch_name.N
3930         """
3931
3932         found = True
3933         for i in range(0,1000):
3934             backup_branch_name = "{0}.{1}".format(branch_name, i)
3935             if not gitBranchExists(backup_branch_name):
3936                 gitUpdateRef(backup_branch_name, branch_name) # copy ref to backup
3937                 gitDeleteRef(branch_name)
3938                 found = True
3939                 print("renamed old unshelve branch to {0}".format(backup_branch_name))
3940                 break
3941
3942         if not found:
3943             sys.exit("gave up trying to rename existing branch {0}".format(sync.branch))
3944
3945     def findLastP4Revision(self, starting_point):
3946         """ Look back from starting_point for the first commit created by git-p4
3947             to find the P4 commit we are based on, and the depot-paths.
3948         """
3949
3950         for parent in (range(65535)):
3951             log = extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))
3952             settings = extractSettingsGitLog(log)
3953             if settings.has_key('change'):
3954                 return settings
3955
3956         sys.exit("could not find git-p4 commits in {0}".format(self.origin))
3957
3958     def run(self, args):
3959         if len(args) != 1:
3960             return False
3961
3962         if not gitBranchExists(self.origin):
3963             sys.exit("origin branch {0} does not exist".format(self.origin))
3964
3965         sync = P4Sync()
3966         changes = args
3967         sync.initialParent = self.origin
3968
3969         # use the first change in the list to construct the branch to unshelve into
3970         change = changes[0]
3971
3972         # if the target branch already exists, rename it
3973         branch_name = "{0}/{1}".format(self.destbranch, change)
3974         if gitBranchExists(branch_name):
3975             self.renameBranch(branch_name)
3976         sync.branch = branch_name
3977
3978         sync.verbose = self.verbose
3979         sync.suppress_meta_comment = True
3980
3981         settings = self.findLastP4Revision(self.origin)
3982         origin_revision = settings['change']
3983         sync.depotPaths = settings['depot-paths']
3984         sync.branchPrefixes = sync.depotPaths
3985
3986         sync.openStreams()
3987         sync.loadUserMapFromCache()
3988         sync.silent = True
3989         sync.importChanges(changes, shelved=True, origin_revision=origin_revision)
3990         sync.closeStreams()
3991
3992         print("unshelved changelist {0} into {1}".format(change, branch_name))
3993
3994         return True
3995
3996 class P4Branches(Command):
3997     def __init__(self):
3998         Command.__init__(self)
3999         self.options = [ ]
4000         self.description = ("Shows the git branches that hold imports and their "
4001                             + "corresponding perforce depot paths")
4002         self.verbose = False
4003
4004     def run(self, args):
4005         if originP4BranchesExist():
4006             createOrUpdateBranchesFromOrigin()
4007
4008         cmdline = "git rev-parse --symbolic "
4009         cmdline += " --remotes"
4010
4011         for line in read_pipe_lines(cmdline):
4012             line = line.strip()
4013
4014             if not line.startswith('p4/') or line == "p4/HEAD":
4015                 continue
4016             branch = line
4017
4018             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
4019             settings = extractSettingsGitLog(log)
4020
4021             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
4022         return True
4023
4024 class HelpFormatter(optparse.IndentedHelpFormatter):
4025     def __init__(self):
4026         optparse.IndentedHelpFormatter.__init__(self)
4027
4028     def format_description(self, description):
4029         if description:
4030             return description + "\n"
4031         else:
4032             return ""
4033
4034 def printUsage(commands):
4035     print "usage: %s <command> [options]" % sys.argv[0]
4036     print ""
4037     print "valid commands: %s" % ", ".join(commands)
4038     print ""
4039     print "Try %s <command> --help for command specific help." % sys.argv[0]
4040     print ""
4041
4042 commands = {
4043     "debug" : P4Debug,
4044     "submit" : P4Submit,
4045     "commit" : P4Submit,
4046     "sync" : P4Sync,
4047     "rebase" : P4Rebase,
4048     "clone" : P4Clone,
4049     "rollback" : P4RollBack,
4050     "branches" : P4Branches,
4051     "unshelve" : P4Unshelve,
4052 }
4053
4054
4055 def main():
4056     if len(sys.argv[1:]) == 0:
4057         printUsage(commands.keys())
4058         sys.exit(2)
4059
4060     cmdName = sys.argv[1]
4061     try:
4062         klass = commands[cmdName]
4063         cmd = klass()
4064     except KeyError:
4065         print "unknown command %s" % cmdName
4066         print ""
4067         printUsage(commands.keys())
4068         sys.exit(2)
4069
4070     options = cmd.options
4071     cmd.gitdir = os.environ.get("GIT_DIR", None)
4072
4073     args = sys.argv[2:]
4074
4075     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
4076     if cmd.needsGit:
4077         options.append(optparse.make_option("--git-dir", dest="gitdir"))
4078
4079     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
4080                                    options,
4081                                    description = cmd.description,
4082                                    formatter = HelpFormatter())
4083
4084     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
4085     global verbose
4086     verbose = cmd.verbose
4087     if cmd.needsGit:
4088         if cmd.gitdir == None:
4089             cmd.gitdir = os.path.abspath(".git")
4090             if not isValidGitDir(cmd.gitdir):
4091                 # "rev-parse --git-dir" without arguments will try $PWD/.git
4092                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
4093                 if os.path.exists(cmd.gitdir):
4094                     cdup = read_pipe("git rev-parse --show-cdup").strip()
4095                     if len(cdup) > 0:
4096                         chdir(cdup);
4097
4098         if not isValidGitDir(cmd.gitdir):
4099             if isValidGitDir(cmd.gitdir + "/.git"):
4100                 cmd.gitdir += "/.git"
4101             else:
4102                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
4103
4104         # so git commands invoked from the P4 workspace will succeed
4105         os.environ["GIT_DIR"] = cmd.gitdir
4106
4107     if not cmd.run(args):
4108         parser.print_help()
4109         sys.exit(2)
4110
4111
4112 if __name__ == '__main__':
4113     main()